From 05e6be108fbfac8862094ba32de6d0053d429c75 Mon Sep 17 00:00:00 2001 From: Nuno Loureiro Date: Fri, 4 Mar 2022 01:14:04 +0000 Subject: [PATCH 01/18] Move scripts to / --- probely_api_examples/__init__.py => __init__.py | 0 .../add_hosts_to_target.py => add_hosts_to_target.py | 0 .../copy_single_target.py => copy_single_target.py | 0 ..._api_key_enterprise.py => create_account_api_key_enterprise.py | 0 ...create_account_api_key_smb.py => create_account_api_key_smb.py | 0 .../create_scanning_agent.py => create_scanning_agent.py | 0 probely_api_examples/create_target.py => create_target.py | 0 probely_api_examples/finding_cvss.py => finding_cvss.py | 0 probely_api_examples/findings_csv.py => findings_csv.py | 0 .../import_postman_env.py => import_postman_env.py | 0 probely_api_examples/start_scan.py => start_scan.py | 0 .../start_scan_profile.py => start_scan_profile.py | 0 .../start_scan_reduced_scope.py => start_scan_reduced_scope.py | 0 probely_api_examples/start_scan_totp.py => start_scan_totp.py | 0 .../target_history_findings.py => target_history_findings.py | 0 15 files changed, 0 insertions(+), 0 deletions(-) rename probely_api_examples/__init__.py => __init__.py (100%) rename probely_api_examples/add_hosts_to_target.py => add_hosts_to_target.py (100%) rename probely_api_examples/copy_single_target.py => copy_single_target.py (100%) rename probely_api_examples/create_account_api_key_enterprise.py => create_account_api_key_enterprise.py (100%) rename probely_api_examples/create_account_api_key_smb.py => create_account_api_key_smb.py (100%) rename probely_api_examples/create_scanning_agent.py => create_scanning_agent.py (100%) rename probely_api_examples/create_target.py => create_target.py (100%) rename probely_api_examples/finding_cvss.py => finding_cvss.py (100%) rename probely_api_examples/findings_csv.py => findings_csv.py (100%) rename probely_api_examples/import_postman_env.py => import_postman_env.py (100%) rename probely_api_examples/start_scan.py => start_scan.py (100%) rename probely_api_examples/start_scan_profile.py => start_scan_profile.py (100%) rename probely_api_examples/start_scan_reduced_scope.py => start_scan_reduced_scope.py (100%) rename probely_api_examples/start_scan_totp.py => start_scan_totp.py (100%) rename probely_api_examples/target_history_findings.py => target_history_findings.py (100%) diff --git a/probely_api_examples/__init__.py b/__init__.py similarity index 100% rename from probely_api_examples/__init__.py rename to __init__.py diff --git a/probely_api_examples/add_hosts_to_target.py b/add_hosts_to_target.py similarity index 100% rename from probely_api_examples/add_hosts_to_target.py rename to add_hosts_to_target.py diff --git a/probely_api_examples/copy_single_target.py b/copy_single_target.py similarity index 100% rename from probely_api_examples/copy_single_target.py rename to copy_single_target.py diff --git a/probely_api_examples/create_account_api_key_enterprise.py b/create_account_api_key_enterprise.py similarity index 100% rename from probely_api_examples/create_account_api_key_enterprise.py rename to create_account_api_key_enterprise.py diff --git a/probely_api_examples/create_account_api_key_smb.py b/create_account_api_key_smb.py similarity index 100% rename from probely_api_examples/create_account_api_key_smb.py rename to create_account_api_key_smb.py diff --git a/probely_api_examples/create_scanning_agent.py b/create_scanning_agent.py similarity index 100% rename from probely_api_examples/create_scanning_agent.py rename to create_scanning_agent.py diff --git a/probely_api_examples/create_target.py b/create_target.py similarity index 100% rename from probely_api_examples/create_target.py rename to create_target.py diff --git a/probely_api_examples/finding_cvss.py b/finding_cvss.py similarity index 100% rename from probely_api_examples/finding_cvss.py rename to finding_cvss.py diff --git a/probely_api_examples/findings_csv.py b/findings_csv.py similarity index 100% rename from probely_api_examples/findings_csv.py rename to findings_csv.py diff --git a/probely_api_examples/import_postman_env.py b/import_postman_env.py similarity index 100% rename from probely_api_examples/import_postman_env.py rename to import_postman_env.py diff --git a/probely_api_examples/start_scan.py b/start_scan.py similarity index 100% rename from probely_api_examples/start_scan.py rename to start_scan.py diff --git a/probely_api_examples/start_scan_profile.py b/start_scan_profile.py similarity index 100% rename from probely_api_examples/start_scan_profile.py rename to start_scan_profile.py diff --git a/probely_api_examples/start_scan_reduced_scope.py b/start_scan_reduced_scope.py similarity index 100% rename from probely_api_examples/start_scan_reduced_scope.py rename to start_scan_reduced_scope.py diff --git a/probely_api_examples/start_scan_totp.py b/start_scan_totp.py similarity index 100% rename from probely_api_examples/start_scan_totp.py rename to start_scan_totp.py diff --git a/probely_api_examples/target_history_findings.py b/target_history_findings.py similarity index 100% rename from probely_api_examples/target_history_findings.py rename to target_history_findings.py From 5c0c5ba9ab8014b5df12df0edc272d734bdaf56d Mon Sep 17 00:00:00 2001 From: Nuno Loureiro Date: Fri, 4 Mar 2022 01:17:39 +0000 Subject: [PATCH 02/18] Update CONTRIBUTING.rst Fixed path --- CONTRIBUTING.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index cc305b9..fbe6d86 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -15,7 +15,7 @@ Types of Contributions Report Bugs ~~~~~~~~~~~ -Report bugs at https://github.com/probely/probely_api_examples/issues. +Report bugs at https://github.com/probely/API_Scripts/issues. If you are reporting a bug, please include: @@ -45,7 +45,7 @@ articles, and such. Submit Feedback ~~~~~~~~~~~~~~~ -The best way to send feedback is to file an issue at https://github.com/probely/probely_api_examples/issues. +The best way to send feedback is to file an issue at https://github.com/probely/API_Scripts/issues. If you are proposing a feature: @@ -57,17 +57,17 @@ If you are proposing a feature: Get Started! ------------ -Ready to contribute? Here's how to set up `probely_api_examples` for local development. +Ready to contribute? Here's how to set up `API_Scripts` for local development. -1. Fork the `probely_api_examples` repo on GitHub. +1. Fork the `API_Scripts` repo on GitHub. 2. Clone your fork locally:: - $ git clone git@github.com:your_name_here/probely_api_examples.git + $ git clone git@github.com:your_name_here/API_Scripts.git 3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: - $ mkvirtualenv probely_api_examples - $ cd probely_api_examples/ + $ mkvirtualenv API_Scripts + $ cd API_Scripts/ $ python setup.py develop 4. Create a branch for local development:: @@ -78,7 +78,7 @@ Ready to contribute? Here's how to set up `probely_api_examples` for local devel 5. When you're done making changes, check that your changes pass flake8:: - $ flake8 probely_api_examples + $ flake8 API_Scripts To get flake8, just pip install it into your virtualenv. From a95b7b08d5e781a2f0e70e60786f616716c82843 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Sun, 23 Oct 2022 01:31:05 +0100 Subject: [PATCH 03/18] add navigation_converter to convert Probely's nav sequences to Selenium and Selenium to Probely's nav sequences --- .gitignore | 1 + utils/README-navigation_converter.md | 23 ++ utils/navigation_converter.py | 326 +++++++++++++++++++++++++++ 3 files changed, 350 insertions(+) create mode 100644 .gitignore create mode 100644 utils/README-navigation_converter.md create mode 100755 utils/navigation_converter.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0794a8e --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +utils/tmp diff --git a/utils/README-navigation_converter.md b/utils/README-navigation_converter.md new file mode 100644 index 0000000..05933a2 --- /dev/null +++ b/utils/README-navigation_converter.md @@ -0,0 +1,23 @@ +# Navigation Converter + +## Converts between Probely's Navigation sequences and Selenium. + +### Required options + + - `--convert/-c`: `selenium2probely` or `probely2selenium` + - `--input/-i`: `/path/to/original_file` + - `--output/-o`: `/path/to/destination_file` (Coverting to Selenium requires `.side` extension) + +### From Selenium to Probely's Navigation sequence + +```sh +$ python3 ./navigation_converter.py -c 'selenium2probely' -i /tmp/selenium_testfile.side -o /tmp/probely_navigation.json +``` + + +### From Probely's Navigation sequence to Selenium + +```sh +$ python3 ./navigation_converter.py -c 'probely2selenium' -i /tmp/probely_navigation.json -o /tmp/selenium_testfile.side +``` + diff --git a/utils/navigation_converter.py b/utils/navigation_converter.py new file mode 100755 index 0000000..7b74138 --- /dev/null +++ b/utils/navigation_converter.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python +""" +Converts between Probely's Navigation sequences and Selenium. + +We can not guarantee that the conversion will be 100% compatible. +""" +import argparse +import json +import re +import uuid +import os +from time import time + + +items_not_supported = [] + + +def getUUID(): + return str(uuid.uuid4()) + + +def getSeleniumCssAndXPath(targets, target): + css = None + xpath = None + for item in targets: + if item[1] == 'css:finder': + css = item[0].replace('css=', '') + if item[1] == 'xpath:position': + xpath = item[0].replace('xpath=', '') + + if css is None and target: + if target.startswith('name='): + css = '[{}]'.format(target) + elif target.startswith('id='): + css = '[{}]'.format(target) + return (css, xpath) + + +def convertProbely2Selenium(data, inputFp, outputFp=None): + name = os.path.basename(inputFp.name) + obj = { + 'id': getUUID(), + 'version': '2.0', + 'name': name, + 'url': None, + 'tests': [{ + 'id': getUUID(), + 'name': name, + 'commands': [] + }], + 'suites': [{ + 'id': getUUID(), + 'name': 'Default Suite', + 'persistSession': False, + 'parallel': False, + 'timeout': 300, + 'tests': [getUUID()] + }], + 'urls': [], + 'plugins': [] + } + if len(data) == 0: + raise Exception('No data in Probely recording file to be converted.') + + for idx, item in enumerate(data): + if idx == 0: + if item['type'] == 'goto': + m = re.search('(https?://[^/]+)(/?.*)', item.get('url')) + url_base = m.group(1) + url_path = m.group(2) + obj['url'] = url_base + obj['urls'].append(url_base) + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'open', + 'target': url_path, + 'targets': [], + 'value': '' + }) + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'setWindowSize', + 'target': '{}x{}'.format(item.get('windowWidth', '1800'), item.get('windowHeight', '1200')), + 'targets': [], + 'value': '' + }) + else: + raise Exception('First item must be a "goto" with an URL') + else: + css = 'css={}'.format(item.get('css')) + xpath = 'xpath={}'.format(item.get('xpath')) + if item.get('type') == 'click' or item.get('type') == 'bclick': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'click', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': '' + }) + elif item.get('type') == 'mouseover': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'mouseOver', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': '' + }) + elif item.get('type') == 'fill_value': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'type', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': item.get('value', '') + }) + elif item.get('type') == 'press_key' and item.get('value') == 13: + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'sendKeys', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': '${KEY_ENTER}' + }) + elif item.get('type') == 'change' and item.get('subtype') == 'select': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'select', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': 'value={}'.format(item.get('value', '')) + }) + elif item.get('type') == 'change' and item.get('subtype') == 'check': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'click', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': '' + }) + elif item.get('type') == 'dbclick': + obj['tests'][0]['commands'].append({ + 'id': getUUID(), + 'comment': '', + 'command': 'doubleClick', + 'target': css, + 'targets': [ + [css, 'css:finder'], + [xpath, 'xpath:position'] + ], + 'value': '' + }) + else: + items_not_supported.append('{} => {}'.format(idx, item.get('type'))) + + return obj + + +def convertSelenium2Probely(data): + obj = [] + timestamp = int(time()) * 1000 + tests_arr = data.get('tests') + if tests_arr is None or len(tests_arr) == 0: + raise Exception('No tests in Selenium file to be converted.') + + commands_arr = tests_arr[0].get('commands') + if commands_arr is None or len(commands_arr) == 0: + raise Exception('No commands in Selenium file to be converted.') + + url_base = data.get('url') + # get first url + first_command = commands_arr[0].get('command') + second_command = commands_arr[1].get('command') + url_path = commands_arr[0].get('target') + dimensions = commands_arr[1].get('target') + if first_command != 'open' or second_command != 'setWindowSize': + raise Exception('Selenium first command must be "open" or second command must be "setWindowSize"') + if url_path is None: + raise Exception('Selenium first command must have a "target"') + + windowWidth = 1800 + windowHeight = 1200 + if dimensions: + m = re.search('([0-9]+)x([0-9]+)', dimensions) + windowWidth = int(m.group(1)) + windowHeight = int(m.group(2)) + + obj.append({ + 'type': 'goto', + 'urlType': 'force', + 'url': '{}{}'.format(url_base, url_path), + 'timestamp': timestamp, + 'windowWidth': windowWidth, + 'windowHeight': windowHeight + }) + for idx, item in enumerate(commands_arr): + if idx == 0 or idx == 1: + continue + else: + css, xpath = getSeleniumCssAndXPath(item.get('targets'), item.get('target')) + if css is None: + continue + if item.get('command') == 'click': + timestamp += 1000 + obj.append({ + 'timestamp': timestamp, + 'type': 'click', + 'css': css, + 'xpath': xpath, + 'value': '', + 'frame': None + }) + elif item.get('command') == 'mouseOver': + timestamp += 1000 + obj.append({ + 'timestamp': timestamp, + 'type': 'mouseover', + 'css': css, + 'xpath': xpath, + 'value': '', + 'frame': None + }) + elif item.get('command') == 'type': + timestamp += 1000 + obj.append({ + 'timestamp': timestamp, + 'type': 'fill_value', + 'css': css, + 'xpath': xpath, + 'value': item.get('value', ''), + 'frame': None + }) + elif item.get('command') == 'sendKeys' and item.get('value') == '${KEY_ENTER}': + timestamp += 1000 + obj.append({ + 'timestamp': timestamp, + 'type': 'press_key', + 'css': css, + 'xpath': xpath, + 'value': 13, + 'frame': None + }) + elif item.get('command') == 'select': + timestamp += 1000 + select_value = item.get('value', '').replace('label=', '').replace('value=', '') + obj.append({ + 'timestamp': timestamp, + 'type': 'change', + 'subtype': 'select', + 'css': css, + 'xpath': xpath, + 'value': select_value, + 'selected': 1, # hardcoded, not correct + 'frame': None + }) + elif item.get('command') == 'doubleClick': + timestamp += 1000 + obj.append({ + 'timestamp': timestamp, + 'type': 'dblclick', + 'css': css, + 'xpath': xpath, + 'value': '', + 'frame': None + }) + else: + items_not_supported.append('{} => {}'.format(idx, item.get('command'))) + + return obj + + +def run(): + parser = argparse.ArgumentParser(description='Converts Probely navigation sequences to selenium or selenium to Probely navigation sequences') + parser.add_argument('-c', '--convert', help='/', choices=['probely2selenium', 'selenium2probely']) + parser.add_argument('-i', '--input', help='Input file', type=argparse.FileType('r')) + parser.add_argument('-o', '--output', help='Output file', type=argparse.FileType('w')) + args = parser.parse_args() + + if args.convert == 'probely2selenium' and not args.output.name.endswith('.side'): + raise Exception('Error: Selenium file name must use ".side" extension') + + input_data = json.load(args.input) + + if args.convert == 'probely2selenium': + out_data = convertProbely2Selenium(input_data, args.input) + elif args.convert == 'selenium2probely': + out_data = convertSelenium2Probely(input_data) + + final_json = json.dumps(out_data, indent=2) + args.output.write(final_json) + + if len(items_not_supported) > 0: + print('Warning:') + print('Items not supported/converted from positions: {}'.format(str(items_not_supported))) + + print('Done.') + print('File converted to file: {}'.format(args.output.name)) + + +if __name__ == '__main__': + run() From cd8876669693a82707af433bd5916e171f0bca22 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Wed, 9 Nov 2022 00:27:13 +0000 Subject: [PATCH 04/18] Add script to export targets to CSV with more information --- targets_csv.py | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 targets_csv.py diff --git a/targets_csv.py b/targets_csv.py new file mode 100755 index 0000000..94fe277 --- /dev/null +++ b/targets_csv.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import csv +import requests +from urllib.parse import urljoin + +def main(): + token = input("API Token:") + csv_filename = input("CSV path to filename (default: ./targets.csv):") + if csv_filename == "": + csv_filename = "./targets.csv" + + headers = {"Authorization": "JWT {}".format(token)} + + api_base_url = "https://api.probely.com" + targets_endpoint = urljoin( + api_base_url, "targets/?include=compliance&length=10000" + ) + + response = requests.get(targets_endpoint, headers=headers) + results = response.json()["results"] + + with open(csv_filename, "w") as csv_file: + csv_writer = csv.writer( + csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL + ) + for result in results: + labels = result["labels"] if result.get("labels") else [] + labels_name = [label["name"] for label in labels] + last_scan = result["last_scan"] if result.get("last_scan") else {} + row = [ + result["id"], + result["site"]["name"], + result["site"]["url"], + last_scan.get("status", ""), + last_scan.get("completed", ""), + *labels_name + ] + csv_writer.writerow(row) + +if __name__ == '__main__': + main() From 8f928c057acb79d6c34add76a9f8d9f664eba9b5 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Fri, 3 Mar 2023 12:40:44 +0000 Subject: [PATCH 05/18] add script to export targets with scheduled scans --- targets_scheduled_scans_csv.py | 50 ++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 targets_scheduled_scans_csv.py diff --git a/targets_scheduled_scans_csv.py b/targets_scheduled_scans_csv.py new file mode 100755 index 0000000..bf2ea81 --- /dev/null +++ b/targets_scheduled_scans_csv.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python + +import csv +import requests +from urllib.parse import urljoin + +recurrence_map = { + '': 'N/A', + 'd': 'daily', + 'w': 'weekly', + 'm': 'monthly', + 'q': 'quarterly' +} + +def main(): + token = input("API Token:") + csv_filename = input("CSV path to filename (default: ./scheduled.csv):") + if csv_filename == "": + csv_filename = "./scheduled.csv" + + headers = {"Authorization": "JWT {}".format(token)} + + api_base_url = "https://api.probely.com" + targets_endpoint = urljoin( + api_base_url, "targets/?include=compliance&length=10000" + ) + + response = requests.get(targets_endpoint, headers=headers) + results = response.json()["results"] + + with open(csv_filename, "w") as csv_file: + csv_writer = csv.writer( + csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL + ) + csv_writer.writerow(["ID", "NAME", "URL", "NEXT SCAN DATE", "RECURRENCE"]) + for result in results: + next_scan = result["next_scan"] if result.get("next_scan") else None + if not next_scan: + continue + row = [ + result["id"], + result["site"]["name"], + result["site"]["url"], + next_scan.get("date_time", ""), + recurrence_map[next_scan.get("recurrence", "")] + ] + csv_writer.writerow(row) + +if __name__ == '__main__': + main() From cc49b1bf104d44fbfe28bfef36c1f69027be3c31 Mon Sep 17 00:00:00 2001 From: Hugo Castilho Date: Wed, 26 Oct 2022 18:34:41 +0100 Subject: [PATCH 06/18] Rotate target pool --- rotate_target_pool.py | 193 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100755 rotate_target_pool.py diff --git a/rotate_target_pool.py b/rotate_target_pool.py new file mode 100755 index 0000000..0a86d49 --- /dev/null +++ b/rotate_target_pool.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +import csv +import logging +import sys +from time import sleep +from urllib.parse import urlparse, urljoin + +import requests + +token = "" +base_url = "https://api.probely.com" +pool_size = 25 +target_list_url = urljoin(base_url, "targets/") +target_detail_url = urljoin(base_url, "targets/{target_id}/") +start_scan_url = urljoin(base_url, "targets/{target_id}/scan_now/") +scan_detail_url = urljoin(base_url, "targets/{target_id}/scans/{scan_id}/") +finding_list_url = urljoin(base_url, "targets/{target_id}/findings/") +session = requests.Session() +sleep_time = 5 * 60 # 5 minutes +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + handlers=[ + logging.StreamHandler(sys.stdout), + ], +) + + +def create_target(url): + hostname = urlparse(url).hostname + logging.info("[%s] Creating target", hostname) + target_payload = { + "site": { + "url": url, + "name": hostname, + }, + "labels": [ + {"name": "Test"}, + ], + } + response = session.post(target_list_url, json=target_payload) + response.raise_for_status() + return response.json() + + +def delete_target(target): + logging.info("[%s] Deleting targets", target["site"]["name"]) + response = session.delete(target_detail_url.format(target_id=target["id"])) + response.raise_for_status() + + +def start_scan(target): + logging.info("[%s] Starting scan", target["site"]["name"]) + response = session.post(start_scan_url.format(target_id=target["id"])) + response.raise_for_status() + return response.json() + + +def get_scan(target, scan): + logging.info("[%s] Retrieving scan", target["site"]["name"]) + response = session.get( + scan_detail_url.format(target_id=target["id"], scan_id=scan["id"]) + ) + response.raise_for_status() + return response.json() + + +def get_scan_findings(target, scan): + page = 1 + page_total = 1 + findings = [] + while page <= page_total: + response = session.get( + finding_list_url.format(target_id=target["id"]), + params={"scan": scan["id"], "length": 10000, "page": 1}, + ) + response.raise_for_status() + response = response.json() + findings.extend(response["results"]) + page_total = response["page_total"] + page += 1 + return findings + + +def create_and_start_scan(target_url, running_scans): + global pool_size + logging.info("Starting %s", target_url) + try: + target = create_target(target_url) + scan = start_scan(target) + except requests.HTTPError as exc: + if ( + exc.response.json().get("non_field_errors", "") + and "The target pool of your subscription has no available slots" + in exc.response.json().get("non_field_errors", "")[0] + ): + logging.error( + "Reached pool limit before filling queue! Reducing pool size." + ) + pool_size -= 1 + if pool_size == 0: + logging.error("Unable to create any more targets") + sys.exit(1) + else: + logging.error(exc) + if hasattr(exc, "response"): + logging.error(exc.response.content) + else: + running_scans[target["id"]] = (target, scan) + + +def save_and_delete(target, scan, output_file): + with open(output_file, "at", newline="") as csv_file: + csv_writer = csv.writer( + csv_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL + ) + findings = get_scan_findings(target, scan) + for finding in findings: + csv_writer.writerow(to_csv(finding)) + delete_target(target) + + +def to_csv(result): + labels = result["labels"] if result.get("labels") else [] + labels_name = [label["name"] for label in labels] + row = [ + result["id"], + result["severity"], + result["definition"]["name"], + result["url"], + result["last_found"], + result["state"], + result.get("assignee")["email"] if result.get("assignee") else " ", + *labels_name, + ] + return row + + +def main(target_list, output_file, token): + # Add token to all requests + global pool_size + session.headers.update({"Authorization": f"JWT {token}"}) + + running_scans = {} + while True: + # Create targets and start scans up to pool_size + while len(running_scans) < pool_size: + try: + target_url = target_list.pop() # Remove from queue + except IndexError: + break + else: + create_and_start_scan(target_url, running_scans) + logging.info("%s Running scans", len(running_scans)) + + sleep(sleep_time) # Wait before checking scans + + # Check running scans + logging.info("Checking on %s running scans", len(running_scans)) + finished_scans = [] + for target, scan in running_scans.values(): + scan = get_scan(target, scan) + logging.info("[%s] Scan status %s", target["site"]["name"], scan["status"]) + if scan["status"] == "queued": + logging.info("[%s] Scan hasn't started yet", target["site"]["name"]) + elif scan["status"] in ("cancelled", "failed"): + logging.error( + "[%s] Scan has unexpectedly stopped", target["site"]["name"] + ) + elif scan["status"] == "completed": + logging.info("[%s] Scan has finished", target["site"]["name"]) + save_and_delete(target, scan, output_file) + finished_scans.append(target["id"]) + logging.info( + "[%s] Results stored and target deleted", target["site"]["name"] + ) + else: + logging.info("[%s] Scan still running", target["site"]["name"]) + + # Remove from running scans + for id_ in finished_scans: + del running_scans[id_] + + if not target_list and not running_scans: + sys.exit(0) + + +if __name__ == "__main__": + target_list = [ + "https://example.org", + ] + output_file = "/tmp/out.csv" + main(target_list, output_file, token) From 5b199857c81a475103be8dc27107ef3454b8de44 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Tue, 25 Jul 2023 00:11:23 +0100 Subject: [PATCH 07/18] script to start a scan in all existing targets --- start_scan_all_targets.py | 49 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100755 start_scan_all_targets.py diff --git a/start_scan_all_targets.py b/start_scan_all_targets.py new file mode 100755 index 0000000..6340af4 --- /dev/null +++ b/start_scan_all_targets.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python + +import requests +from urllib.parse import urljoin + +def main(): + token = input("API Token:") + + headers = {"Authorization": "JWT {}".format(token)} + + api_base_url = "https://api.probely.com" + targets_endpoint = urljoin( + api_base_url, "targets/?include=compliance&length=10000" + ) + + response = requests.get(targets_endpoint, headers=headers) + results = None + try: + results = response.json()["results"] + except: + print('Failed getting the list of targets, confirm if the API Token is correct.') + return + + for result in results: + target_id = result.get("id", None) + if target_id is not None: + target_name = result["site"]["name"] + target_url = result["site"]["url"] + + scan_now_endpoint = urljoin(api_base_url, "targets/{target_id}/scan_now/") + + try: + scan_response = requests.post(scan_now_endpoint.format(target_id=target_id), + headers=headers) + scan_result = scan_response.json() + + if "error" in scan_result: + error = scan_result["error"] + print(f"Error: {error} => ({target_name}) {target_url}") + elif "id" not in scan_result: + print(f"Error: Starting scan on ({target_name}) {target_url} failed") + else: + print(f"Started scan on ({target_name}) {target_url}") + except: + print(f"Failed starting scan on ({target_name}) {target_url}") + + +if __name__ == '__main__': + main() From 7cd9ad7bb1e7181cbc388948cca1202713cc600e Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Tue, 23 Jan 2024 01:58:08 +0000 Subject: [PATCH 08/18] add findings to DefectDojo Generic JSON format --- findings_to_defectdojo.py | 97 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100755 findings_to_defectdojo.py diff --git a/findings_to_defectdojo.py b/findings_to_defectdojo.py new file mode 100755 index 0000000..b311b62 --- /dev/null +++ b/findings_to_defectdojo.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +""" +Export all findings in a target to DefectDojo JSON format + +Choose "Generic Findings Import" under "Findings" > "Import Scan Results" > "Scan type" + +Run: + +$ python3 findings_to_defectdojo.py -t '' -o + +""" +import argparse +import requests +import json +from urllib.parse import urljoin +from datetime import datetime + +# Define the JWT or it will be asked when you run the script +jwt_token = None + +api_base_url = 'https://api.probely.com' +target_endpoint = urljoin(api_base_url, "targets/{target}/") +finding_list_endpoint = urljoin(api_base_url, "targets/{target}/findings/") + +def map_severity(probely_severity): + if probely_severity == 10: + return 'Low' + elif probely_severity == 20: + return 'Medium' + elif probely_severity == 30: + return 'High' + else: + return None + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-t', '--target', help='Target id', required=True) + parser.add_argument('-o', '--output', help='Output file', type=argparse.FileType('w'), required=True) + args = parser.parse_args() + + if jwt_token is None: + token = input("API Token:") + else: + token = jwt_token + + if token is None or token == '': + print('Error: JWT is required') + return + headers = {'Authorization': "JWT {}".format(token)} + + response_target = requests.get( + target_endpoint.format(target=args.target), + headers=headers + ) + response_target.raise_for_status() + target_res = response_target.json() + + target_name = target_res['site'].get('name') + target_url = target_res['site'].get('url') + print(f'Exporting findings: {target_name} - {target_url}') + + # Findings + response = requests.get( + finding_list_endpoint.format(target=args.target), + headers=headers, + params={'length': 500} + ) + response.raise_for_status() + findings_res = response.json()['results'] + + result = { + 'name': f'{target_name} - {target_url}', + 'findings': [] + } + for finding in findings_res: + result['findings'].append({ + 'title': finding['definition']['name'], + 'unique_id_from_tool': finding['id'], + 'description': finding['definition']['desc'], + 'severity': map_severity(finding['severity']), + 'mitigation': finding['fix'], + 'date': datetime.strptime(finding['last_found'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y-%m-%d"), + 'cve': None, + 'cwe': None, + 'cvssv3': finding['cvss_vector'], + 'file_path': finding['path'], + 'endpoints': [finding['path']], + 'active': True if finding['state'] == 'notfixed' else False, + 'verified': True, + 'false_p': True if finding['state'] == 'invalid' else False + }) + + args.output.write(json.dumps(result, indent=2)) + print('Done') + +if __name__ == '__main__': + main() From 49d0abfb1d1db566c923dac53c6b9dfefd465281 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Tue, 30 Jan 2024 01:02:48 +0000 Subject: [PATCH 09/18] add script to export the list of scans by the status to CSV --- scans_by_status_csv.py | 75 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 scans_by_status_csv.py diff --git a/scans_by_status_csv.py b/scans_by_status_csv.py new file mode 100644 index 0000000..16d1beb --- /dev/null +++ b/scans_by_status_csv.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python +""" +Export all scans on every target with a specific status to CSV format + +Run: + +$ python3 scans_by_state_csv.py -s -o + +""" +import argparse +import requests +import csv +from urllib.parse import urljoin +from datetime import datetime + +# Define the JWT or it will be asked when you run the script +jwt_token = None + +api_base_url = 'https://api.probely.com' +scans_endpoint = urljoin(api_base_url, "scans/?length=1000&page=1&scan_profile_name=true&search=&status={status}&exclude=target_options") + +def map_severity(probely_severity): + if probely_severity == 10: + return 'Low' + elif probely_severity == 20: + return 'Medium' + elif probely_severity == 30: + return 'High' + else: + return None + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--status', help='Status', required=True, choices=[ + 'started', 'paused', 'under_review', 'completed', 'failed', 'canceled']) + parser.add_argument('-o', '--output', help='Output file', type=argparse.FileType('w'), required=True) + args = parser.parse_args() + + if jwt_token is None: + token = input("API Token:") + else: + token = jwt_token + + if token is None or token == '': + print('Error: JWT is required') + return + headers = {'Authorization': "JWT {}".format(token)} + + response_scans = requests.get( + scans_endpoint.format(status=args.status), + headers=headers + ) + response_scans.raise_for_status() + scans_res = response_scans.json()['results'] + + csv_writer = csv.writer( + args.output, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL + ) + row = ['Target ID', 'Target URL', 'Scan profile', 'Started', 'Completed', 'Status'] + csv_writer.writerow(row) + for scan in scans_res: + row = [ + scan['target']['id'], + scan['target']['site']['url'], + scan['scan_profile']['name'], + datetime.strptime(scan['started'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y-%m-%d %H:%M:%S") if scan['started'] is not None else '', + datetime.strptime(scan['completed'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime("%Y-%m-%d %H:%M:%S") if scan['completed'] is not None else '', + scan['status'] + ] + csv_writer.writerow(row) + + print('Done') + +if __name__ == '__main__': + main() From de7d63e28c681a71ed88054ed84d01aa3e903837 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Mon, 27 May 2024 03:03:40 +0100 Subject: [PATCH 10/18] add util to convert burp or HAR files to Probely's navigation sequences --- utils/README-burp_har_to_navigation_seq.md | 43 ++++++++++++++++++++++ utils/burp_har_to_navigation_seq.py | 40 ++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 utils/README-burp_har_to_navigation_seq.md create mode 100755 utils/burp_har_to_navigation_seq.py diff --git a/utils/README-burp_har_to_navigation_seq.md b/utils/README-burp_har_to_navigation_seq.md new file mode 100644 index 0000000..692f1be --- /dev/null +++ b/utils/README-burp_har_to_navigation_seq.md @@ -0,0 +1,43 @@ +# Burp and HAR files converter + +## Converts Burp or HAR files to Probely's Navigation sequences. + +### Required options + + - `--type/-t`: `burp` or `har` (input file format) + - `--input/-i`: `/path/to/original_file` + - `--output/-o`: `/path/to/destination_file` + +### Optional + +- `--crawl` or `--no-crawl` (default: `crawl`) + +### From Burp file to Probely's Navigation sequence + +```sh +$ python3 ./burp_har_to_navigation_seq.py -t burp -i /tmp/burp_exported_file.xml -o /tmp/probely_navigation.json +``` + + +### From HAR file to Probely's Navigation sequence + +```sh +$ python3 ./burp_har_to_navigation_seq.py -t har -i /tmp/har_exported_file.har -o /tmp/probely_navigation.json +``` + +```sh +$ python3 ./burp_har_to_navigation_seq.py -h +usage: burp_har_to_navigation_seq.py [-h] -t {burp,har} -i INPUT -o OUTPUT [--crawl | --no-crawl] + +Converts Burp or HAR files to Probely's navigation sequences + +options: + -h, --help show this help message and exit + -t {burp,har}, --type {burp,har} + burp/har + -i INPUT, --input INPUT + Input file + -o OUTPUT, --output OUTPUT + Output file + --crawl, --no-crawl Crawl the requests +``` \ No newline at end of file diff --git a/utils/burp_har_to_navigation_seq.py b/utils/burp_har_to_navigation_seq.py new file mode 100755 index 0000000..8c3b96d --- /dev/null +++ b/utils/burp_har_to_navigation_seq.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Converts Burp or HAR files to Probely's Navigation sequences. +""" +import argparse +import json + +def run(): + parser = argparse.ArgumentParser(description='Converts Burp or HAR files to Probely\'s navigation sequences') + parser.add_argument('-t', '--type', help='burp/har', choices=['burp', 'har'], required=True) + parser.add_argument('-i', '--input', help='Input file', type=argparse.FileType('r'), required=True) + parser.add_argument('-o', '--output', help='Output file', type=argparse.FileType('w'), required=True) + parser.add_argument('--crawl', help='Crawl the requests', default=True, action=argparse.BooleanOptionalAction) + args = parser.parse_args() + + with args.input as file: + input_data = file.read() + + if args.type == 'burp': + out_data = [{ + 'file_type': 'burp', + 'crawl': args.crawl, + 'file_data': input_data + }] + elif args.type == 'har': + out_data = [{ + 'file_type': 'har', + 'crawl': args.crawl, + 'file_data': input_data + }] + + final_json = json.dumps(out_data, indent=2) + args.output.write(final_json) + + print('Done.') + print('File converted to file: {}'.format(args.output.name)) + + +if __name__ == '__main__': + run() From 0595956a2e9a10e0bd9159dcde23609f0f9d4c5b Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Fri, 12 Jul 2024 01:23:16 +0100 Subject: [PATCH 11/18] add export discovery assets to CSV format --- discovery_assets_to_csv.py | 81 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 discovery_assets_to_csv.py diff --git a/discovery_assets_to_csv.py b/discovery_assets_to_csv.py new file mode 100755 index 0000000..1308df8 --- /dev/null +++ b/discovery_assets_to_csv.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +Export all discovery assets with a specific optional score to CSV format + +Run: + +$ python3 discovery_assets_to_csv.py -s -o + +""" +import argparse +import requests +import csv +from urllib.parse import urljoin, quote +from datetime import datetime + +# Define the JWT or it will be asked when you run the script +jwt_token = None + +api_base_url = 'https://api.probely.com' +discovery_assets_endpoint = urljoin(api_base_url, "discovery/assets/?length=10000&page=1&ordering=-last_seen&{score_str}") + +def map_risk(probely_risk): + if probely_risk == 10: + return 'Low' + elif probely_risk == 20: + return 'Normal' + elif probely_risk == 30: + return 'High' + else: + return 'NA' + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('-s', '--score', help='Score', required=False, choices=[ + 'A+', 'A', 'B', 'C', 'D', 'E', 'F', 'R', 'NA']) + parser.add_argument('-o', '--output', help='Output CSV file', type=argparse.FileType('w'), required=True) + args = parser.parse_args() + + if jwt_token is None: + token = input("API Token:") + else: + token = jwt_token + + if token is None or token == '': + print('Error: JWT is required') + return + headers = {'Authorization': "JWT {}".format(token)} + + score_str = f'score={quote(args.score)}' + if args.score is None: + score_str = '' + + response_assets = requests.get( + discovery_assets_endpoint.format(score_str=score_str), + headers=headers + ) + response_assets.raise_for_status() + assets_res = response_assets.json()['results'] + + csv_writer = csv.writer( + args.output, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL + ) + row = ['ID', 'Name', 'URL', 'Type', 'Last Seen', 'Risk', 'Score', 'State'] + csv_writer.writerow(row) + for asset in assets_res: + row = [ + asset['id'], + asset['name'], + asset['url'], + asset['type'], + datetime.strptime(asset['last_seen'], "%Y-%m-%dT%H:%M:%SZ").strftime("%Y-%m-%d %H:%M:%S"), + map_risk(asset['risk']), + asset['score'], + asset['state'], + ] + csv_writer.writerow(row) + + print('Done') + +if __name__ == '__main__': + main() From d7e0a1921d9b4a047edea20f4fdeb6886cb941bc Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Fri, 12 Jul 2024 01:33:01 +0100 Subject: [PATCH 12/18] check score to quote it --- discovery_assets_to_csv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/discovery_assets_to_csv.py b/discovery_assets_to_csv.py index 1308df8..1694089 100755 --- a/discovery_assets_to_csv.py +++ b/discovery_assets_to_csv.py @@ -46,9 +46,9 @@ def main(): return headers = {'Authorization': "JWT {}".format(token)} - score_str = f'score={quote(args.score)}' - if args.score is None: - score_str = '' + score_str = '' + if args.score is not None: + score_str = f'score={quote(args.score)}' response_assets = requests.get( discovery_assets_endpoint.format(score_str=score_str), From 75626d1667355b98942efdcf45f34391f1159261 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Sun, 2 Mar 2025 23:59:55 +0000 Subject: [PATCH 13/18] add fix when .side has frame selector --- utils/navigation_converter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/navigation_converter.py b/utils/navigation_converter.py index 7b74138..cffc3b2 100755 --- a/utils/navigation_converter.py +++ b/utils/navigation_converter.py @@ -23,6 +23,8 @@ def getSeleniumCssAndXPath(targets, target): css = None xpath = None for item in targets: + if not item or len(item) < 2: + return (None, None) if item[1] == 'css:finder': css = item[0].replace('css=', '') if item[1] == 'xpath:position': From dd695522503c10dd530c56094138e6d665963262 Mon Sep 17 00:00:00 2001 From: cvaidas Date: Tue, 25 Mar 2025 14:07:56 +0200 Subject: [PATCH 14/18] chore: add codeowners --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..70d148d --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @probely/probely-backend \ No newline at end of file From 4b2e4b2561acfcaedd0a8074ed80e7d3f5ccb7e4 Mon Sep 17 00:00:00 2001 From: cvaidas Date: Thu, 3 Apr 2025 10:22:09 +0300 Subject: [PATCH 15/18] chore: add asset classification --- catalog-info.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 catalog-info.yaml diff --git a/catalog-info.yaml b/catalog-info.yaml new file mode 100644 index 0000000..fa4a911 --- /dev/null +++ b/catalog-info.yaml @@ -0,0 +1,8 @@ +apiVersion: backstage.io/v1alpha1 +kind: Component +metadata: + name: API_Scripts +spec: + type: examples + lifecycle: "-" + owner: probely-backend \ No newline at end of file From bba6d81ab00374499496bf0719a79bb7c6f1abcd Mon Sep 17 00:00:00 2001 From: Cristina Vaida Date: Tue, 8 Apr 2025 13:29:50 +0300 Subject: [PATCH 16/18] chore: add secret scanning [DAST-14] (#19) * chore: add secret scanning * chore: add slack channel --- .github/workflows/secrets-scanning.yml | 13 +++++++++++++ .pre-commit-config.yaml | 14 ++++++++++++++ catalog-info.yaml | 2 +- 3 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/secrets-scanning.yml create mode 100644 .pre-commit-config.yaml diff --git a/.github/workflows/secrets-scanning.yml b/.github/workflows/secrets-scanning.yml new file mode 100644 index 0000000..f6c2a19 --- /dev/null +++ b/.github/workflows/secrets-scanning.yml @@ -0,0 +1,13 @@ +name: Detect Secrets +on: + pull_request: + push: + workflow_dispatch: +jobs: + secrets-scan: + uses: probely/snyk-prodsec/.github/workflows/secrets-scanning.yml@main + with: + channel: probely-alerts + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_SECRET }} + GITLEAKS_LICENSE: ${{ secrets.GITLEAKS_LICENSE }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..cca0d62 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - repo: https://github.com/gitleaks/gitleaks + rev: v8.22.0 + hooks: + - id: gitleaks diff --git a/catalog-info.yaml b/catalog-info.yaml index fa4a911..cfa3752 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -5,4 +5,4 @@ metadata: spec: type: examples lifecycle: "-" - owner: probely-backend \ No newline at end of file + owner: probely-backend From 17a6cc22e7ae7c90ce25282550865aa372ae9236 Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Wed, 8 Oct 2025 15:45:51 +0100 Subject: [PATCH 17/18] add import postman vars v2 (#25) --- import_postman_env_v2.py | 107 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 import_postman_env_v2.py diff --git a/import_postman_env_v2.py b/import_postman_env_v2.py new file mode 100755 index 0000000..82f6f97 --- /dev/null +++ b/import_postman_env_v2.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +""" +Import postman environment file - V2 + +To set postman environment for existing API target +with Postman collection schema type +or OpenAPI API targets with custom variables + +""" +import argparse +import json +from urllib.parse import urljoin +import requests + +# Define the JWT or it will be asked when you run the script +jwt_token = None + +api_base_url = 'https://api.probely.com' + +def main(): + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument('-t', '--target', help='Target ID', required=True) + parser.add_argument('-e', '--env-file', help='Environment Variables file', type=argparse.FileType('r'), required=True) + parser.add_argument('--ignore-enabled', help='Ignore enabled property', action='store_true', required=False) + parser.add_argument('--null-to', help='Converts null values to defined value.\n--null-to "null": converts null values to string "null"\n--null-to "": converts null values to empty string (default)', default="", required=False) + parser.add_argument('--include-empty', help='Include empty values', action='store_true', required=False) + parser.add_argument('--clear', help='Clear existing values', action='store_true', required=False) + args = parser.parse_args() + + if jwt_token is None: + token = input("API Token:") + else: + token = jwt_token + + if token is None or token == '': + print('Error: JWT is required') + return + headers = {'Authorization': "JWT {}".format(token)} + + target_id = args.target + + postman_env = json.load(args.env_file) + + if "values" not in postman_env or "values" in postman_env and len(postman_env["values"]) == 0: + print("No values on Environment Variables file") + return + + parsed_env = [] + + for item in postman_env["values"]: + + if "key" not in item or "value" not in item: + continue + + if args.ignore_enabled == False: + if not "enabled" in item or not item["enabled"]: + continue + + if "value" in item and item["value"] is None: + item["value"] = args.null_to + + if "value" in item and item["value"] == "" and args.include_empty == False: + continue + + parsed_env.append({ + "name": item["key"], + "value": item["value"] + }) + + target_endpoint = urljoin(api_base_url, "targets/{target_id}/") + + response = requests.get(target_endpoint.format(target_id=target_id), headers=headers) + assert response.status_code == 200, response.json() + + if args.clear == True: + custom_api_parameters = [] + else: + custom_api_parameters = ( + response.json()["site"]["api_scan_settings"]["custom_api_parameters"] or [] + ) + + updated_field_names = [entry["name"] for entry in parsed_env] + + custom_api_parameters = [ + *[ + entry + for entry in custom_api_parameters + if entry["name"] not in updated_field_names + ], + *parsed_env, + ] + + response = requests.patch( + target_endpoint.format(target_id=target_id), + headers=headers, + json={ + "site": {"api_scan_settings": {"custom_api_parameters": custom_api_parameters}} + }, + ) + assert response.status_code == 200, response.json() + +if __name__ == '__main__': + main() + print("Done.") + + From 3adeffe708c27ed06738d1651bd713688645254f Mon Sep 17 00:00:00 2001 From: Claudio Gamboa Date: Thu, 5 Mar 2026 14:35:01 +0000 Subject: [PATCH 18/18] add burp to login/navigation sequence converter (#26) --- .../README-burp_to_saw_sequence_converter.md | 26 ++ utils/burp_to_saw_sequence_converter.py | 324 ++++++++++++++++++ 2 files changed, 350 insertions(+) create mode 100644 utils/README-burp_to_saw_sequence_converter.md create mode 100644 utils/burp_to_saw_sequence_converter.py diff --git a/utils/README-burp_to_saw_sequence_converter.md b/utils/README-burp_to_saw_sequence_converter.md new file mode 100644 index 0000000..924038a --- /dev/null +++ b/utils/README-burp_to_saw_sequence_converter.md @@ -0,0 +1,26 @@ +# Burp to Snyk API&Web Sequence Converter + +## Description + +Converts Burp Suite Navigation Recorder sequences to Snyk API&Web's sequence format. + +This converter transforms recordings made with the Burp Suite Navigation Recorder browser plugin into the format expected by Snyk API&Web for login and navigation sequences. + + +## Required Options + +- `--input/-i`: Path to the Burp recording JSON file +- `--output/-o`: Path to the output Snyk API&Web sequence JSON file + + +## Usage Examples + +### Convert a recorded file to Snyk API&Web sequence + +```sh +python3 ./burp_to_saw_sequence_converter.py -i /tmp/burp_recorded_file.json -o /tmp/snyk_login_sequence.json +``` + + + + diff --git a/utils/burp_to_saw_sequence_converter.py b/utils/burp_to_saw_sequence_converter.py new file mode 100644 index 0000000..f12f08f --- /dev/null +++ b/utils/burp_to_saw_sequence_converter.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python +""" +Converts Burp Suite Navigation Recorder sequences to Snyk API&Web's sequence format. +""" +import argparse +import json +from typing import Dict, List, Any, Optional +from urllib.parse import urlparse + + +def extract_css_selector(event: Dict[str, Any]) -> Optional[str]: + """ + Generate a CSS selector from Burp event data. + Priority: id > name > placeholder > class > tag + """ + tag = event.get('tagName', '').lower() + if not tag: + return None + + # ID + element_id = event.get('id') + if element_id: + return f"#{element_id}" + + # Name attribute + name = event.get('name') + if name: + return f"{tag}[name='{name}']" + + # Placeholder for inputs + placeholder = event.get('placeholder') + if placeholder and tag == 'input': + return f"input[placeholder='{placeholder}']" + + # Class name + class_name = event.get('className', '').strip() + if class_name: + classes = class_name.replace(' ', '.') + return f"{tag}.{classes}" + + # href + href = event.get('href') + if href and tag == 'a': + return f"a[href='{href}']" + + # Fallback: tag name + return tag + + +def build_frame_hierarchy(burp_data: List[Dict]) -> Dict[int, Dict]: + """ + Map of all frames + """ + frame_map = {0: {'selector': None, 'parent': None}} + + start_event = burp_data[0] if burp_data and burp_data[0].get('eventType') == 'start' else None + if not start_event: + return frame_map + + iframes = start_event.get('iframes', []) + for iframe in iframes: + frame_id = iframe.get('frameId') + attrs = iframe.get('attributes', {}) + + iframe_id = attrs.get('id') + if iframe_id: + selector = f"iframe#{iframe_id}" + elif attrs.get('src'): + selector = f"iframe[src='{attrs.get('src')}']" + elif iframe.get('xPath'): + selector = iframe.get('xPath') + else: + selector = f"iframe:nth-child({iframe.get('iframeIndex', 0) + 1})" + + frame_map[frame_id] = { + 'selector': selector, + 'parent': 0, + 'url': attrs.get('src', '') + } + + # Discover nested iframes + frame_urls = {} + for event in burp_data: + frame_id = event.get('frameId', 0) + if frame_id != 0 and event.get('isIframe'): + event_url = event.get('url', '') + if event_url: + frame_urls[frame_id] = event_url + + # Collect all top-level iframe IDs for combined selector + top_level_iframe_ids = [fid for fid in frame_map.keys() if fid != 0] + + for frame_id, url in frame_urls.items(): + if frame_id not in frame_map: + parsed = urlparse(url) + path = parsed.path + + if path: + path_parts = path.rstrip('/').split('/') + filename = path_parts[-1] if path_parts else path + selector = f"iframe[src*='{filename}']" + else: + selector = "iframe" + + # Combine parent selector (all top-level iframes) + frame_map[frame_id] = { + 'selector': selector, + 'parent': None, + 'url': url, + 'possible_parents': top_level_iframe_ids + } + + return frame_map + + +def build_frame_selector(frame_map: Dict[int, Dict], frame_id: int) -> Optional[str]: + """ + Build frame selector for iframes + """ + if frame_id == 0: + return None + + if frame_id not in frame_map: + return None + + frame_info = frame_map.get(frame_id) + if not frame_info: + return None + + selector = frame_info.get('selector') + if not selector: + return None + + possible_parents = frame_info.get('possible_parents') + if possible_parents: + parent_selectors = [] + for parent_id in possible_parents: + parent_info = frame_map.get(parent_id) + if parent_info and parent_info.get('selector'): + parent_selectors.append(parent_info.get('selector')) + + if parent_selectors: + # Join selectors with " >>> " + combined_parents = ','.join(parent_selectors) + return f"{combined_parents} >>> {selector}" + else: + return selector + + selectors = [selector] + current_frame_id = frame_info.get('parent') + + while current_frame_id is not None and current_frame_id != 0: + parent_info = frame_map.get(current_frame_id) + if not parent_info: + break + + parent_selector = parent_info.get('selector') + if parent_selector: + selectors.append(parent_selector) + + current_frame_id = parent_info.get('parent') + + if len(selectors) == 1: + return selectors[0] + + selectors.reverse() + return ' >>> '.join(selectors) + + +def convert_burp_to_saw(burp_data: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """ + Convert Burp Suite Navigation Recorder format to SAW sequence format. + """ + saw_sequence = [] + + if not burp_data or len(burp_data) == 0: + raise Exception('No data in Burp recording file to be converted.') + + frame_map = build_frame_hierarchy(burp_data) + + for idx, event in enumerate(burp_data): + event_type = event.get('eventType') + + # Skip the start item + if event_type == 'start': + continue + + # goto + if event_type == 'goto': + url = event.get('url') + timestamp = event.get('timestamp', 0) + window_width = 1800 + window_height = 1200 + + if idx + 1 < len(burp_data): + next_event = burp_data[idx + 1] + window_width = next_event.get('windowInnerWidth', window_width) + window_height = next_event.get('windowInnerHeight', window_height) + + saw_sequence.append({ + 'type': 'goto', + 'urlType': 'force', + 'url': url, + 'timestamp': timestamp, + 'windowWidth': window_width, + 'windowHeight': window_height + }) + + # click + elif event_type == 'click': + css = extract_css_selector(event) + xpath = event.get('xPath') + timestamp = event.get('timestamp', 0) + frame_id = event.get('frameId', 0) + + frame = build_frame_selector(frame_map, frame_id) if event.get('isIframe') else None + + # Value + value = (event.get('value') or event.get('textContent') or '').strip()[:20].replace('\n', '') + + saw_sequence.append({ + 'timestamp': timestamp, + 'type': 'click', + 'css': css, + 'xpath': xpath, + 'value': value, + 'frame': frame + }) + + # fill_value + elif event_type == 'typing': + css = extract_css_selector(event) + xpath = event.get('xPath') + typed_value = event.get('typedValue', '') + timestamp = event.get('timestamp', 0) + frame_id = event.get('frameId', 0) + + frame = build_frame_selector(frame_map, frame_id) if event.get('isIframe') else None + + saw_sequence.append({ + 'timestamp': timestamp, + 'type': 'fill_value', + 'css': css, + 'xpath': xpath, + 'value': typed_value, + 'frame': frame + }) + + # dblclick + elif event_type == 'dblclick': + css = extract_css_selector(event) + xpath = event.get('xPath') + timestamp = event.get('timestamp', 0) + frame_id = event.get('frameId', 0) + + frame = build_frame_selector(frame_map, frame_id) if event.get('isIframe') else None + value = (event.get('value') or event.get('textContent') or '').strip()[:20].replace('\n', '') + + saw_sequence.append({ + 'timestamp': timestamp, + 'type': 'dblclick', + 'css': css, + 'xpath': xpath, + 'value': value, + 'frame': frame + }) + + # skip userNavigate + elif event_type == 'userNavigate': + pass + + # Log unsupported steps + else: + print(f"Warning: Unsupported step '{event_type}' at index {idx}") + + return saw_sequence + + +def run(): + """ + Main + """ + parser = argparse.ArgumentParser( + description='Converts Burp Suite Navigation Recorder sequences to Snyk API&Web sequence format' + ) + parser.add_argument( + '-i', '--input', + help='Input Burp recording JSON file', + type=argparse.FileType('r'), + required=True + ) + parser.add_argument( + '-o', '--output', + help='Output Snyk API&Web sequence JSON file', + type=argparse.FileType('w'), + required=True + ) + + args = parser.parse_args() + + # Load Burp recording + try: + burp_data = json.load(args.input) + except json.JSONDecodeError as e: + print(f"Error: Invalid JSON in input file: {e}") + return 1 + + # Convert to Snyk API&Web format + try: + saw_data = convert_burp_to_saw(burp_data) + except Exception as e: + print(f"Error during conversion: {e}") + return 1 + + json.dump(saw_data, args.output, indent=2) + + print(f"✓ Successfully converted {len(burp_data)} Burp steps to {len(saw_data)} Snyk API&Web") + print(f"✓ Output written to: {args.output.name}") + + return 0 + + +if __name__ == '__main__': + run()