From cf2d4a2b2631ba7fec1b29976aa5024966b5a323 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sat, 13 Mar 2021 23:40:26 +0100 Subject: [PATCH 01/24] Added a new configuration option to obay the GitHub and GitLab ratelimits. --- scanners/git-repo-scanner/README.md.gotmpl | 24 ++++++----- .../scanner/git_repo_scanner.py | 42 ++++++++++++++----- .../scanner/git_repo_scanner_test.py | 1 + .../git-repo-scanner/scanner/requirements.txt | 4 +- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/scanners/git-repo-scanner/README.md.gotmpl b/scanners/git-repo-scanner/README.md.gotmpl index b573606f14..697ce17740 100644 --- a/scanners/git-repo-scanner/README.md.gotmpl +++ b/scanners/git-repo-scanner/README.md.gotmpl @@ -32,22 +32,24 @@ or ``` #### GitHub -For type github you can use the following options: -- `--organization`: The name of the github organization you want to scan. -- `--url`: The url of the api for a github enterprise server. Skip this option for repos on . -- `--access-token`: Your personal github access token. -- `--ignore-repos`: A list of github repository ids you want to ignore +For type GitHub you can use the following options: +- `--organization`: The name of the GitHub organization you want to scan. +- `--url`: The url of the api for a GitHub enterprise server. Skip this option for repos on . +- `--access-token`: Your personal GitHub access token. +- `--ignore-repos`: A list of GitHub repository ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. #### GitLab -For type gitlab you can use the following options: -- `--url`: The url of the gitlab server. -- `--access-token`: Your personal gitlab access token. -- `--group`: A specific gitlab group id you want to san, including subgroups. -- `--ignore-groups`: A list of gitlab group ids you want to ignore -- `--ignore-repos`: A list of gitlab project ids you want to ignore +For type GitLab you can use the following options: +- `--url`: The url of the GitLab server. +- `--access-token`: Your personal GitLab access token. +- `--group`: A specific GitLab group id you want to san, including subgroups. +- `--ignore-groups`: A list of GitLab group ids you want to ignore +- `--ignore-repos`: A list of GitLab project ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner.py index 7f111bfdf6..aa3f7c67ff 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner.py @@ -2,6 +2,8 @@ import logging import sys import json +import calendar +import time from typing import List from pathlib import Path @@ -59,22 +61,22 @@ def write_findings_to_file(args, findings): def get_parser_args(args=None): parser = argparse.ArgumentParser(description='Scan public or private git repositories of organizations or groups') parser.add_argument('--git-type', - help='Repository type can be github or gitlab', + help='Repository type can be github or GitLab', choices=['github', 'gitlab'], required=True) parser.add_argument('--file-output', help='The path of the output file', required=True), - parser.add_argument('--url', help='The gitlab url or a github enterprise api url.', + parser.add_argument('--url', help='The GitLab url or a GitHub enterprise api url.', required=False) parser.add_argument('--access-token', help='An access token for authentication', required=False) parser.add_argument('--organization', - help='The name of the githup organization to scan', + help='The name of the GitHub organization to scan', required=False) parser.add_argument('--group', - help='The id of the gitlab group to scan', + help='The id of the GitLab group to scan', required=False) parser.add_argument('--ignore-repos', help='A list of repo ids to ignore', @@ -84,12 +86,18 @@ def get_parser_args(args=None): default=[], required=False) parser.add_argument('--ignore-groups', - help='A list of gitlab group ids to ignore', + help='A list of GitLab group ids to ignore', action='extend', nargs='+', type=int, default=[], required=False) + parser.add_argument('--obey-rate-limit', + help='True to obey the rate limit of the GitLab or GitHub server (default), otherwise False', + type=bool, + default=True, + required=False) + if args: return parser.parse_args(args) else: @@ -99,7 +107,7 @@ def get_parser_args(args=None): def parse_gitlab(args): gl: gitlab.Gitlab if not args.url: - logger.info(' URL required for gitlab connection.') + logger.info(' URL required for GitLab connection.') sys.exit(-1) logger.info(' Gitlab authentication...') @@ -128,12 +136,12 @@ def process_gitlab_projects(args, projects): def get_gitlab_projects(args, gl): if args.group: try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True) + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, obey_rate_limit=args.obey_rate_limit) except gitlab.exceptions.GitlabGetError: logger.info(' Group does not exist.') sys.exit(-1) else: - projects = gl.projects.list(all=True, max_retries=12) + projects = gl.projects.list(all=True, max_retries=12, obey_rate_limit=args.obey_rate_limit) return projects @@ -146,7 +154,7 @@ def gitlab_authenticate(args): except gitlab.exceptions.GitlabAuthenticationError: gl = gitlab_authenticate_oauth(args) else: - logger.info(' Access token required for gitlab authentication.') + logger.info(' Access token required for GitLab authentication.') sys.exit(-1) logger.info(' Success') return gl @@ -174,22 +182,34 @@ def parse_github(args): logger.info(' No organization provided') sys.exit(-1) +def respect_github_ratelimit(args, gh): + if args.obey_rate_limit: + api_limit = gh.get_rate_limit().core + reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) + seconds_until_reset = reset_timestamp - calendar.timegm(time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset + sleep_time = seconds_until_reset / api_limit.remaining + + logger.info(' Checking Rate-Limit ('+ str(args.obey_rate_limit) +') [remainingApiCalls: ' + str(api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str(sleep_time) + ']') + time.sleep(sleep_time) def process_github_repos(args, gh): findings = [] org: Organization = gh.get_organization(args.organization) repos: PaginatedList[Repository] = org.get_repos(type='all') for i in range(repos.totalCount): - process_github_repos_page(args, findings, repos.get_page(i)) + process_github_repos_page(args, findings, repos.get_page(i), gh) return findings -def process_github_repos_page(args, findings, repos): +def process_github_repos_page(args, findings, repos, gh): repo: Repository for repo in repos: if repo.id not in args.ignore_repos: logger.info(f' {len(findings) + 1} - {repo.name}') + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + def setup_github(args): diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py index 5724657a06..71f2247039 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py @@ -94,6 +94,7 @@ def test_parse_github_with_no_org_should_exit(self): def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): args = ['--git-type', 'gitlab', '--file-output', 'out', + '--obey-rate-limit', False, '--ignore-repos', str(ignore_projects), '--ignore-groups', str(ignore_groups)] if url: diff --git a/scanners/git-repo-scanner/scanner/requirements.txt b/scanners/git-repo-scanner/scanner/requirements.txt index 3140bb4c74..95198b0130 100644 --- a/scanners/git-repo-scanner/scanner/requirements.txt +++ b/scanners/git-repo-scanner/scanner/requirements.txt @@ -1,4 +1,4 @@ -PyGithub == 1.53 -python-gitlab == 2.5.0 +PyGithub == 1.54.1 +python-gitlab == 2.6.0 munch == 2.5.0 mock == 4.0.2 From 8cf7bb1ccd152f6723a31a72ea5e2a46696d22a6 Mon Sep 17 00:00:00 2001 From: rseedorff Date: Sat, 13 Mar 2021 22:41:20 +0000 Subject: [PATCH 02/24] Updating Helm Docs --- scanners/git-repo-scanner/README.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index 89702467e1..ebcd88b534 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -31,22 +31,24 @@ or ``` #### GitHub -For type github you can use the following options: -- `--organization`: The name of the github organization you want to scan. -- `--url`: The url of the api for a github enterprise server. Skip this option for repos on . -- `--access-token`: Your personal github access token. -- `--ignore-repos`: A list of github repository ids you want to ignore +For type GitHub you can use the following options: +- `--organization`: The name of the GitHub organization you want to scan. +- `--url`: The url of the api for a GitHub enterprise server. Skip this option for repos on . +- `--access-token`: Your personal GitHub access token. +- `--ignore-repos`: A list of GitHub repository ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. #### GitLab -For type gitlab you can use the following options: -- `--url`: The url of the gitlab server. -- `--access-token`: Your personal gitlab access token. -- `--group`: A specific gitlab group id you want to san, including subgroups. -- `--ignore-groups`: A list of gitlab group ids you want to ignore -- `--ignore-repos`: A list of gitlab project ids you want to ignore +For type GitLab you can use the following options: +- `--url`: The url of the GitLab server. +- `--access-token`: Your personal GitLab access token. +- `--group`: A specific GitLab group id you want to san, including subgroups. +- `--ignore-groups`: A list of GitLab group ids you want to ignore +- `--ignore-repos`: A list of GitLab project ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. From d6d696c02850a4c384df51609c4fab563ed8f337 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sun, 14 Mar 2021 02:39:55 +0100 Subject: [PATCH 03/24] Added a new configuration option to filter git repos by latest acivity date. --- scanners/git-repo-scanner/README.md.gotmpl | 9 ++ .../scanner/git_repo_scanner.py | 148 ++++++++++++++++-- .../scanner/git_repo_scanner_test.py | 32 ++-- .../git-repo-scanner/scanner/requirements.txt | 2 + 4 files changed, 160 insertions(+), 31 deletions(-) diff --git a/scanners/git-repo-scanner/README.md.gotmpl b/scanners/git-repo-scanner/README.md.gotmpl index 697ce17740..02ab3188cd 100644 --- a/scanners/git-repo-scanner/README.md.gotmpl +++ b/scanners/git-repo-scanner/README.md.gotmpl @@ -38,6 +38,10 @@ For type GitHub you can use the following options: - `--access-token`: Your personal GitHub access token. - `--ignore-repos`: A list of GitHub repository ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. @@ -50,6 +54,11 @@ For type GitLab you can use the following options: - `--ignore-groups`: A list of GitLab group ids you want to ignore - `--ignore-repos`: A list of GitLab project ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. + For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner.py index aa3f7c67ff..e762690ad9 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner.py @@ -4,9 +4,17 @@ import json import calendar import time +from datetime import datetime +import pytz + from typing import List from pathlib import Path +# https://pypi.org/project/pytimeparse/ +from pytimeparse.timeparse import timeparse +# https://docs.python.org/3/library/datetime.html +from datetime import timedelta + import gitlab from gitlab.v4.objects import Project @@ -97,6 +105,14 @@ def get_parser_args(args=None): type=bool, default=True, required=False) + parser.add_argument('--activity-since-duration', + help='Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration)', + type=str, + required=False) + parser.add_argument('--activity-until-duration', + help='Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration)', + type=str, + required=False) if args: return parser.parse_args(args) @@ -109,31 +125,70 @@ def parse_gitlab(args): if not args.url: logger.info(' URL required for GitLab connection.') sys.exit(-1) - logger.info(' Gitlab authentication...') + logger.info(' Gitlab authentication...') gl = gitlab_authenticate(args) - projects: List[Project] = get_gitlab_projects(args, gl) + logger.info(' Gitlab retrieve all repositories...') + now_utc = pytz.utc.localize(datetime.utcnow()) + # Respect time filtering based on "pushed_at" (not "updated_at") + # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. + # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. + # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. + duration = 0 + activityDeltaDatetime = now_utc + if args.activity_since_duration: + activityDuration = timeparse(args.activity_since_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitLab Repos (filtered by last activity since '+ str(activityDeltaDatetime) +' ago.)') + + projects: List[Project] = get_gitlab_projects_active_since(args, gl) + elif args.activity_until_duration: + activityDuration = timeparse(args.activity_until_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitLab Repos (filtered by last activity until '+ str(activityDeltaDatetime) +' ago.)') + + projects: List[Project] = get_gitlab_projects_active_until(args, gl) + else: + logger.info(' Get all Gitlab Repos (not filtered)') + projects: List[Project] = get_gitlab_projects_all(args, gl) logger.info(' Process Projects...') - - findings = process_gitlab_projects(args, projects) + activityDate = now_utc - activityDeltaDatetime + findings = process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate) return findings -def process_gitlab_projects(args, projects): +def process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate): findings = [] i = 1 for project in projects: if is_not_on_ignore_list_gitlab(project, args.ignore_groups, args.ignore_repos): - logger.info(f' {i} - {project.name}') + lastUpDatetime = datetime.fromisoformat(project.last_activity_at) + logger.info(f' {i} - Name: {project.name} - LastUpdate: {lastUpDatetime}') i += 1 - findings.append(create_finding_gitlab(project)) + + # respect time filtering + if args.activity_since_duration: + if lastUpDatetime > activityDate: + findings.append(create_finding_gitlab(project)) + else: + logger.info(f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({ str(activityDate) }).') + break + elif args.activity_until_duration: + if lastUpDatetime < activityDate: + findings.append(create_finding_gitlab(project)) + else: + logger.info(f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({ str(activityDate) }).') + break + else: + findings.append(create_finding_gitlab(project)) + return findings -def get_gitlab_projects(args, gl): +def get_gitlab_projects_all(args, gl): if args.group: try: projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, obey_rate_limit=args.obey_rate_limit) @@ -144,6 +199,28 @@ def get_gitlab_projects(args, gl): projects = gl.projects.list(all=True, max_retries=12, obey_rate_limit=args.obey_rate_limit) return projects +def get_gitlab_projects_active_since(args, gl): + if args.group: + try: + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, order_by='last_activity_at', sort='desc', obey_rate_limit=args.obey_rate_limit) + except gitlab.exceptions.GitlabGetError: + logger.info(' Group does not exist.') + sys.exit(-1) + else: + projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='desc', obey_rate_limit=args.obey_rate_limit) + return projects + +def get_gitlab_projects_active_until(args, gl): + if args.group: + try: + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, order_by='last_activity_at', sort='asc', obey_rate_limit=args.obey_rate_limit) + except gitlab.exceptions.GitlabGetError: + logger.info(' Group does not exist.') + sys.exit(-1) + else: + projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='asc', obey_rate_limit=args.obey_rate_limit) + return projects + def gitlab_authenticate(args): gl: gitlab.Gitlab @@ -195,22 +272,59 @@ def respect_github_ratelimit(args, gh): def process_github_repos(args, gh): findings = [] org: Organization = gh.get_organization(args.organization) - repos: PaginatedList[Repository] = org.get_repos(type='all') + + # Respect time filtering based on "pushed_at" (not "updated_at") + # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. + # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. + # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. + duration = 0 + activityDeltaDatetime = datetime.now() + if args.activity_since_duration: + activityDuration = timeparse(args.activity_since_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitHub Repos (filtered by last activity since '+ str(activityDeltaDatetime) +' ago.)') + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') + elif args.activity_until_duration: + activityDuration = timeparse(args.activity_until_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitHub Repos (filtered by last activity until '+ str(activityDeltaDatetime) +' ago.)') + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') + else: + logger.info(' Get all GitHub Repos (not filtered)') + repos: PaginatedList[Repository] = org.get_repos(type='all') + + activityDate = datetime.now() - activityDeltaDatetime + for i in range(repos.totalCount): - process_github_repos_page(args, findings, repos.get_page(i), gh) + process_github_repos_page(args, findings, repos.get_page(i), gh, activityDeltaDatetime, activityDate) return findings - -def process_github_repos_page(args, findings, repos, gh): +def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, activityDate): repo: Repository for repo in repos: if repo.id not in args.ignore_repos: - logger.info(f' {len(findings) + 1} - {repo.name}') + logger.info(f' {len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - - + # respect time filtering + if args.activity_since_duration: + if repo.updated_at > activityDate: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + else: + logger.info(f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({ str(activityDate) }).') + break + elif args.activity_until_duration: + if repo.updated_at < activityDate: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + else: + logger.info(f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({ str(activityDate) }).') + break + else: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) def setup_github(args): if args.url: diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py index 71f2247039..402fb5fb09 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py @@ -1,4 +1,5 @@ import datetime +from datetime import timezone import unittest import git_repo_scanner from munch import Munch @@ -13,7 +14,7 @@ def test_process_gitlab_projects_with_no_ignore_list(self): projects = assemble_projects() args = get_args() # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) # then self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') self.assertEqual(findings[0]['name'], 'GitLab Repo', msg='Test finding output') @@ -25,7 +26,7 @@ def test_process_gitlab_projects_with_ignore_group(self): projects = assemble_projects() args = get_args(ignore_groups=33) # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) # then self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg='Test finding output') @@ -36,7 +37,7 @@ def test_process_gitlab_projects_with_ignore_project(self): projects = assemble_projects() args = get_args(ignore_projects=1) # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) # then self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg='Test finding output') @@ -118,14 +119,16 @@ def create_mocks(github_mock, org_mock, pag_mock, repos): def assemble_projects(): - project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=11, o_kind='group', + created = datetime.datetime(2020, 10, 10, tzinfo=timezone.utc).isoformat() + updated = datetime.datetime(2020, 11, 10, tzinfo=timezone.utc).isoformat() + project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created=created, + date_updated=updated, visibility='private', o_id=11, o_kind='group', o_name='name11') - project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=22, o_kind='user', + project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created=created, + date_updated=updated, visibility='private', o_id=22, o_kind='user', o_name='name22') - project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=33, o_kind='group', + project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created=created, + date_updated=updated, visibility='private', o_id=33, o_kind='group', o_name='name33') return [project1, project2, project3] @@ -148,20 +151,20 @@ def assemble_project(p_id, name, url, path, date_created, date_updated, visibili def assemble_repos(): - date = datetime.datetime(2020, 5, 17) + date = datetime.datetime(2020, 5, 17, tzinfo=timezone.utc) project1 = assemble_repository(p_id=1, name='name1', url='url1', path='path1', date_created=date, - date_updated=date, visibility=True, o_id=11, o_kind='organization', + date_updated=date, date_pushed=date, visibility=True, o_id=11, o_kind='organization', o_name='name11') project2 = assemble_repository(p_id=2, name='name2', url='url2', path='path2', date_created=date, - date_updated=date, visibility=False, o_id=22, o_kind='organization', + date_updated=date, date_pushed=date, visibility=False, o_id=22, o_kind='organization', o_name='name22') project3 = assemble_repository(p_id=3, name='name3', url='url3', path='path3', date_created=date, - date_updated=date, visibility=False, o_id=33, o_kind='organization', + date_updated=date, date_pushed=date, visibility=False, o_id=33, o_kind='organization', o_name='name33') return [project1, project2, project3] -def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, visibility: bool, o_id, +def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, date_pushed: datetime, visibility: bool, o_id, o_kind, o_name): repo = Munch() repo.id = p_id @@ -169,6 +172,7 @@ def assemble_repository(p_id, name, url, path, date_created: datetime, date_upda repo.html_url = url repo.full_name = path repo.created_at = date_created + repo.pushed_at = date_pushed repo.updated_at = date_updated repo.private = visibility repo.owner = Munch(type=o_kind, id=o_id, name=o_name) diff --git a/scanners/git-repo-scanner/scanner/requirements.txt b/scanners/git-repo-scanner/scanner/requirements.txt index 95198b0130..110b788b82 100644 --- a/scanners/git-repo-scanner/scanner/requirements.txt +++ b/scanners/git-repo-scanner/scanner/requirements.txt @@ -2,3 +2,5 @@ PyGithub == 1.54.1 python-gitlab == 2.6.0 munch == 2.5.0 mock == 4.0.2 +pytimeparse == 1.1.8 +pytz == 2021.1 From b1ac75121b0072f3fc2111678982b1bc33d9c4f2 Mon Sep 17 00:00:00 2001 From: rseedorff Date: Sun, 14 Mar 2021 01:40:36 +0000 Subject: [PATCH 04/24] Updating Helm Docs --- scanners/git-repo-scanner/README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index ebcd88b534..10258cc286 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -37,6 +37,10 @@ For type GitHub you can use the following options: - `--access-token`: Your personal GitHub access token. - `--ignore-repos`: A list of GitHub repository ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. @@ -49,6 +53,10 @@ For type GitLab you can use the following options: - `--ignore-groups`: A list of GitLab group ids you want to ignore - `--ignore-repos`: A list of GitLab project ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. From 22aea9e4586cb8bcdc822c5f95050d59e64ef8d4 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sun, 14 Mar 2021 20:26:14 +0100 Subject: [PATCH 05/24] Fixing missing python requirements bug in docker image. --- .github/workflows/ci.yaml | 2 +- scanners/git-repo-scanner/scanner/.dockerignore | 3 +++ scanners/git-repo-scanner/scanner/Dockerfile | 4 ++-- .../{git_repo_scanner_test.py => git_repo_scanner.test.py} | 0 4 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 scanners/git-repo-scanner/scanner/.dockerignore rename scanners/git-repo-scanner/scanner/{git_repo_scanner_test.py => git_repo_scanner.test.py} (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 65bc690705..b0c2a078d4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: working-directory: scanners/git-repo-scanner/scanner/ run: | pip install pytest - pytest ${{ matrix.unit }}_test.py + pytest ${{ matrix.unit }}.test.py # ---- Unit-Test | JavaScript ---- diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore new file mode 100644 index 0000000000..bf3dcb3752 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -0,0 +1,3 @@ +__pytest_cache +.pytest_cache +*.test.py diff --git a/scanners/git-repo-scanner/scanner/Dockerfile b/scanners/git-repo-scanner/scanner/Dockerfile index 48d3de19b1..50ccc700a1 100644 --- a/scanners/git-repo-scanner/scanner/Dockerfile +++ b/scanners/git-repo-scanner/scanner/Dockerfile @@ -1,5 +1,5 @@ FROM python:3.9.0-alpine -COPY git_repo_scanner.py /scripts/git_repo_scanner.py -RUN pip install PyGithub python-gitlab +COPY . /scripts/ +RUN pip install -r /scripts/requirements.txt CMD ["/bin/sh"] ENTRYPOINT ["python","/scripts/git_repo_scanner.py"] diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner.test.py similarity index 100% rename from scanners/git-repo-scanner/scanner/git_repo_scanner_test.py rename to scanners/git-repo-scanner/scanner/git_repo_scanner.test.py From 0b87d3a401f152d52b973456689c7455a83929b2 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Sun, 14 Mar 2021 20:31:12 +0100 Subject: [PATCH 06/24] Fixing failed pytest. --- .github/workflows/ci.yaml | 2 +- scanners/git-repo-scanner/scanner/.dockerignore | 2 +- .../{git_repo_scanner.test.py => git_repo_scanner_test.py} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename scanners/git-repo-scanner/scanner/{git_repo_scanner.test.py => git_repo_scanner_test.py} (100%) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b0c2a078d4..65bc690705 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: working-directory: scanners/git-repo-scanner/scanner/ run: | pip install pytest - pytest ${{ matrix.unit }}.test.py + pytest ${{ matrix.unit }}_test.py # ---- Unit-Test | JavaScript ---- diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore index bf3dcb3752..f88f2f169d 100644 --- a/scanners/git-repo-scanner/scanner/.dockerignore +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -1,3 +1,3 @@ __pytest_cache .pytest_cache -*.test.py +*_test.py diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py similarity index 100% rename from scanners/git-repo-scanner/scanner/git_repo_scanner.test.py rename to scanners/git-repo-scanner/scanner/git_repo_scanner_test.py From a3a777ccb604c5fdaa8eb6cb3e1f89b9c55b4efa Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 26 Mar 2021 09:45:42 +0100 Subject: [PATCH 07/24] git-repo-scanner refactor --- scanners/git-repo-scanner/README.md | 26 +- .../scanner/git_repo_scanner.py | 631 +++++++++--------- .../scanner/git_repo_scanner/__init__.py | 0 .../git_repo_scanner/abstract_scanner.py | 0 4 files changed, 343 insertions(+), 314 deletions(-) create mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py create mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index 10258cc286..f305bb2dd4 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -37,10 +37,14 @@ For type GitHub you can use the following options: - `--access-token`: Your personal GitHub access token. - `--ignore-repos`: A list of GitHub repository ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False -- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each - with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. -- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with - optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific + date expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each + with an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units + are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date + expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each with + an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm + ', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. @@ -49,14 +53,18 @@ for authentication. If not provided the rate limiting will kick in after about 3 For type GitLab you can use the following options: - `--url`: The url of the GitLab server. - `--access-token`: Your personal GitLab access token. -- `--group`: A specific GitLab group id you want to san, including subgroups. +- `--group`: A specific GitLab group id you want to scan, including subgroups. - `--ignore-groups`: A list of GitLab group ids you want to ignore - `--ignore-repos`: A list of GitLab project ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False -- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each - with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. -- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with - optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific + date expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each + with an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units + are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date + expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each with + an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm + ', 'h', 'd', 'w'. For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner.py index e762690ad9..dbd692994c 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner.py @@ -28,378 +28,399 @@ def main(): - args = get_parser_args() + args = get_parser_args() - findings = process(args) + findings = process(args) - logger.info(' Write findings to file...') - write_findings_to_file(args, findings) - logger.info(' Finished!') + logger.info(' Write findings to file...') + write_findings_to_file(args, findings) + logger.info(' Finished!') def process(args): - if args.git_type == 'gitlab': - return process_gitlab(args) - else: - return process_github(args) + if args.git_type == 'gitlab': + return process_gitlab(args) + else: + return process_github(args) def process_github(args): - try: - return parse_github(args) - except github.GithubException as e: - logger.info(f' Github API Exception: {e.status} -> {e.data["message"]}') - sys.exit(-1) + try: + return parse_github(args) + except github.GithubException as e: + logger.info(f' Github API Exception: {e.status} -> {e.data["message"]}') + sys.exit(-1) def process_gitlab(args): - try: - return parse_gitlab(args) - except gitlab.GitlabError as e: - logger.info(f' Gitlab API Exception: {e}') - sys.exit(-1) + try: + return parse_gitlab(args) + except gitlab.GitlabError as e: + logger.info(f' Gitlab API Exception: {e}') + sys.exit(-1) def write_findings_to_file(args, findings): - Path(args.file_output).mkdir(parents=True, exist_ok=True) - with open(f'{args.file_output}/git-repo-scanner-findings.json', 'w') as out: - json.dump(findings, out) + Path(args.file_output).mkdir(parents=True, exist_ok=True) + with open(f'{args.file_output}/git-repo-scanner-findings.json', 'w') as out: + json.dump(findings, out) def get_parser_args(args=None): - parser = argparse.ArgumentParser(description='Scan public or private git repositories of organizations or groups') - parser.add_argument('--git-type', - help='Repository type can be github or GitLab', - choices=['github', 'gitlab'], - required=True) - parser.add_argument('--file-output', - help='The path of the output file', - required=True), - parser.add_argument('--url', help='The GitLab url or a GitHub enterprise api url.', - required=False) - parser.add_argument('--access-token', - help='An access token for authentication', - required=False) - parser.add_argument('--organization', - help='The name of the GitHub organization to scan', - required=False) - parser.add_argument('--group', - help='The id of the GitLab group to scan', - required=False) - parser.add_argument('--ignore-repos', - help='A list of repo ids to ignore', - action='extend', - nargs='+', - type=int, - default=[], - required=False) - parser.add_argument('--ignore-groups', - help='A list of GitLab group ids to ignore', - action='extend', - nargs='+', - type=int, - default=[], - required=False) - parser.add_argument('--obey-rate-limit', - help='True to obey the rate limit of the GitLab or GitHub server (default), otherwise False', - type=bool, - default=True, - required=False) - parser.add_argument('--activity-since-duration', - help='Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration)', - type=str, - required=False) - parser.add_argument('--activity-until-duration', - help='Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration)', - type=str, - required=False) - - if args: - return parser.parse_args(args) - else: - return parser.parse_args() + parser = argparse.ArgumentParser(description='Scan public or private git repositories of organizations or groups') + parser.add_argument('--git-type', + help='Repository type can be github or GitLab', + choices=['github', 'gitlab'], + required=True) + parser.add_argument('--file-output', + help='The path of the output file', + required=True), + parser.add_argument('--url', help='The GitLab url or a GitHub enterprise api url.', + required=False) + parser.add_argument('--access-token', + help='An access token for authentication', + required=False) + parser.add_argument('--organization', + help='The name of the GitHub organization to scan', + required=False) + parser.add_argument('--group', + help='The id of the GitLab group to scan', + required=False) + parser.add_argument('--ignore-repos', + help='A list of repo ids to ignore', + action='extend', + nargs='+', + type=int, + default=[], + required=False) + parser.add_argument('--ignore-groups', + help='A list of GitLab group ids to ignore', + action='extend', + nargs='+', + type=int, + default=[], + required=False) + parser.add_argument('--obey-rate-limit', + help='True to obey the rate limit of the GitLab or GitHub server (default), otherwise False', + type=bool, + default=True, + required=False) + parser.add_argument('--activity-since-duration', + help='Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration)', + type=str, + required=False) + parser.add_argument('--activity-until-duration', + help='Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration)', + type=str, + required=False) + + if args: + return parser.parse_args(args) + else: + return parser.parse_args() def parse_gitlab(args): - gl: gitlab.Gitlab - if not args.url: - logger.info(' URL required for GitLab connection.') - sys.exit(-1) - - logger.info(' Gitlab authentication...') - gl = gitlab_authenticate(args) - - logger.info(' Gitlab retrieve all repositories...') - now_utc = pytz.utc.localize(datetime.utcnow()) - # Respect time filtering based on "pushed_at" (not "updated_at") - # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. - # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. - # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. - duration = 0 - activityDeltaDatetime = now_utc - if args.activity_since_duration: - activityDuration = timeparse(args.activity_since_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitLab Repos (filtered by last activity since '+ str(activityDeltaDatetime) +' ago.)') - - projects: List[Project] = get_gitlab_projects_active_since(args, gl) - elif args.activity_until_duration: - activityDuration = timeparse(args.activity_until_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitLab Repos (filtered by last activity until '+ str(activityDeltaDatetime) +' ago.)') - - projects: List[Project] = get_gitlab_projects_active_until(args, gl) - else: - logger.info(' Get all Gitlab Repos (not filtered)') - projects: List[Project] = get_gitlab_projects_all(args, gl) - - logger.info(' Process Projects...') - activityDate = now_utc - activityDeltaDatetime - findings = process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate) - - return findings + gl: gitlab.Gitlab + if not args.url: + logger.info(' URL required for GitLab connection.') + sys.exit(-1) + + logger.info(' Gitlab authentication...') + gl = gitlab_authenticate(args) + + logger.info(' Gitlab retrieve all repositories...') + now_utc = pytz.utc.localize(datetime.utcnow()) + # Respect time filtering based on "pushed_at" (not "updated_at") + # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. + # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. + # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. + duration = 0 + activityDeltaDatetime = now_utc + if args.activity_since_duration: + activityDuration = timeparse(args.activity_since_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitLab Repos (filtered by last activity since ' + str(activityDeltaDatetime) + ' ago.)') + + projects: List[Project] = get_gitlab_projects_active_since(args, gl) + elif args.activity_until_duration: + activityDuration = timeparse(args.activity_until_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitLab Repos (filtered by last activity until ' + str(activityDeltaDatetime) + ' ago.)') + + projects: List[Project] = get_gitlab_projects_active_until(args, gl) + else: + logger.info(' Get all Gitlab Repos (not filtered)') + projects: List[Project] = get_gitlab_projects_all(args, gl) + + logger.info(' Process Projects...') + activityDate = now_utc - activityDeltaDatetime + findings = process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate) + + return findings def process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate): - findings = [] - i = 1 - for project in projects: - if is_not_on_ignore_list_gitlab(project, args.ignore_groups, args.ignore_repos): - lastUpDatetime = datetime.fromisoformat(project.last_activity_at) - logger.info(f' {i} - Name: {project.name} - LastUpdate: {lastUpDatetime}') - i += 1 - - # respect time filtering - if args.activity_since_duration: - if lastUpDatetime > activityDate: - findings.append(create_finding_gitlab(project)) - else: - logger.info(f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({ str(activityDate) }).') - break - elif args.activity_until_duration: - if lastUpDatetime < activityDate: - findings.append(create_finding_gitlab(project)) - else: - logger.info(f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({ str(activityDate) }).') - break - else: - findings.append(create_finding_gitlab(project)) - - return findings + findings = [] + i = 1 + for project in projects: + if is_not_on_ignore_list_gitlab(project, args.ignore_groups, args.ignore_repos): + lastUpDatetime = datetime.fromisoformat(project.last_activity_at) + logger.info(f' {i} - Name: {project.name} - LastUpdate: {lastUpDatetime}') + i += 1 + + # respect time filtering + if args.activity_since_duration: + if lastUpDatetime > activityDate: + findings.append(create_finding_gitlab(project)) + else: + logger.info( + f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + elif args.activity_until_duration: + if lastUpDatetime < activityDate: + findings.append(create_finding_gitlab(project)) + else: + logger.info( + f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + else: + findings.append(create_finding_gitlab(project)) + + return findings def get_gitlab_projects_all(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, obey_rate_limit=args.obey_rate_limit) - return projects + if args.group: + try: + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, + obey_rate_limit=args.obey_rate_limit) + except gitlab.exceptions.GitlabGetError: + logger.info(' Group does not exist.') + sys.exit(-1) + else: + projects = gl.projects.list(all=True, max_retries=12, obey_rate_limit=args.obey_rate_limit) + return projects + def get_gitlab_projects_active_since(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, order_by='last_activity_at', sort='desc', obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='desc', obey_rate_limit=args.obey_rate_limit) - return projects + if args.group: + try: + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, + order_by='last_activity_at', + sort='desc', obey_rate_limit=args.obey_rate_limit) + except gitlab.exceptions.GitlabGetError: + logger.info(' Group does not exist.') + sys.exit(-1) + else: + projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='desc', + obey_rate_limit=args.obey_rate_limit) + return projects + def get_gitlab_projects_active_until(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, order_by='last_activity_at', sort='asc', obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='asc', obey_rate_limit=args.obey_rate_limit) - return projects + if args.group: + try: + projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, + order_by='last_activity_at', + sort='asc', obey_rate_limit=args.obey_rate_limit) + except gitlab.exceptions.GitlabGetError: + logger.info(' Group does not exist.') + sys.exit(-1) + else: + projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='asc', + obey_rate_limit=args.obey_rate_limit) + return projects def gitlab_authenticate(args): - gl: gitlab.Gitlab - if args.access_token: - try: - gl = gitlab.Gitlab(args.url, args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - gl = gitlab_authenticate_oauth(args) - else: - logger.info(' Access token required for GitLab authentication.') - sys.exit(-1) - logger.info(' Success') - return gl + gl: gitlab.Gitlab + if args.access_token: + try: + gl = gitlab.Gitlab(args.url, args.access_token) + gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + gl = gitlab_authenticate_oauth(args) + else: + logger.info(' Access token required for GitLab authentication.') + sys.exit(-1) + logger.info(' Success') + return gl def gitlab_authenticate_oauth(args): - try: - gl = gitlab.Gitlab(args.url, oauth_token=args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - logger.info(' No permission. Check your access token.') - sys.exit(-1) - return gl + try: + gl = gitlab.Gitlab(args.url, oauth_token=args.access_token) + gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + logger.info(' No permission. Check your access token.') + sys.exit(-1) + return gl def parse_github(args): - gh: github.Github = setup_github(args) + gh: github.Github = setup_github(args) - logger.info(' Process Repositories...') + logger.info(' Process Repositories...') + + if args.organization: + findings = process_github_repos(args, gh) + return findings + else: + logger.info(' No organization provided') + sys.exit(-1) - if args.organization: - findings = process_github_repos(args, gh) - return findings - else: - logger.info(' No organization provided') - sys.exit(-1) def respect_github_ratelimit(args, gh): - if args.obey_rate_limit: - api_limit = gh.get_rate_limit().core - reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) - seconds_until_reset = reset_timestamp - calendar.timegm(time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset - sleep_time = seconds_until_reset / api_limit.remaining + if args.obey_rate_limit: + api_limit = gh.get_rate_limit().core + reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) + seconds_until_reset = reset_timestamp - calendar.timegm( + time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset + sleep_time = seconds_until_reset / api_limit.remaining + + logger.info(' Checking Rate-Limit (' + str(args.obey_rate_limit) + ') [remainingApiCalls: ' + str( + api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str( + sleep_time) + ']') + time.sleep(sleep_time) - logger.info(' Checking Rate-Limit ('+ str(args.obey_rate_limit) +') [remainingApiCalls: ' + str(api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str(sleep_time) + ']') - time.sleep(sleep_time) def process_github_repos(args, gh): - findings = [] - org: Organization = gh.get_organization(args.organization) - - # Respect time filtering based on "pushed_at" (not "updated_at") - # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. - # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. - # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. - duration = 0 - activityDeltaDatetime = datetime.now() - if args.activity_since_duration: - activityDuration = timeparse(args.activity_since_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitHub Repos (filtered by last activity since '+ str(activityDeltaDatetime) +' ago.)') - - repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') - elif args.activity_until_duration: - activityDuration = timeparse(args.activity_until_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitHub Repos (filtered by last activity until '+ str(activityDeltaDatetime) +' ago.)') - - repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') - else: - logger.info(' Get all GitHub Repos (not filtered)') - repos: PaginatedList[Repository] = org.get_repos(type='all') - - activityDate = datetime.now() - activityDeltaDatetime - - for i in range(repos.totalCount): - process_github_repos_page(args, findings, repos.get_page(i), gh, activityDeltaDatetime, activityDate) - return findings + findings = [] + org: Organization = gh.get_organization(args.organization) + + # Respect time filtering based on "pushed_at" (not "updated_at") + # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. + # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. + # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. + duration = 0 + activityDeltaDatetime = datetime.now() + if args.activity_since_duration: + activityDuration = timeparse(args.activity_since_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitHub Repos (filtered by last activity since ' + str(activityDeltaDatetime) + ' ago.)') + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') + elif args.activity_until_duration: + activityDuration = timeparse(args.activity_until_duration) + activityDeltaDatetime = timedelta(seconds=activityDuration) + logger.info(' Get all GitHub Repos (filtered by last activity until ' + str(activityDeltaDatetime) + ' ago.)') + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') + else: + logger.info(' Get all GitHub Repos (not filtered)') + repos: PaginatedList[Repository] = org.get_repos(type='all') + + activityDate = datetime.now() - activityDeltaDatetime + + for i in range(repos.totalCount): + process_github_repos_page(args, findings, repos.get_page(i), gh, activityDeltaDatetime, activityDate) + return findings + def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, activityDate): - repo: Repository - for repo in repos: - if repo.id not in args.ignore_repos: - logger.info(f' {len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') - - # respect time filtering - if args.activity_since_duration: - if repo.updated_at > activityDate: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - else: - logger.info(f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({ str(activityDate) }).') - break - elif args.activity_until_duration: - if repo.updated_at < activityDate: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - else: - logger.info(f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({ str(activityDate) }).') - break - else: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) + repo: Repository + for repo in repos: + if repo.id not in args.ignore_repos: + logger.info( + f' {len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') + + # respect time filtering + if args.activity_since_duration: + if repo.updated_at > activityDate: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + else: + logger.info( + f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + elif args.activity_until_duration: + if repo.updated_at < activityDate: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + else: + logger.info( + f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + else: + findings.append(create_finding_github(repo)) + respect_github_ratelimit(args, gh) + def setup_github(args): - if args.url: - return setup_github_with_url(args) - else: - return setup_github_without_url(args) + if args.url: + return setup_github_with_url(args) + else: + return setup_github_without_url(args) def setup_github_without_url(args): - if args.access_token: - return github.Github(args.access_token) - else: - return github.Github() + if args.access_token: + return github.Github(args.access_token) + else: + return github.Github() def setup_github_with_url(args): - if args.access_token: - return github.Github(base_url=args.url, login_or_token=args.access_token) - else: - logger.info(' Access token required for github enterprise authentication.') - sys.exit(-1) + if args.access_token: + return github.Github(base_url=args.url, login_or_token=args.access_token) + else: + logger.info(' Access token required for github enterprise authentication.') + sys.exit(-1) def is_not_on_ignore_list_gitlab(project: Project, groups: List, repos: List): - id_project = project.id - kind = project.namespace['kind'] - id_namespace = project.namespace['id'] - if id_project in repos: - return False - if kind == 'group' and id_namespace in groups: - return False - return True + id_project = project.id + kind = project.namespace['kind'] + id_namespace = project.namespace['id'] + if id_project in repos: + return False + if kind == 'group' and id_namespace in groups: + return False + return True def create_finding_gitlab(project: Project): - return { - 'name': 'GitLab Repo', - 'description': 'A GitLab repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': project.id, - 'web_url': project.web_url, - 'full_name': project.path_with_namespace, - 'owner_type': project.namespace['kind'], - 'owner_id': project.namespace['id'], - 'owner_name': project.namespace['name'], - 'created_at': project.created_at, - 'last_activity_at': project.last_activity_at, - 'visibility': project.visibility + return { + 'name': 'GitLab Repo', + 'description': 'A GitLab repository', + 'category': 'Git Repository', + 'osi_layer': 'APPLICATION', + 'severity': 'INFORMATIONAL', + 'attributes': { + 'id': project.id, + 'web_url': project.web_url, + 'full_name': project.path_with_namespace, + 'owner_type': project.namespace['kind'], + 'owner_id': project.namespace['id'], + 'owner_name': project.namespace['name'], + 'created_at': project.created_at, + 'last_activity_at': project.last_activity_at, + 'visibility': project.visibility + } } - } def create_finding_github(repo: Repository): - return { - 'name': 'GitHub Repo', - 'description': 'A GitHub repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': repo.id, - 'web_url': repo.html_url, - 'full_name': repo.full_name, - 'owner_type': repo.owner.type, - 'owner_id': repo.owner.id, - 'owner_name': repo.owner.name, - 'created_at': repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'last_activity_at': repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'visibility': 'private' if repo.private else 'public' + return { + 'name': 'GitHub Repo', + 'description': 'A GitHub repository', + 'category': 'Git Repository', + 'osi_layer': 'APPLICATION', + 'severity': 'INFORMATIONAL', + 'attributes': { + 'id': repo.id, + 'web_url': repo.html_url, + 'full_name': repo.full_name, + 'owner_type': repo.owner.type, + 'owner_id': repo.owner.id, + 'owner_name': repo.owner.name, + 'created_at': repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + 'last_activity_at': repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + 'visibility': 'private' if repo.private else 'public' + } } - } if __name__ == '__main__': - main() + main() diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py new file mode 100644 index 0000000000..e69de29bb2 From a789d7601a6f83664548ceb66d8ff29a2523e6a3 Mon Sep 17 00:00:00 2001 From: paulschmelzer Date: Fri, 26 Mar 2021 08:46:08 +0000 Subject: [PATCH 08/24] Updating Helm Docs --- scanners/git-repo-scanner/README.md | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index f305bb2dd4..10258cc286 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -37,14 +37,10 @@ For type GitHub you can use the following options: - `--access-token`: Your personal GitHub access token. - `--ignore-repos`: A list of GitHub repository ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False -- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific - date expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each - with an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units - are 'm', 'h', 'd', 'w'. -- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date - expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each with - an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm - ', 'h', 'd', 'w'. +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. @@ -53,18 +49,14 @@ for authentication. If not provided the rate limiting will kick in after about 3 For type GitLab you can use the following options: - `--url`: The url of the GitLab server. - `--access-token`: Your personal GitLab access token. -- `--group`: A specific GitLab group id you want to scan, including subgroups. +- `--group`: A specific GitLab group id you want to san, including subgroups. - `--ignore-groups`: A list of GitLab group ids you want to ignore - `--ignore-repos`: A list of GitLab project ids you want to ignore - `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False -- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific - date expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each - with an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units - are 'm', 'h', 'd', 'w'. -- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date - expressed by a duration (now - duration). A duration string is a possibly signed sequence of decimal numbers, each with - an optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm - ', 'h', 'd', 'w'. +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. From 2e927cc255b8cb2220b6dcf0c16e2650c1586b57 Mon Sep 17 00:00:00 2001 From: Paul Date: Fri, 26 Mar 2021 13:33:47 +0100 Subject: [PATCH 09/24] revert new structure --- scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py | 0 .../git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py deleted file mode 100644 index e69de29bb2..0000000000 From 09706c33af2ecadb91d494f3980bd4e8a589b1f9 Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Fri, 26 Mar 2021 13:21:10 +0100 Subject: [PATCH 10/24] Fix wrong indention for python files --- .editorconfig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.editorconfig b/.editorconfig index f48effb0ca..a4e4605676 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,5 +14,8 @@ indent_size = 2 [*.go] indent_style = tab +[*.py] +indent_size = 4 + [Makefile] indent_style = tab From 54b8ca5b6e746582d4d622dfbae4b84a4e133d0b Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Fri, 26 Mar 2021 13:27:02 +0100 Subject: [PATCH 11/24] Split up files and implement and optimize GitLabScanner --- scanners/git-repo-scanner/scanner/Dockerfile | 2 +- .../__main__.py} | 272 +++++------------- .../git_repo_scanner/abstract_scanner.py | 38 +++ .../git_repo_scanner/gitlab_scanner.py | 108 +++++++ .../scanner/git_repo_scanner_test.py | 183 ------------ .../scanner/tests/__init__.py | 0 .../scanner/tests/git_repo_scanner_test.py | 188 ++++++++++++ 7 files changed, 409 insertions(+), 382 deletions(-) rename scanners/git-repo-scanner/scanner/{git_repo_scanner.py => git_repo_scanner/__main__.py} (51%) create mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py create mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py delete mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner_test.py create mode 100644 scanners/git-repo-scanner/scanner/tests/__init__.py create mode 100644 scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py diff --git a/scanners/git-repo-scanner/scanner/Dockerfile b/scanners/git-repo-scanner/scanner/Dockerfile index 50ccc700a1..01cceb1a8d 100644 --- a/scanners/git-repo-scanner/scanner/Dockerfile +++ b/scanners/git-repo-scanner/scanner/Dockerfile @@ -2,4 +2,4 @@ FROM python:3.9.0-alpine COPY . /scripts/ RUN pip install -r /scripts/requirements.txt CMD ["/bin/sh"] -ENTRYPOINT ["python","/scripts/git_repo_scanner.py"] +ENTRYPOINT ["python", "-m", "/scripts/git_repo_scanner"] diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py similarity index 51% rename from scanners/git-repo-scanner/scanner/git_repo_scanner.py rename to scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py index dbd692994c..7669ff1000 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py @@ -1,63 +1,76 @@ import argparse +import calendar +import json import logging import sys -import json -import calendar import time from datetime import datetime -import pytz - -from typing import List -from pathlib import Path - -# https://pypi.org/project/pytimeparse/ -from pytimeparse.timeparse import timeparse # https://docs.python.org/3/library/datetime.html from datetime import timedelta - -import gitlab -from gitlab.v4.objects import Project +from pathlib import Path import github +import gitlab +import pytz from github.Organization import Organization -from github.Repository import Repository from github.PaginatedList import PaginatedList +from github.Repository import Repository +from gitlab.v4.objects import Project +# https://pypi.org/project/pytimeparse/ +from pytimeparse.timeparse import timeparse -logging.basicConfig(level=logging.INFO) +from git_repo_scanner.gitlab_scanner import GitLabScanner + +log_format = '%(asctime)s - %(levelname)-7s - %(name)s - %(message)s' +logging.basicConfig(level=logging.INFO, format=log_format) logger = logging.getLogger('git_repo_scanner') +now_utc = pytz.utc.localize(datetime.utcnow()) + def main(): args = get_parser_args() findings = process(args) - logger.info(' Write findings to file...') + logger.info('Write findings to file...') write_findings_to_file(args, findings) - logger.info(' Finished!') + logger.info('Finished!') def process(args): if args.git_type == 'gitlab': - return process_gitlab(args) + scanner = GitLabScanner( + url=args.url, + access_token=args.access_token, + group=args.group, + ignored_groups=args.ignore_groups, + ignore_repos=args.ignore_repos, + obey_rate_limit=args.obey_rate_limit + ) else: - return process_github(args) - - -def process_github(args): - try: return parse_github(args) - except github.GithubException as e: - logger.info(f' Github API Exception: {e.status} -> {e.data["message"]}') - sys.exit(-1) - -def process_gitlab(args): try: - return parse_gitlab(args) + scanner.process( + args.activity_since_duration, + args.activity_until_duration + ) + except argparse.ArgumentError as e: + logger.error(f'Argument error: {e}') + sys.exit(1) + except gitlab.exceptions.GitlabAuthenticationError: + logger.info('No permission. Check your access token.') + sys.exit(1) + except github.GithubException as e: + logger.error(f'Github API Exception: {e.status} -> {e.data["message"]}') + sys.exit(2) except gitlab.GitlabError as e: - logger.info(f' Gitlab API Exception: {e}') - sys.exit(-1) + logger.error(f'Gitlab API Exception: {e}') + sys.exit(2) + except Exception as e: + logger.error(f'Unexpected error: {e}') + sys.exit(3) def write_findings_to_file(args, findings): @@ -66,8 +79,20 @@ def write_findings_to_file(args, findings): json.dump(findings, out) +def parse_duration_as_datetime(val: str): + try: + parsed = timeparse(val) + if parsed is None: + raise argparse.ArgumentTypeError(f'Not a valid duration: {val}.') + delta = timedelta(seconds=parsed) + return now_utc - delta + except Exception: + raise argparse.ArgumentTypeError(f'Not a valid duration: {val}.') + + def get_parser_args(args=None): - parser = argparse.ArgumentParser(description='Scan public or private git repositories of organizations or groups') + parser = argparse.ArgumentParser(prog='git_repo_scanner', + description='Scan public or private git repositories of organizations or groups') parser.add_argument('--git-type', help='Repository type can be github or GitLab', choices=['github', 'gitlab'], @@ -85,6 +110,7 @@ def get_parser_args(args=None): required=False) parser.add_argument('--group', help='The id of the GitLab group to scan', + type=int, required=False) parser.add_argument('--ignore-repos', help='A list of repo ids to ignore', @@ -106,168 +132,29 @@ def get_parser_args(args=None): default=True, required=False) parser.add_argument('--activity-since-duration', - help='Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration)', - type=str, + help='Return git repo findings with repo activity (e.g. commits) more recent than a specific ' + 'date expressed by a duration (now - duration)', + type=parse_duration_as_datetime, required=False) parser.add_argument('--activity-until-duration', - help='Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration)', - type=str, + help='Return git repo findings with repo activity (e.g. commits) older than a specific date ' + 'expressed by a duration (now - duration)', + type=parse_duration_as_datetime, required=False) - if args: - return parser.parse_args(args) - else: - return parser.parse_args() - - -def parse_gitlab(args): - gl: gitlab.Gitlab - if not args.url: - logger.info(' URL required for GitLab connection.') - sys.exit(-1) - - logger.info(' Gitlab authentication...') - gl = gitlab_authenticate(args) - - logger.info(' Gitlab retrieve all repositories...') - now_utc = pytz.utc.localize(datetime.utcnow()) - # Respect time filtering based on "pushed_at" (not "updated_at") - # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. - # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. - # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. - duration = 0 - activityDeltaDatetime = now_utc - if args.activity_since_duration: - activityDuration = timeparse(args.activity_since_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitLab Repos (filtered by last activity since ' + str(activityDeltaDatetime) + ' ago.)') - - projects: List[Project] = get_gitlab_projects_active_since(args, gl) - elif args.activity_until_duration: - activityDuration = timeparse(args.activity_until_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitLab Repos (filtered by last activity until ' + str(activityDeltaDatetime) + ' ago.)') - - projects: List[Project] = get_gitlab_projects_active_until(args, gl) - else: - logger.info(' Get all Gitlab Repos (not filtered)') - projects: List[Project] = get_gitlab_projects_all(args, gl) - - logger.info(' Process Projects...') - activityDate = now_utc - activityDeltaDatetime - findings = process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate) - - return findings - - -def process_gitlab_projects(args, projects, activityDeltaDatetime, activityDate): - findings = [] - i = 1 - for project in projects: - if is_not_on_ignore_list_gitlab(project, args.ignore_groups, args.ignore_repos): - lastUpDatetime = datetime.fromisoformat(project.last_activity_at) - logger.info(f' {i} - Name: {project.name} - LastUpdate: {lastUpDatetime}') - i += 1 - - # respect time filtering - if args.activity_since_duration: - if lastUpDatetime > activityDate: - findings.append(create_finding_gitlab(project)) - else: - logger.info( - f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') - break - elif args.activity_until_duration: - if lastUpDatetime < activityDate: - findings.append(create_finding_gitlab(project)) - else: - logger.info( - f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') - break - else: - findings.append(create_finding_gitlab(project)) - - return findings - - -def get_gitlab_projects_all(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, - obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, obey_rate_limit=args.obey_rate_limit) - return projects - - -def get_gitlab_projects_active_since(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, - order_by='last_activity_at', - sort='desc', obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='desc', - obey_rate_limit=args.obey_rate_limit) - return projects - - -def get_gitlab_projects_active_until(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True, - order_by='last_activity_at', - sort='asc', obey_rate_limit=args.obey_rate_limit) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12, order_by='last_activity_at', sort='asc', - obey_rate_limit=args.obey_rate_limit) - return projects - - -def gitlab_authenticate(args): - gl: gitlab.Gitlab - if args.access_token: - try: - gl = gitlab.Gitlab(args.url, args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - gl = gitlab_authenticate_oauth(args) - else: - logger.info(' Access token required for GitLab authentication.') - sys.exit(-1) - logger.info(' Success') - return gl - - -def gitlab_authenticate_oauth(args): - try: - gl = gitlab.Gitlab(args.url, oauth_token=args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - logger.info(' No permission. Check your access token.') - sys.exit(-1) - return gl + return parser.parse_args(args) def parse_github(args): gh: github.Github = setup_github(args) - logger.info(' Process Repositories...') + logger.info('Process Repositories...') if args.organization: findings = process_github_repos(args, gh) return findings else: - logger.info(' No organization provided') + logger.info('No organization provided') sys.exit(-1) @@ -279,7 +166,7 @@ def respect_github_ratelimit(args, gh): time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset sleep_time = seconds_until_reset / api_limit.remaining - logger.info(' Checking Rate-Limit (' + str(args.obey_rate_limit) + ') [remainingApiCalls: ' + str( + logger.info('Checking Rate-Limit (' + str(args.obey_rate_limit) + ') [remainingApiCalls: ' + str( api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str( sleep_time) + ']') time.sleep(sleep_time) @@ -298,17 +185,17 @@ def process_github_repos(args, gh): if args.activity_since_duration: activityDuration = timeparse(args.activity_since_duration) activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitHub Repos (filtered by last activity since ' + str(activityDeltaDatetime) + ' ago.)') + logger.info('Get all GitHub Repos (filtered by last activity since ' + str(activityDeltaDatetime) + 'ago.)') repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') elif args.activity_until_duration: activityDuration = timeparse(args.activity_until_duration) activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info(' Get all GitHub Repos (filtered by last activity until ' + str(activityDeltaDatetime) + ' ago.)') + logger.info('Get all GitHub Repos (filtered by last activity until ' + str(activityDeltaDatetime) + 'ago.)') repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') else: - logger.info(' Get all GitHub Repos (not filtered)') + logger.info('Get all GitHub Repos (not filtered)') repos: PaginatedList[Repository] = org.get_repos(type='all') activityDate = datetime.now() - activityDeltaDatetime @@ -323,7 +210,7 @@ def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, for repo in repos: if repo.id not in args.ignore_repos: logger.info( - f' {len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') + f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') # respect time filtering if args.activity_since_duration: @@ -332,7 +219,7 @@ def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, respect_github_ratelimit(args, gh) else: logger.info( - f' Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') + f'Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') break elif args.activity_until_duration: if repo.updated_at < activityDate: @@ -340,7 +227,7 @@ def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, respect_github_ratelimit(args, gh) else: logger.info( - f' Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') + f'Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') break else: findings.append(create_finding_github(repo)) @@ -365,21 +252,10 @@ def setup_github_with_url(args): if args.access_token: return github.Github(base_url=args.url, login_or_token=args.access_token) else: - logger.info(' Access token required for github enterprise authentication.') + logger.info('Access token required for github enterprise authentication.') sys.exit(-1) -def is_not_on_ignore_list_gitlab(project: Project, groups: List, repos: List): - id_project = project.id - kind = project.namespace['kind'] - id_namespace = project.namespace['id'] - if id_project in repos: - return False - if kind == 'group' and id_namespace in groups: - return False - return True - - def create_finding_gitlab(project: Project): return { 'name': 'GitLab Repo', diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py new file mode 100644 index 0000000000..1ebb66e072 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py @@ -0,0 +1,38 @@ +import abc +from datetime import datetime +from typing import Dict, List, Optional + +FINDING = Dict[str, any] + + +class AbstractScanner(abc.ABC): + + @property + @abc.abstractmethod + def git_type(self) -> str: + raise NotImplementedError() + + @abc.abstractmethod + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + raise NotImplementedError() + + def _create_finding(self, repo_id: str, web_url: str, full_name: str, owner_type: str, owner_id: str, + owner_name: str, created_at: str, last_activity_at: str, visibility: str) -> FINDING: + return { + 'name': f'{self.git_type} Repo', + 'description': f'A {self.git_type} repository', + 'category': 'Git Repository', + 'osi_layer': 'APPLICATION', + 'severity': 'INFORMATIONAL', + 'attributes': { + 'id': repo_id, + 'web_url': web_url, + 'full_name': full_name, + 'owner_type': owner_type, + 'owner_id': owner_id, + 'owner_name': owner_name, + 'created_at': created_at, + 'last_activity_at': last_activity_at, + 'visibility': visibility + } + } diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py new file mode 100644 index 0000000000..541321e8a1 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py @@ -0,0 +1,108 @@ +import argparse +import logging +from datetime import datetime +from typing import List, Optional + +import gitlab +from gitlab.v4.objects import Project, ProjectManager + +from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING + +logger = logging.getLogger('git_repo_scanner') + + +class GitLabScanner(AbstractScanner): + LOGGER = logging.getLogger('git_repo_scanner') + + def __init__(self, url: str, access_token: str, group: int, ignored_groups: List[int], ignore_repos: List[int], + obey_rate_limit: bool = True) -> None: + super().__init__() + if not url: + raise argparse.ArgumentError(None, 'URL required for GitLab connection.') + if not access_token: + raise argparse.ArgumentError(None, 'Access token required for GitLab authentication.') + + self._url = url + self._access_token = access_token + self._group = group + self._ignored_groups = ignored_groups + self._ignore_repos = ignore_repos + self._obey_rate_limit = obey_rate_limit + self._gl: Optional[gitlab.Gitlab] = None + + @property + def git_type(self) -> str: + return 'GitLab' + + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + self._authenticate() + + projects: List[Project] = self._get_projects(start_time, end_time) + return self._process_projects(projects) + + def _get_projects(self, start_time: Optional[datetime], end_time: Optional[datetime]): + logger.info(f'Get GitLab repositories with last activity between {start_time} and {end_time}.') + + project_manager: ProjectManager = self._gl.projects + options = dict( + all=True, + order_by='last_activity_at', + sort='desc', + obey_rate_limit=self._obey_rate_limit, + max_retries=12 + ) + if start_time is not None: + options['last_activity_after'] = start_time + if end_time is not None: + options['last_activity_before'] = end_time + + if self._group: + options['include_subgroups'] = True + project_manager = self._gl.groups.get(self._group).projects + + return project_manager.list(**options) + + def _process_projects(self, projects: List[Project]) -> List[FINDING]: + project_count = len(projects) + return [ + self._create_finding_from_project(project, i, project_count) + for i, project in enumerate(projects) + if self._is_not_ignored(project) + ] + + def _authenticate(self): + logger.info('Start GitLab authentication') + try: + self._gl = gitlab.Gitlab(self._url, private_token=self._access_token) + self._gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + self._gl = gitlab.Gitlab(self._url, oauth_token=self._access_token) + self._gl.auth() + + logger.info('GitLab authentication succeeded') + + def _is_not_ignored(self, project: Project) -> bool: + id_project = project.id + kind = project.namespace['kind'] + id_namespace = project.namespace['id'] + if id_project in self._ignore_repos: + return False + if kind == 'group' and id_namespace in self._ignored_groups: + return False + return True + + def _create_finding_from_project(self, project: Project, index: int, total: int) -> FINDING: + logger.info( + f'({index + 1}/{total}) Add finding for repo {project.name} with last activity at ' + f'{datetime.fromisoformat(project.last_activity_at)}') + return super()._create_finding( + project.id, + project.web_url, + project.path_with_namespace, + project.namespace['kind'], + project.namespace['id'], + project.namespace['name'], + project.created_at, + project.last_activity_at, + project.visibility + ) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py deleted file mode 100644 index 402fb5fb09..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py +++ /dev/null @@ -1,183 +0,0 @@ -import datetime -from datetime import timezone -import unittest -import git_repo_scanner -from munch import Munch -from mock import patch -from mock import MagicMock - - -class GitRepoScannerTests(unittest.TestCase): - - def test_process_gitlab_projects_with_no_ignore_list(self): - # given - projects = assemble_projects() - args = get_args() - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) - # then - self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') - self.assertEqual(findings[0]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[1]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[2]['name'], 'GitLab Repo', msg='Test finding output') - - def test_process_gitlab_projects_with_ignore_group(self): - # given - projects = assemble_projects() - args = get_args(ignore_groups=33) - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) - # then - self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg='Test finding output') - - def test_process_gitlab_projects_with_ignore_project(self): - # given - projects = assemble_projects() - args = get_args(ignore_projects=1) - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) - # then - self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg='Test finding output') - - @patch('github.Github') - @patch('github.Organization') - @patch('github.PaginatedList') - def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): - # given - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args() - # when - findings = git_repo_scanner.process_github_repos(args, github_mock) - # then - org_mock.get_repos.assert_called_with(type='all') - self.assertEqual(6, len(findings), msg='There should be exactly 6 findings') - for finding in findings: - self.assertEqual(finding['name'], 'GitHub Repo', msg='Test finding output') - - @patch('github.Github') - @patch('github.Organization') - @patch('github.PaginatedList') - def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): - # given - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args(ignore_projects=1, org='org') - # when - findings = git_repo_scanner.process_github_repos(args, github_mock) - # then - github_mock.get_organization.assert_called_with('org') - self.assertEqual(4, len(findings), msg='There should be exactly 4 findings') - - def test_setup_github_with_url_and_no_token_should_exit(self): - # given - args = get_args(url='url') - # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.setup_github(args) - # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') - - def test_parse_github_with_no_org_should_exit(self): - # given - args = get_args() - # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.parse_github(args) - # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') - - -def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): - args = ['--git-type', 'gitlab', - '--file-output', 'out', - '--obey-rate-limit', False, - '--ignore-repos', str(ignore_projects), - '--ignore-groups', str(ignore_groups)] - if url: - args.append('--url') - args.append(url) - if access_token: - args.append('--access-token') - args.append(access_token) - if org: - args.append('--organization') - args.append(org) - - return git_repo_scanner.get_parser_args(args) - - -def create_mocks(github_mock, org_mock, pag_mock, repos): - pag_mock.totalCount = 2 - pag_mock.get_page = MagicMock(return_value=repos) - org_mock.get_repos = MagicMock(return_value=pag_mock) - github_mock.get_organization = MagicMock(return_value=org_mock) - - -def assemble_projects(): - created = datetime.datetime(2020, 10, 10, tzinfo=timezone.utc).isoformat() - updated = datetime.datetime(2020, 11, 10, tzinfo=timezone.utc).isoformat() - project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created=created, - date_updated=updated, visibility='private', o_id=11, o_kind='group', - o_name='name11') - project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created=created, - date_updated=updated, visibility='private', o_id=22, o_kind='user', - o_name='name22') - project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created=created, - date_updated=updated, visibility='private', o_id=33, o_kind='group', - o_name='name33') - return [project1, project2, project3] - - -def assemble_project(p_id, name, url, path, date_created, date_updated, visibility, o_id, o_kind, o_name): - project = Munch() - project.id = p_id - project.name = name - project.web_url = url - project.path_with_namespace = path - project.created_at = date_created - project.last_activity_at = date_updated - project.visibility = visibility - project.namespace = { - 'kind': o_kind, - 'id': o_id, - 'name': o_name - } - return project - - -def assemble_repos(): - date = datetime.datetime(2020, 5, 17, tzinfo=timezone.utc) - project1 = assemble_repository(p_id=1, name='name1', url='url1', path='path1', date_created=date, - date_updated=date, date_pushed=date, visibility=True, o_id=11, o_kind='organization', - o_name='name11') - project2 = assemble_repository(p_id=2, name='name2', url='url2', path='path2', date_created=date, - date_updated=date, date_pushed=date, visibility=False, o_id=22, o_kind='organization', - o_name='name22') - project3 = assemble_repository(p_id=3, name='name3', url='url3', path='path3', date_created=date, - date_updated=date, date_pushed=date, visibility=False, o_id=33, o_kind='organization', - o_name='name33') - return [project1, project2, project3] - - -def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, date_pushed: datetime, visibility: bool, o_id, - o_kind, o_name): - repo = Munch() - repo.id = p_id - repo.name = name - repo.html_url = url - repo.full_name = path - repo.created_at = date_created - repo.pushed_at = date_pushed - repo.updated_at = date_updated - repo.private = visibility - repo.owner = Munch(type=o_kind, id=o_id, name=o_name) - return repo - - -if __name__ == '__main__': - unittest.main() diff --git a/scanners/git-repo-scanner/scanner/tests/__init__.py b/scanners/git-repo-scanner/scanner/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py new file mode 100644 index 0000000000..79c74d51f1 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py @@ -0,0 +1,188 @@ +import datetime +import unittest +from datetime import timezone + +from mock import MagicMock +from mock import patch +from munch import Munch + +import git_repo_scanner + + +class GitRepoScannerTests(unittest.TestCase): + + def test_process_gitlab_projects_with_no_ignore_list(self): + # given + projects = assemble_projects() + args = get_args() + # when + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + # then + self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') + self.assertEqual(findings[0]['name'], 'GitLab Repo', msg='Test finding output') + self.assertEqual(findings[1]['name'], 'GitLab Repo', msg='Test finding output') + self.assertEqual(findings[2]['name'], 'GitLab Repo', msg='Test finding output') + + def test_process_gitlab_projects_with_ignore_group(self): + # given + projects = assemble_projects() + args = get_args(ignore_groups=33) + # when + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + # then + self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') + self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg='Test finding output') + self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg='Test finding output') + + def test_process_gitlab_projects_with_ignore_project(self): + # given + projects = assemble_projects() + args = get_args(ignore_projects=1) + # when + findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + # then + self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') + self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg='Test finding output') + self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg='Test finding output') + + @patch('github.Github') + @patch('github.Organization') + @patch('github.PaginatedList') + def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): + # given + repos = assemble_repos() + create_mocks(github_mock, org_mock, pag_mock, repos) + args = get_args() + # when + findings = git_repo_scanner.process_github_repos(args, github_mock) + # then + org_mock.get_repos.assert_called_with(type='all') + self.assertEqual(6, len(findings), msg='There should be exactly 6 findings') + for finding in findings: + self.assertEqual(finding['name'], 'GitHub Repo', msg='Test finding output') + + @patch('github.Github') + @patch('github.Organization') + @patch('github.PaginatedList') + def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): + # given + repos = assemble_repos() + create_mocks(github_mock, org_mock, pag_mock, repos) + args = get_args(ignore_projects=1, org='org') + # when + findings = git_repo_scanner.process_github_repos(args, github_mock) + # then + github_mock.get_organization.assert_called_with('org') + self.assertEqual(4, len(findings), msg='There should be exactly 4 findings') + + def test_setup_github_with_url_and_no_token_should_exit(self): + # given + args = get_args(url='url') + # when + with self.assertRaises(SystemExit) as cm: + git_repo_scanner.setup_github(args) + # then + self.assertEqual(cm.exception.code, -1, msg='Process should exit') + + def test_parse_github_with_no_org_should_exit(self): + # given + args = get_args() + # when + with self.assertRaises(SystemExit) as cm: + git_repo_scanner.parse_github(args) + # then + self.assertEqual(cm.exception.code, -1, msg='Process should exit') + + +def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): + args = ['--git-type', 'gitlab', + '--file-output', 'out', + '--obey-rate-limit', False, + '--ignore-repos', str(ignore_projects), + '--ignore-groups', str(ignore_groups)] + if url: + args.append('--url') + args.append(url) + if access_token: + args.append('--access-token') + args.append(access_token) + if org: + args.append('--organization') + args.append(org) + + return git_repo_scanner.get_parser_args(args) + + +def create_mocks(github_mock, org_mock, pag_mock, repos): + pag_mock.totalCount = 2 + pag_mock.get_page = MagicMock(return_value=repos) + org_mock.get_repos = MagicMock(return_value=pag_mock) + github_mock.get_organization = MagicMock(return_value=org_mock) + + +def assemble_projects(): + created = datetime.datetime(2020, 10, 10, tzinfo=timezone.utc).isoformat() + updated = datetime.datetime(2020, 11, 10, tzinfo=timezone.utc).isoformat() + project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created=created, + date_updated=updated, visibility='private', o_id=11, o_kind='group', + o_name='name11') + project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created=created, + date_updated=updated, visibility='private', o_id=22, o_kind='user', + o_name='name22') + project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created=created, + date_updated=updated, visibility='private', o_id=33, o_kind='group', + o_name='name33') + return [project1, project2, project3] + + +def assemble_project(p_id, name, url, path, date_created, date_updated, visibility, o_id, o_kind, o_name): + project = Munch() + project.id = p_id + project.name = name + project.web_url = url + project.path_with_namespace = path + project.created_at = date_created + project.last_activity_at = date_updated + project.visibility = visibility + project.namespace = { + 'kind': o_kind, + 'id': o_id, + 'name': o_name + } + return project + + +def assemble_repos(): + date = datetime.datetime(2020, 5, 17, tzinfo=timezone.utc) + project1 = assemble_repository(p_id=1, name='name1', url='url1', path='path1', date_created=date, + date_updated=date, date_pushed=date, visibility=True, o_id=11, o_kind='organization', + o_name='name11') + project2 = assemble_repository(p_id=2, name='name2', url='url2', path='path2', date_created=date, + date_updated=date, date_pushed=date, visibility=False, o_id=22, + o_kind='organization', + o_name='name22') + project3 = assemble_repository(p_id=3, name='name3', url='url3', path='path3', date_created=date, + date_updated=date, date_pushed=date, visibility=False, o_id=33, + o_kind='organization', + o_name='name33') + return [project1, project2, project3] + + +def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, date_pushed: datetime, + visibility: bool, o_id, + o_kind, o_name): + repo = Munch() + repo.id = p_id + repo.name = name + repo.html_url = url + repo.full_name = path + repo.created_at = date_created + repo.pushed_at = date_pushed + repo.updated_at = date_updated + repo.private = visibility + repo.owner = Munch(type=o_kind, id=o_id, name=o_name) + return repo + + +if __name__ == '__main__': + unittest.main() From d86c38ed0f348008a27125c6f176a2db9dca3b0a Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Mon, 29 Mar 2021 08:30:56 +0200 Subject: [PATCH 12/24] Ignore Python cache directories --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 160ec0b5f4..0b709a200d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ coverage/ **.log **/*.monopic .s3_credentials +**/__pycache__ + ### IntelliJ IDEA ### .idea From d34711dfaf3234c656d257f91d82ac9639aebc29 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 30 Mar 2021 13:58:10 +0200 Subject: [PATCH 13/24] git-repo-scanner refactor --- .../git-repo-scanner/scanner/.dockerignore | 2 +- .../scanner/git_repo_scanner/__main__.py | 174 ++---------------- .../git_repo_scanner/github_scanner.py | 129 +++++++++++++ 3 files changed, 149 insertions(+), 156 deletions(-) create mode 100644 scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore index f88f2f169d..d42a947b35 100644 --- a/scanners/git-repo-scanner/scanner/.dockerignore +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -1,3 +1,3 @@ __pytest_cache .pytest_cache -*_test.py +/tests diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py index 7669ff1000..70bf91cb9f 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py @@ -19,6 +19,8 @@ # https://pypi.org/project/pytimeparse/ from pytimeparse.timeparse import timeparse +from git_repo_scanner.abstract_scanner import AbstractScanner +from git_repo_scanner.github_scanner import GitHubScanner from git_repo_scanner.gitlab_scanner import GitLabScanner log_format = '%(asctime)s - %(levelname)-7s - %(name)s - %(message)s' @@ -31,6 +33,10 @@ def main(): args = get_parser_args() + if not args.git_type: + logger.info('Argument error: No git type specified') + sys.exit(1) + findings = process(args) logger.info('Write findings to file...') @@ -39,6 +45,8 @@ def main(): def process(args): + scanner: AbstractScanner + if args.git_type == 'gitlab': scanner = GitLabScanner( url=args.url, @@ -48,11 +56,20 @@ def process(args): ignore_repos=args.ignore_repos, obey_rate_limit=args.obey_rate_limit ) + if args.git_type == 'github': + scanner = GitHubScanner( + url=args.url, + access_token=args.access_token, + organization=args.organization, + ignore_repos=args.ignore_repos, + obey_rate_limit=args.obey_rate_limit + ) else: - return parse_github(args) + logger.info('Argument error: Unkown git type') + sys.exit(1) try: - scanner.process( + return scanner.process( args.activity_since_duration, args.activity_until_duration ) @@ -145,158 +162,5 @@ def get_parser_args(args=None): return parser.parse_args(args) -def parse_github(args): - gh: github.Github = setup_github(args) - - logger.info('Process Repositories...') - - if args.organization: - findings = process_github_repos(args, gh) - return findings - else: - logger.info('No organization provided') - sys.exit(-1) - - -def respect_github_ratelimit(args, gh): - if args.obey_rate_limit: - api_limit = gh.get_rate_limit().core - reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) - seconds_until_reset = reset_timestamp - calendar.timegm( - time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset - sleep_time = seconds_until_reset / api_limit.remaining - - logger.info('Checking Rate-Limit (' + str(args.obey_rate_limit) + ') [remainingApiCalls: ' + str( - api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str( - sleep_time) + ']') - time.sleep(sleep_time) - - -def process_github_repos(args, gh): - findings = [] - org: Organization = gh.get_organization(args.organization) - - # Respect time filtering based on "pushed_at" (not "updated_at") - # The difference is that "pushed_at" represents the date and time of the last commit, whereas the "updated_at" represents the date and time of the last change the the repository. - # A change to the repository might be a commit, but it may also be other things, such as changing the description of the repo, creating wiki pages, etc. - # In other words, commits are a subset of updates, and the pushed_at timestamp will therefore either be the same as the updated_at timestamp, or it will be an earlier timestamp. - duration = 0 - activityDeltaDatetime = datetime.now() - if args.activity_since_duration: - activityDuration = timeparse(args.activity_since_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info('Get all GitHub Repos (filtered by last activity since ' + str(activityDeltaDatetime) + 'ago.)') - - repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') - elif args.activity_until_duration: - activityDuration = timeparse(args.activity_until_duration) - activityDeltaDatetime = timedelta(seconds=activityDuration) - logger.info('Get all GitHub Repos (filtered by last activity until ' + str(activityDeltaDatetime) + 'ago.)') - - repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') - else: - logger.info('Get all GitHub Repos (not filtered)') - repos: PaginatedList[Repository] = org.get_repos(type='all') - - activityDate = datetime.now() - activityDeltaDatetime - - for i in range(repos.totalCount): - process_github_repos_page(args, findings, repos.get_page(i), gh, activityDeltaDatetime, activityDate) - return findings - - -def process_github_repos_page(args, findings, repos, gh, activityDeltaDatetime, activityDate): - repo: Repository - for repo in repos: - if repo.id not in args.ignore_repos: - logger.info( - f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') - - # respect time filtering - if args.activity_since_duration: - if repo.updated_at > activityDate: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - else: - logger.info( - f'Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') - break - elif args.activity_until_duration: - if repo.updated_at < activityDate: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - else: - logger.info( - f'Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') - break - else: - findings.append(create_finding_github(repo)) - respect_github_ratelimit(args, gh) - - -def setup_github(args): - if args.url: - return setup_github_with_url(args) - else: - return setup_github_without_url(args) - - -def setup_github_without_url(args): - if args.access_token: - return github.Github(args.access_token) - else: - return github.Github() - - -def setup_github_with_url(args): - if args.access_token: - return github.Github(base_url=args.url, login_or_token=args.access_token) - else: - logger.info('Access token required for github enterprise authentication.') - sys.exit(-1) - - -def create_finding_gitlab(project: Project): - return { - 'name': 'GitLab Repo', - 'description': 'A GitLab repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': project.id, - 'web_url': project.web_url, - 'full_name': project.path_with_namespace, - 'owner_type': project.namespace['kind'], - 'owner_id': project.namespace['id'], - 'owner_name': project.namespace['name'], - 'created_at': project.created_at, - 'last_activity_at': project.last_activity_at, - 'visibility': project.visibility - } - } - - -def create_finding_github(repo: Repository): - return { - 'name': 'GitHub Repo', - 'description': 'A GitHub repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': repo.id, - 'web_url': repo.html_url, - 'full_name': repo.full_name, - 'owner_type': repo.owner.type, - 'owner_id': repo.owner.id, - 'owner_name': repo.owner.name, - 'created_at': repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'last_activity_at': repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'visibility': 'private' if repo.private else 'public' - } - } - - if __name__ == '__main__': main() diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py new file mode 100644 index 0000000000..97e1a86b50 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py @@ -0,0 +1,129 @@ +import argparse +import logging +import time +from calendar import calendar +from datetime import datetime +from typing import Optional, List + +import github +from github.Organization import Organization +from github.PaginatedList import PaginatedList +from github.Repository import Repository + +from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING + + +class GitHubScanner(AbstractScanner): + LOGGER = logging.getLogger('git_repo_scanner') + + def __init__(self, url: str, access_token: str, organization: str, ignore_repos: List[int], + obey_rate_limit: bool = True) -> None: + super().__init__() + if not url: + raise argparse.ArgumentError(None, 'URL required for GitLab connection.') + if not organization: + raise argparse.ArgumentError(None, 'Organization required for GitLab connection.') + + self._url = url + self._access_token = access_token + self._organization = organization + self._ignore_repos = ignore_repos + self._obey_rate_limit = obey_rate_limit + self._gh: Optional[github.Github] = None + + @property + def git_type(self) -> str: + return 'GitHub' + + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + self._setup() + return self._process_repos(start_time, end_time) + + def _process_repos(self, start_time: Optional[datetime], end_time: Optional[datetime]): + findings = [] + org: Organization = self._gh.get_organization(self._organization) + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') + + for i in range(repos.totalCount): + _process_repos_page(findings, repos.get_page(i), start_time, end_time) + return findings + + def _process_repos_page(self, + findings: List[FINDING], + repos: PaginatedList[Repository], + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None): + repo: Repository + for repo in repos: + if repo.id not in self._ignore_repos: + self.LOGGER.info( + f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') + + # respect time filtering + if start_time: + if repo.updated_at > start_time: + findings.append(self._create_finding_from_repo(repo)) + self._respect_github_ratelimit() + else: + self.LOGGER.info( + f'Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + elif end_time: + if repo.updated_at < end_time: + findings.append(self._create_finding_from_repo(repo)) + self._respect_github_ratelimit() + else: + self.LOGGER.info( + f'Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') + break + else: + findings.append(self._create_finding_from_repo(repo)) + self._respect_github_ratelimit() + + def _respect_github_ratelimit(self): + if self._obey_rate_limit: + api_limit = self._gh.get_rate_limit().core + reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) + seconds_until_reset = reset_timestamp - calendar.timegm( + time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset + sleep_time = seconds_until_reset / api_limit.remaining + + self.LOGGER.info('Checking Rate-Limit (' + str(self._obey_rate_limit) + ') [remainingApiCalls: ' + str( + api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str( + sleep_time) + ']') + time.sleep(sleep_time) + + def _setup(self): + if self._url: + self._setup_with_url() + else: + self._setup_without_url() + + def _setup_without_url(self): + if self._access_token: + self._gh = github.Github(self._access_token) + else: + self._gh = github.Github() + + def _setup_with_url(self): + if self._access_token: + self._gh = github.Github(base_url=self._url, login_or_token=self._access_token) + else: + raise argparse.ArgumentError(None, 'Access token required for github enterprise authentication.') + + def _create_finding_from_repo(self, repo: Repository, index: int, total: int) -> FINDING: + self.LOGGER.info( + f'({index + 1}/{total}) Add finding for repo {repo.full_name} with last activity at ' + f'{repo.updated_at}') + return super()._create_finding( + str(repo.id), + repo.html_url, + repo.full_name, + repo.owner.type, + str(repo.owner.id), + repo.owner.name, + repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + 'private' if repo.private else 'public' + ) From e0424637aa6efcbacc16fdf2fb2b286306708172 Mon Sep 17 00:00:00 2001 From: Paul Date: Tue, 30 Mar 2021 18:41:49 +0200 Subject: [PATCH 14/24] adds github class --- .../scanner/git_repo_scanner/__main__.py | 6 -- .../git_repo_scanner/github_scanner.py | 58 ++++++++++--------- 2 files changed, 30 insertions(+), 34 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py index 70bf91cb9f..c40aecdf93 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py @@ -1,9 +1,7 @@ import argparse -import calendar import json import logging import sys -import time from datetime import datetime # https://docs.python.org/3/library/datetime.html from datetime import timedelta @@ -12,10 +10,6 @@ import github import gitlab import pytz -from github.Organization import Organization -from github.PaginatedList import PaginatedList -from github.Repository import Repository -from gitlab.v4.objects import Project # https://pypi.org/project/pytimeparse/ from pytimeparse.timeparse import timeparse diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py index 97e1a86b50..0df34a2b9d 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py @@ -19,8 +19,6 @@ class GitHubScanner(AbstractScanner): def __init__(self, url: str, access_token: str, organization: str, ignore_repos: List[int], obey_rate_limit: bool = True) -> None: super().__init__() - if not url: - raise argparse.ArgumentError(None, 'URL required for GitLab connection.') if not organization: raise argparse.ArgumentError(None, 'Organization required for GitLab connection.') @@ -43,15 +41,18 @@ def _process_repos(self, start_time: Optional[datetime], end_time: Optional[date findings = [] org: Organization = self._gh.get_organization(self._organization) - repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='desc') + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') + + if start_time: + repos = org.get_repos(type='all', sort='pushed', direction='desc') for i in range(repos.totalCount): - _process_repos_page(findings, repos.get_page(i), start_time, end_time) + self._process_repos_page(findings, repos.get_page(i), start_time, end_time) return findings def _process_repos_page(self, findings: List[FINDING], - repos: PaginatedList[Repository], + repos: List[Repository], start_time: Optional[datetime] = None, end_time: Optional[datetime] = None): repo: Repository @@ -60,26 +61,30 @@ def _process_repos_page(self, self.LOGGER.info( f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') - # respect time filtering - if start_time: - if repo.updated_at > start_time: - findings.append(self._create_finding_from_repo(repo)) - self._respect_github_ratelimit() - else: - self.LOGGER.info( - f'Reached activity limit! Ignoring all repos with latest activity since `{activityDeltaDatetime}` ago ({str(activityDate)}).') - break - elif end_time: - if repo.updated_at < end_time: - findings.append(self._create_finding_from_repo(repo)) - self._respect_github_ratelimit() - else: - self.LOGGER.info( - f'Reached activity limit! Ignoring all repos with latest activity until `{activityDeltaDatetime}` ago ({str(activityDate)}).') + if start_time or end_time: + isInTimeFrame = self._check_repo_is_in_time_frame(repo.pushed_at, start_time, end_time) + if not isInTimeFrame: break - else: - findings.append(self._create_finding_from_repo(repo)) - self._respect_github_ratelimit() + + findings.append(self._create_finding_from_repo(repo)) + self._respect_github_ratelimit() + + def _check_repo_is_in_time_frame(self, + pushed_at: datetime, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None): + if start_time: + if pushed_at > start_time: + return True + else: + self.LOGGER.info(f'Reached activity limit! Ignoring all repos with activity since `{start_time}`.') + return False + elif end_time: + if pushed_at < end_time: + return True + else: + self.LOGGER.info(f'Reached activity limit! Ignoring all repos with activity until `{end_time}`.') + return False def _respect_github_ratelimit(self): if self._obey_rate_limit: @@ -112,10 +117,7 @@ def _setup_with_url(self): else: raise argparse.ArgumentError(None, 'Access token required for github enterprise authentication.') - def _create_finding_from_repo(self, repo: Repository, index: int, total: int) -> FINDING: - self.LOGGER.info( - f'({index + 1}/{total}) Add finding for repo {repo.full_name} with last activity at ' - f'{repo.updated_at}') + def _create_finding_from_repo(self, repo: Repository) -> FINDING: return super()._create_finding( str(repo.id), repo.html_url, From 749879650cf983296ffb97db6430a27961e417d5 Mon Sep 17 00:00:00 2001 From: Paul Date: Wed, 31 Mar 2021 17:36:10 +0200 Subject: [PATCH 15/24] - finishes refactoring - fixed tests - CLI execution in docker and tests in pipeline needs to be fixed --- .../git_repo_scanner/github_scanner.py | 14 ++-- .../git_repo_scanner/gitlab_scanner.py | 6 +- .../scanner/tests/git_repo_scanner_test.py | 79 ++++++++++--------- 3 files changed, 54 insertions(+), 45 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py index 0df34a2b9d..b49b6ae377 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py @@ -1,7 +1,7 @@ import argparse import logging import time -from calendar import calendar +from calendar import timegm from datetime import datetime from typing import Optional, List @@ -16,11 +16,13 @@ class GitHubScanner(AbstractScanner): LOGGER = logging.getLogger('git_repo_scanner') - def __init__(self, url: str, access_token: str, organization: str, ignore_repos: List[int], + def __init__(self, url: Optional[str], access_token: Optional[str], organization: str, ignore_repos: List[int], obey_rate_limit: bool = True) -> None: super().__init__() if not organization: - raise argparse.ArgumentError(None, 'Organization required for GitLab connection.') + raise argparse.ArgumentError(None, 'Organization required for GitHab connection.') + if url and not access_token: + raise argparse.ArgumentError(None, 'Access token required for GitHab connection.') self._url = url self._access_token = access_token @@ -89,9 +91,9 @@ def _check_repo_is_in_time_frame(self, def _respect_github_ratelimit(self): if self._obey_rate_limit: api_limit = self._gh.get_rate_limit().core - reset_timestamp = calendar.timegm(api_limit.reset.timetuple()) - seconds_until_reset = reset_timestamp - calendar.timegm( - time.gmtime()) + 5 # add 5 seconds to be sure the rate limit has been reset + reset_timestamp = timegm(api_limit.reset.timetuple()) + # add 5 seconds to be sure the rate limit has been reset + seconds_until_reset = reset_timestamp - timegm(time.gmtime()) + 5 sleep_time = seconds_until_reset / api_limit.remaining self.LOGGER.info('Checking Rate-Limit (' + str(self._obey_rate_limit) + ') [remainingApiCalls: ' + str( diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py index 541321e8a1..233a376195 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py @@ -14,7 +14,11 @@ class GitLabScanner(AbstractScanner): LOGGER = logging.getLogger('git_repo_scanner') - def __init__(self, url: str, access_token: str, group: int, ignored_groups: List[int], ignore_repos: List[int], + def __init__(self, url: str, + access_token: str, + group: Optional[int], + ignored_groups: List[int], + ignore_repos: List[int], obey_rate_limit: bool = True) -> None: super().__init__() if not url: diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py index 79c74d51f1..d8f7db2814 100644 --- a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py +++ b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py @@ -1,101 +1,104 @@ -import datetime +import argparse import unittest +import gitlab +import datetime from datetime import timezone +from gitlab.v4.objects import Project, ProjectManager from mock import MagicMock from mock import patch from munch import Munch -import git_repo_scanner +from git_repo_scanner.__main__ import get_parser_args + +from git_repo_scanner.github_scanner import GitHubScanner +from git_repo_scanner.gitlab_scanner import GitLabScanner class GitRepoScannerTests(unittest.TestCase): + @property + def wrong_output_msg(self) -> str: + return 'Test finding output' + def test_process_gitlab_projects_with_no_ignore_list(self): # given + scanner = GitLabScanner('url', 'token', None, [], []) projects = assemble_projects() - args = get_args() # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + findings = scanner._process_projects(projects) # then self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') - self.assertEqual(findings[0]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[1]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[2]['name'], 'GitLab Repo', msg='Test finding output') + self.assertEqual(findings[0]['name'], 'GitLab Repo', msg=self.wrong_output_msg) + self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) + self.assertEqual(findings[2]['attributes']['web_url'], 'url3', msg=self.wrong_output_msg) def test_process_gitlab_projects_with_ignore_group(self): # given + scanner = GitLabScanner('url', 'token', None, [33], []) projects = assemble_projects() - args = get_args(ignore_groups=33) # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + findings = scanner._process_projects(projects) # then self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg='Test finding output') + self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) def test_process_gitlab_projects_with_ignore_project(self): # given + scanner = GitLabScanner('url', 'token', None, [], [1]) projects = assemble_projects() - args = get_args(ignore_projects=1) # when - findings = git_repo_scanner.process_gitlab_projects(args, projects, 0, datetime.datetime.now()) + findings = scanner._process_projects(projects) # then self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg='Test finding output') + self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg=self.wrong_output_msg) @patch('github.Github') @patch('github.Organization') @patch('github.PaginatedList') def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): # given + scanner = GitHubScanner('url', 'token', 'org', []) repos = assemble_repos() create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args() + scanner._gh = github_mock # when - findings = git_repo_scanner.process_github_repos(args, github_mock) + findings = scanner._process_repos(None, None) # then - org_mock.get_repos.assert_called_with(type='all') + org_mock.get_repos.assert_called_with(type='all', sort='pushed', direction='asc') self.assertEqual(6, len(findings), msg='There should be exactly 6 findings') for finding in findings: - self.assertEqual(finding['name'], 'GitHub Repo', msg='Test finding output') + self.assertEqual(finding['name'], 'GitHub Repo', msg=self.wrong_output_msg) @patch('github.Github') @patch('github.Organization') @patch('github.PaginatedList') def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): # given + scanner = GitHubScanner('url', 'token', 'org', [1]) repos = assemble_repos() create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args(ignore_projects=1, org='org') + scanner._gh = github_mock # when - findings = git_repo_scanner.process_github_repos(args, github_mock) + findings = scanner._process_repos(None, None) # then github_mock.get_organization.assert_called_with('org') self.assertEqual(4, len(findings), msg='There should be exactly 4 findings') def test_setup_github_with_url_and_no_token_should_exit(self): - # given - args = get_args(url='url') - # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.setup_github(args) - # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') - - def test_parse_github_with_no_org_should_exit(self): - # given - args = get_args() # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.parse_github(args) + with self.assertRaises(argparse.ArgumentError) as cm: + GitHubScanner('url', None, 'org', []) # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') + self.assertEqual(cm.exception.args[1], 'Access token required for GitHab connection.', + msg='Process should exit') def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): - args = ['--git-type', 'gitlab', + args = ['--git-type', 'someType', '--file-output', 'out', '--obey-rate-limit', False, '--ignore-repos', str(ignore_projects), @@ -110,7 +113,7 @@ def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, or args.append('--organization') args.append(org) - return git_repo_scanner.get_parser_args(args) + return get_parser_args(args) def create_mocks(github_mock, org_mock, pag_mock, repos): @@ -136,7 +139,7 @@ def assemble_projects(): def assemble_project(p_id, name, url, path, date_created, date_updated, visibility, o_id, o_kind, o_name): - project = Munch() + project = Project(ProjectManager(gitlab), {}) project.id = p_id project.name = name project.web_url = url From 643a44eb19ccb97736bb4ceacf53c6165dfed3a9 Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Thu, 8 Apr 2021 15:29:41 +0200 Subject: [PATCH 16/24] Fix missing elif that rendered starting for "gitlab" unreachable --- .../git-repo-scanner/scanner/git_repo_scanner/__main__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py index c40aecdf93..4a3e78edbe 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py @@ -50,7 +50,7 @@ def process(args): ignore_repos=args.ignore_repos, obey_rate_limit=args.obey_rate_limit ) - if args.git_type == 'github': + elif args.git_type == 'github': scanner = GitHubScanner( url=args.url, access_token=args.access_token, @@ -59,7 +59,7 @@ def process(args): obey_rate_limit=args.obey_rate_limit ) else: - logger.info('Argument error: Unkown git type') + logger.info('Argument error: Unknown git type') sys.exit(1) try: From 6eb76121c808af8f1c82f51d038245693ec2b794 Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Thu, 8 Apr 2021 15:59:40 +0200 Subject: [PATCH 17/24] Use correct work dir so that Python can find the module --- .github/workflows/ci.yaml | 2 +- scanners/git-repo-scanner/scanner/Dockerfile | 3 ++- .../git-repo-scanner/templates/git-repo-scanner-scan-type.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 65bc690705..5bcfda3143 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -41,7 +41,7 @@ jobs: working-directory: scanners/git-repo-scanner/scanner/ run: | pip install pytest - pytest ${{ matrix.unit }}_test.py + pytest # ---- Unit-Test | JavaScript ---- diff --git a/scanners/git-repo-scanner/scanner/Dockerfile b/scanners/git-repo-scanner/scanner/Dockerfile index 01cceb1a8d..f99e726a1c 100644 --- a/scanners/git-repo-scanner/scanner/Dockerfile +++ b/scanners/git-repo-scanner/scanner/Dockerfile @@ -2,4 +2,5 @@ FROM python:3.9.0-alpine COPY . /scripts/ RUN pip install -r /scripts/requirements.txt CMD ["/bin/sh"] -ENTRYPOINT ["python", "-m", "/scripts/git_repo_scanner"] +WORKDIR /scripts +ENTRYPOINT ["python", "-m", "git_repo_scanner"] diff --git a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml index 7205b0f9a1..ae026e91c1 100644 --- a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml +++ b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml @@ -20,7 +20,7 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.Version }}" command: - "python" - - "/scripts/git_repo_scanner.py" + - "-m git_repo_scanner" - "--file-output" - "/home/securecodebox" resources: From 0d8542fec538ac98bc031e2a8aec9046d2170c12 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 8 Apr 2021 17:26:32 +0200 Subject: [PATCH 18/24] little fixes --- scanners/git-repo-scanner/scanner/.dockerignore | 1 - .../git-repo-scanner/templates/git-repo-scanner-scan-type.yaml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore index d42a947b35..b684402faa 100644 --- a/scanners/git-repo-scanner/scanner/.dockerignore +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -1,3 +1,2 @@ -__pytest_cache .pytest_cache /tests diff --git a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml index ae026e91c1..a525f043bd 100644 --- a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml +++ b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml @@ -20,7 +20,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.Version }}" command: - "python" - - "-m git_repo_scanner" + - "-m" + - "git_repo_scanner" - "--file-output" - "/home/securecodebox" resources: From 4d372616a9a61796cb2e4a2a41f407d5d805aa9a Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 8 Apr 2021 17:34:01 +0200 Subject: [PATCH 19/24] little fixes --- .../scanner/git_repo_scanner/github_scanner.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py index b49b6ae377..24e3e8e218 100644 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py @@ -63,10 +63,9 @@ def _process_repos_page(self, self.LOGGER.info( f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') - if start_time or end_time: - isInTimeFrame = self._check_repo_is_in_time_frame(repo.pushed_at, start_time, end_time) - if not isInTimeFrame: - break + if (start_time or end_time) \ + and not self._check_repo_is_in_time_frame(repo.pushed_at, start_time, end_time): + break findings.append(self._create_finding_from_repo(repo)) self._respect_github_ratelimit() From 5d6579672f0c8a58b58c125b53dba3349bb0b5d8 Mon Sep 17 00:00:00 2001 From: Paul Date: Thu, 8 Apr 2021 17:51:54 +0200 Subject: [PATCH 20/24] fixes tests --- .../git-repo-scanner/scanner/tests/git_repo_scanner_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py index d8f7db2814..80bdc1dfc0 100644 --- a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py +++ b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py @@ -61,7 +61,7 @@ def test_process_gitlab_projects_with_ignore_project(self): @patch('github.PaginatedList') def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): # given - scanner = GitHubScanner('url', 'token', 'org', []) + scanner = GitHubScanner('url', 'token', 'org', [], False) repos = assemble_repos() create_mocks(github_mock, org_mock, pag_mock, repos) scanner._gh = github_mock @@ -78,7 +78,7 @@ def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, p @patch('github.PaginatedList') def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): # given - scanner = GitHubScanner('url', 'token', 'org', [1]) + scanner = GitHubScanner('url', 'token', 'org', [1], False) repos = assemble_repos() create_mocks(github_mock, org_mock, pag_mock, repos) scanner._gh = github_mock From 3d6ad3c66b6e29be5559d4802210da5073e9acc5 Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Fri, 9 Apr 2021 07:31:39 +0200 Subject: [PATCH 21/24] Ignore Python bytecode and pytest cache directories in Docker build --- scanners/git-repo-scanner/scanner/.dockerignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore index b684402faa..3ea279f7fb 100644 --- a/scanners/git-repo-scanner/scanner/.dockerignore +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -1,2 +1,3 @@ -.pytest_cache +**/.pytest_cache +**/__pycache__ /tests From 37f3dbd7c564ea40a11888c09b2045775c3efc7d Mon Sep 17 00:00:00 2001 From: Tim Walter Date: Fri, 9 Apr 2021 07:56:44 +0200 Subject: [PATCH 22/24] Remove unnecessary test dependencies --- .../git-repo-scanner/scanner/requirements.txt | 2 -- .../scanner/tests/git_repo_scanner_test.py | 21 +++++++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scanners/git-repo-scanner/scanner/requirements.txt b/scanners/git-repo-scanner/scanner/requirements.txt index 110b788b82..510569354b 100644 --- a/scanners/git-repo-scanner/scanner/requirements.txt +++ b/scanners/git-repo-scanner/scanner/requirements.txt @@ -1,6 +1,4 @@ PyGithub == 1.54.1 python-gitlab == 2.6.0 -munch == 2.5.0 -mock == 4.0.2 pytimeparse == 1.1.8 pytz == 2021.1 diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py index 80bdc1dfc0..ca91e00c72 100644 --- a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py +++ b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py @@ -1,16 +1,14 @@ import argparse -import unittest -import gitlab import datetime +import unittest from datetime import timezone -from gitlab.v4.objects import Project, ProjectManager +from unittest.mock import MagicMock, Mock +from unittest.mock import patch -from mock import MagicMock -from mock import patch -from munch import Munch +import gitlab +from gitlab.v4.objects import Project, ProjectManager from git_repo_scanner.__main__ import get_parser_args - from git_repo_scanner.github_scanner import GitHubScanner from git_repo_scanner.gitlab_scanner import GitLabScanner @@ -174,7 +172,12 @@ def assemble_repos(): def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, date_pushed: datetime, visibility: bool, o_id, o_kind, o_name): - repo = Munch() + + repo = Mock() + owner = Mock() + owner.type = o_kind + owner.id = o_id + owner.name = o_name repo.id = p_id repo.name = name repo.html_url = url @@ -183,7 +186,7 @@ def assemble_repository(p_id, name, url, path, date_created: datetime, date_upda repo.pushed_at = date_pushed repo.updated_at = date_updated repo.private = visibility - repo.owner = Munch(type=o_kind, id=o_id, name=o_name) + repo.owner = owner return repo From 4fca19733786b7b804988186d5c2c96e514f2167 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Mon, 12 Apr 2021 10:51:41 +0200 Subject: [PATCH 23/24] Added integration test for git-repo-scanner. --- .../scanner/git-repo-scanner.test.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/integration/scanner/git-repo-scanner.test.js diff --git a/tests/integration/scanner/git-repo-scanner.test.js b/tests/integration/scanner/git-repo-scanner.test.js new file mode 100644 index 0000000000..7ffc6100e5 --- /dev/null +++ b/tests/integration/scanner/git-repo-scanner.test.js @@ -0,0 +1,17 @@ +const {scan} = require('../helpers'); + +test( + 'gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation', + async () => { + const {categories, severities, count} = await scan( + 'git-repo-scanner-dummy-scan', + 'git-repo-scanner', + ['--git-type', 'github', '--organization', 'secureCodeBox'], + 180 + ); + // There must be >= 28 Repositories found in the GitHub secureCodeBox organisation. + expect(count).toBeGreaterThanOrEqual(28); + }, + 3 * 60 * 1000 +); + From 4457a3c1df9e5d2801e16e3f37eaaa9df01e30e2 Mon Sep 17 00:00:00 2001 From: Robert Seedorff Date: Mon, 12 Apr 2021 10:57:33 +0200 Subject: [PATCH 24/24] Updated integration test for git-repo-scanner. --- tests/integration/scanner/git-repo-scanner.test.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/scanner/git-repo-scanner.test.js b/tests/integration/scanner/git-repo-scanner.test.js index 7ffc6100e5..2bd74737db 100644 --- a/tests/integration/scanner/git-repo-scanner.test.js +++ b/tests/integration/scanner/git-repo-scanner.test.js @@ -3,11 +3,13 @@ const {scan} = require('../helpers'); test( 'gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation', async () => { - const {categories, severities, count} = await scan( + // This integration tests runs about 30min because of the GitHub Public API call rate limit. + // If you want to speed up you need to add an valid access token like: ['--git-type', 'github', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'], + const {count} = await scan( 'git-repo-scanner-dummy-scan', 'git-repo-scanner', ['--git-type', 'github', '--organization', 'secureCodeBox'], - 180 + 90 ); // There must be >= 28 Repositories found in the GitHub secureCodeBox organisation. expect(count).toBeGreaterThanOrEqual(28);