diff --git a/.editorconfig b/.editorconfig index f48effb0ca..a4e4605676 100644 --- a/.editorconfig +++ b/.editorconfig @@ -14,5 +14,8 @@ indent_size = 2 [*.go] indent_style = tab +[*.py] +indent_size = 4 + [Makefile] indent_style = tab diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 697c6b53af..0216a0128e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -60,7 +60,7 @@ jobs: working-directory: scanners/git-repo-scanner/scanner/ run: | pip install pytest - pytest ${{ matrix.unit }}_test.py + pytest # ---- Unit-Test | JavaScript ---- diff --git a/.gitignore b/.gitignore index 160ec0b5f4..0b709a200d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ coverage/ **.log **/*.monopic .s3_credentials +**/__pycache__ + ### IntelliJ IDEA ### .idea diff --git a/scanners/git-repo-scanner/README.md b/scanners/git-repo-scanner/README.md index 89702467e1..10258cc286 100644 --- a/scanners/git-repo-scanner/README.md +++ b/scanners/git-repo-scanner/README.md @@ -31,22 +31,32 @@ or ``` #### GitHub -For type github you can use the following options: -- `--organization`: The name of the github organization you want to scan. -- `--url`: The url of the api for a github enterprise server. Skip this option for repos on . -- `--access-token`: Your personal github access token. -- `--ignore-repos`: A list of github repository ids you want to ignore +For type GitHub you can use the following options: +- `--organization`: The name of the GitHub organization you want to scan. +- `--url`: The url of the api for a GitHub enterprise server. Skip this option for repos on . +- `--access-token`: Your personal GitHub access token. +- `--ignore-repos`: A list of GitHub repository ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. #### GitLab -For type gitlab you can use the following options: -- `--url`: The url of the gitlab server. -- `--access-token`: Your personal gitlab access token. -- `--group`: A specific gitlab group id you want to san, including subgroups. -- `--ignore-groups`: A list of gitlab group ids you want to ignore -- `--ignore-repos`: A list of gitlab project ids you want to ignore +For type GitLab you can use the following options: +- `--url`: The url of the GitLab server. +- `--access-token`: Your personal GitLab access token. +- `--group`: A specific GitLab group id you want to san, including subgroups. +- `--ignore-groups`: A list of GitLab group ids you want to ignore +- `--ignore-repos`: A list of GitLab project ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. diff --git a/scanners/git-repo-scanner/README.md.gotmpl b/scanners/git-repo-scanner/README.md.gotmpl index b573606f14..02ab3188cd 100644 --- a/scanners/git-repo-scanner/README.md.gotmpl +++ b/scanners/git-repo-scanner/README.md.gotmpl @@ -32,22 +32,33 @@ or ``` #### GitHub -For type github you can use the following options: -- `--organization`: The name of the github organization you want to scan. -- `--url`: The url of the api for a github enterprise server. Skip this option for repos on . -- `--access-token`: Your personal github access token. -- `--ignore-repos`: A list of github repository ids you want to ignore +For type GitHub you can use the following options: +- `--organization`: The name of the GitHub organization you want to scan. +- `--url`: The url of the api for a GitHub enterprise server. Skip this option for repos on . +- `--access-token`: Your personal GitHub access token. +- `--ignore-repos`: A list of GitHub repository ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitHub server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. For now only organizations are supported so the option is mandatory. We **strongly recommend** providing an access token for authentication. If not provided the rate limiting will kick in after about 30 repositories scanned. #### GitLab -For type gitlab you can use the following options: -- `--url`: The url of the gitlab server. -- `--access-token`: Your personal gitlab access token. -- `--group`: A specific gitlab group id you want to san, including subgroups. -- `--ignore-groups`: A list of gitlab group ids you want to ignore -- `--ignore-repos`: A list of gitlab project ids you want to ignore +For type GitLab you can use the following options: +- `--url`: The url of the GitLab server. +- `--access-token`: Your personal GitLab access token. +- `--group`: A specific GitLab group id you want to san, including subgroups. +- `--ignore-groups`: A list of GitLab group ids you want to ignore +- `--ignore-repos`: A list of GitLab project ids you want to ignore +- `--obey-rate-limit`: True to obey the rate limit of the GitLab server (default), otherwise False +- `--activity-since-duration`: Return git repo findings with repo activity (e.g. commits) more recent than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each + with optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. +- `--activity-until-duration`: Return git repo findings with repo activity (e.g. commits) older than a specific date expresed by an duration (now + duration). A duration string is a possibly signed sequence of decimal numbers, each with + optional fraction and a unit suffix, such as '1h' or '2h45m'. Valid time units are 'm', 'h', 'd', 'w'. + For gitlab the url and the access token is mandatory. If you don't provide a specific group id all projects on the gitlab server are going to be discovered. diff --git a/scanners/git-repo-scanner/scanner/.dockerignore b/scanners/git-repo-scanner/scanner/.dockerignore new file mode 100644 index 0000000000..3ea279f7fb --- /dev/null +++ b/scanners/git-repo-scanner/scanner/.dockerignore @@ -0,0 +1,3 @@ +**/.pytest_cache +**/__pycache__ +/tests diff --git a/scanners/git-repo-scanner/scanner/Dockerfile b/scanners/git-repo-scanner/scanner/Dockerfile index 48d3de19b1..f99e726a1c 100644 --- a/scanners/git-repo-scanner/scanner/Dockerfile +++ b/scanners/git-repo-scanner/scanner/Dockerfile @@ -1,5 +1,6 @@ FROM python:3.9.0-alpine -COPY git_repo_scanner.py /scripts/git_repo_scanner.py -RUN pip install PyGithub python-gitlab +COPY . /scripts/ +RUN pip install -r /scripts/requirements.txt CMD ["/bin/sh"] -ENTRYPOINT ["python","/scripts/git_repo_scanner.py"] +WORKDIR /scripts +ENTRYPOINT ["python", "-m", "git_repo_scanner"] diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner.py deleted file mode 100644 index 7f111bfdf6..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner.py +++ /dev/null @@ -1,271 +0,0 @@ -import argparse -import logging -import sys -import json -from typing import List -from pathlib import Path - -import gitlab -from gitlab.v4.objects import Project - -import github -from github.Organization import Organization -from github.Repository import Repository -from github.PaginatedList import PaginatedList - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger('git_repo_scanner') - - -def main(): - args = get_parser_args() - - findings = process(args) - - logger.info(' Write findings to file...') - write_findings_to_file(args, findings) - logger.info(' Finished!') - - -def process(args): - if args.git_type == 'gitlab': - return process_gitlab(args) - else: - return process_github(args) - - -def process_github(args): - try: - return parse_github(args) - except github.GithubException as e: - logger.info(f' Github API Exception: {e.status} -> {e.data["message"]}') - sys.exit(-1) - - -def process_gitlab(args): - try: - return parse_gitlab(args) - except gitlab.GitlabError as e: - logger.info(f' Gitlab API Exception: {e}') - sys.exit(-1) - - -def write_findings_to_file(args, findings): - Path(args.file_output).mkdir(parents=True, exist_ok=True) - with open(f'{args.file_output}/git-repo-scanner-findings.json', 'w') as out: - json.dump(findings, out) - - -def get_parser_args(args=None): - parser = argparse.ArgumentParser(description='Scan public or private git repositories of organizations or groups') - parser.add_argument('--git-type', - help='Repository type can be github or gitlab', - choices=['github', 'gitlab'], - required=True) - parser.add_argument('--file-output', - help='The path of the output file', - required=True), - parser.add_argument('--url', help='The gitlab url or a github enterprise api url.', - required=False) - parser.add_argument('--access-token', - help='An access token for authentication', - required=False) - parser.add_argument('--organization', - help='The name of the githup organization to scan', - required=False) - parser.add_argument('--group', - help='The id of the gitlab group to scan', - required=False) - parser.add_argument('--ignore-repos', - help='A list of repo ids to ignore', - action='extend', - nargs='+', - type=int, - default=[], - required=False) - parser.add_argument('--ignore-groups', - help='A list of gitlab group ids to ignore', - action='extend', - nargs='+', - type=int, - default=[], - required=False) - if args: - return parser.parse_args(args) - else: - return parser.parse_args() - - -def parse_gitlab(args): - gl: gitlab.Gitlab - if not args.url: - logger.info(' URL required for gitlab connection.') - sys.exit(-1) - logger.info(' Gitlab authentication...') - - gl = gitlab_authenticate(args) - - projects: List[Project] = get_gitlab_projects(args, gl) - - logger.info(' Process Projects...') - - findings = process_gitlab_projects(args, projects) - - return findings - - -def process_gitlab_projects(args, projects): - findings = [] - i = 1 - for project in projects: - if is_not_on_ignore_list_gitlab(project, args.ignore_groups, args.ignore_repos): - logger.info(f' {i} - {project.name}') - i += 1 - findings.append(create_finding_gitlab(project)) - return findings - - -def get_gitlab_projects(args, gl): - if args.group: - try: - projects = gl.groups.get(args.group).projects.list(all=True, include_subgroups=True) - except gitlab.exceptions.GitlabGetError: - logger.info(' Group does not exist.') - sys.exit(-1) - else: - projects = gl.projects.list(all=True, max_retries=12) - return projects - - -def gitlab_authenticate(args): - gl: gitlab.Gitlab - if args.access_token: - try: - gl = gitlab.Gitlab(args.url, args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - gl = gitlab_authenticate_oauth(args) - else: - logger.info(' Access token required for gitlab authentication.') - sys.exit(-1) - logger.info(' Success') - return gl - - -def gitlab_authenticate_oauth(args): - try: - gl = gitlab.Gitlab(args.url, oauth_token=args.access_token) - gl.auth() - except gitlab.exceptions.GitlabAuthenticationError: - logger.info(' No permission. Check your access token.') - sys.exit(-1) - return gl - - -def parse_github(args): - gh: github.Github = setup_github(args) - - logger.info(' Process Repositories...') - - if args.organization: - findings = process_github_repos(args, gh) - return findings - else: - logger.info(' No organization provided') - sys.exit(-1) - - -def process_github_repos(args, gh): - findings = [] - org: Organization = gh.get_organization(args.organization) - repos: PaginatedList[Repository] = org.get_repos(type='all') - for i in range(repos.totalCount): - process_github_repos_page(args, findings, repos.get_page(i)) - return findings - - -def process_github_repos_page(args, findings, repos): - repo: Repository - for repo in repos: - if repo.id not in args.ignore_repos: - logger.info(f' {len(findings) + 1} - {repo.name}') - findings.append(create_finding_github(repo)) - - -def setup_github(args): - if args.url: - return setup_github_with_url(args) - else: - return setup_github_without_url(args) - - -def setup_github_without_url(args): - if args.access_token: - return github.Github(args.access_token) - else: - return github.Github() - - -def setup_github_with_url(args): - if args.access_token: - return github.Github(base_url=args.url, login_or_token=args.access_token) - else: - logger.info(' Access token required for github enterprise authentication.') - sys.exit(-1) - - -def is_not_on_ignore_list_gitlab(project: Project, groups: List, repos: List): - id_project = project.id - kind = project.namespace['kind'] - id_namespace = project.namespace['id'] - if id_project in repos: - return False - if kind == 'group' and id_namespace in groups: - return False - return True - - -def create_finding_gitlab(project: Project): - return { - 'name': 'GitLab Repo', - 'description': 'A GitLab repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': project.id, - 'web_url': project.web_url, - 'full_name': project.path_with_namespace, - 'owner_type': project.namespace['kind'], - 'owner_id': project.namespace['id'], - 'owner_name': project.namespace['name'], - 'created_at': project.created_at, - 'last_activity_at': project.last_activity_at, - 'visibility': project.visibility - } - } - - -def create_finding_github(repo: Repository): - return { - 'name': 'GitHub Repo', - 'description': 'A GitHub repository', - 'category': 'Git Repository', - 'osi_layer': 'APPLICATION', - 'severity': 'INFORMATIONAL', - 'attributes': { - 'id': repo.id, - 'web_url': repo.html_url, - 'full_name': repo.full_name, - 'owner_type': repo.owner.type, - 'owner_id': repo.owner.id, - 'owner_name': repo.owner.name, - 'created_at': repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'last_activity_at': repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), - 'visibility': 'private' if repo.private else 'public' - } - } - - -if __name__ == '__main__': - main() diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py new file mode 100644 index 0000000000..4a3e78edbe --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/__main__.py @@ -0,0 +1,160 @@ +import argparse +import json +import logging +import sys +from datetime import datetime +# https://docs.python.org/3/library/datetime.html +from datetime import timedelta +from pathlib import Path + +import github +import gitlab +import pytz +# https://pypi.org/project/pytimeparse/ +from pytimeparse.timeparse import timeparse + +from git_repo_scanner.abstract_scanner import AbstractScanner +from git_repo_scanner.github_scanner import GitHubScanner +from git_repo_scanner.gitlab_scanner import GitLabScanner + +log_format = '%(asctime)s - %(levelname)-7s - %(name)s - %(message)s' +logging.basicConfig(level=logging.INFO, format=log_format) +logger = logging.getLogger('git_repo_scanner') + +now_utc = pytz.utc.localize(datetime.utcnow()) + + +def main(): + args = get_parser_args() + + if not args.git_type: + logger.info('Argument error: No git type specified') + sys.exit(1) + + findings = process(args) + + logger.info('Write findings to file...') + write_findings_to_file(args, findings) + logger.info('Finished!') + + +def process(args): + scanner: AbstractScanner + + if args.git_type == 'gitlab': + scanner = GitLabScanner( + url=args.url, + access_token=args.access_token, + group=args.group, + ignored_groups=args.ignore_groups, + ignore_repos=args.ignore_repos, + obey_rate_limit=args.obey_rate_limit + ) + elif args.git_type == 'github': + scanner = GitHubScanner( + url=args.url, + access_token=args.access_token, + organization=args.organization, + ignore_repos=args.ignore_repos, + obey_rate_limit=args.obey_rate_limit + ) + else: + logger.info('Argument error: Unknown git type') + sys.exit(1) + + try: + return scanner.process( + args.activity_since_duration, + args.activity_until_duration + ) + except argparse.ArgumentError as e: + logger.error(f'Argument error: {e}') + sys.exit(1) + except gitlab.exceptions.GitlabAuthenticationError: + logger.info('No permission. Check your access token.') + sys.exit(1) + except github.GithubException as e: + logger.error(f'Github API Exception: {e.status} -> {e.data["message"]}') + sys.exit(2) + except gitlab.GitlabError as e: + logger.error(f'Gitlab API Exception: {e}') + sys.exit(2) + except Exception as e: + logger.error(f'Unexpected error: {e}') + sys.exit(3) + + +def write_findings_to_file(args, findings): + Path(args.file_output).mkdir(parents=True, exist_ok=True) + with open(f'{args.file_output}/git-repo-scanner-findings.json', 'w') as out: + json.dump(findings, out) + + +def parse_duration_as_datetime(val: str): + try: + parsed = timeparse(val) + if parsed is None: + raise argparse.ArgumentTypeError(f'Not a valid duration: {val}.') + delta = timedelta(seconds=parsed) + return now_utc - delta + except Exception: + raise argparse.ArgumentTypeError(f'Not a valid duration: {val}.') + + +def get_parser_args(args=None): + parser = argparse.ArgumentParser(prog='git_repo_scanner', + description='Scan public or private git repositories of organizations or groups') + parser.add_argument('--git-type', + help='Repository type can be github or GitLab', + choices=['github', 'gitlab'], + required=True) + parser.add_argument('--file-output', + help='The path of the output file', + required=True), + parser.add_argument('--url', help='The GitLab url or a GitHub enterprise api url.', + required=False) + parser.add_argument('--access-token', + help='An access token for authentication', + required=False) + parser.add_argument('--organization', + help='The name of the GitHub organization to scan', + required=False) + parser.add_argument('--group', + help='The id of the GitLab group to scan', + type=int, + required=False) + parser.add_argument('--ignore-repos', + help='A list of repo ids to ignore', + action='extend', + nargs='+', + type=int, + default=[], + required=False) + parser.add_argument('--ignore-groups', + help='A list of GitLab group ids to ignore', + action='extend', + nargs='+', + type=int, + default=[], + required=False) + parser.add_argument('--obey-rate-limit', + help='True to obey the rate limit of the GitLab or GitHub server (default), otherwise False', + type=bool, + default=True, + required=False) + parser.add_argument('--activity-since-duration', + help='Return git repo findings with repo activity (e.g. commits) more recent than a specific ' + 'date expressed by a duration (now - duration)', + type=parse_duration_as_datetime, + required=False) + parser.add_argument('--activity-until-duration', + help='Return git repo findings with repo activity (e.g. commits) older than a specific date ' + 'expressed by a duration (now - duration)', + type=parse_duration_as_datetime, + required=False) + + return parser.parse_args(args) + + +if __name__ == '__main__': + main() diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py new file mode 100644 index 0000000000..1ebb66e072 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/abstract_scanner.py @@ -0,0 +1,38 @@ +import abc +from datetime import datetime +from typing import Dict, List, Optional + +FINDING = Dict[str, any] + + +class AbstractScanner(abc.ABC): + + @property + @abc.abstractmethod + def git_type(self) -> str: + raise NotImplementedError() + + @abc.abstractmethod + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + raise NotImplementedError() + + def _create_finding(self, repo_id: str, web_url: str, full_name: str, owner_type: str, owner_id: str, + owner_name: str, created_at: str, last_activity_at: str, visibility: str) -> FINDING: + return { + 'name': f'{self.git_type} Repo', + 'description': f'A {self.git_type} repository', + 'category': 'Git Repository', + 'osi_layer': 'APPLICATION', + 'severity': 'INFORMATIONAL', + 'attributes': { + 'id': repo_id, + 'web_url': web_url, + 'full_name': full_name, + 'owner_type': owner_type, + 'owner_id': owner_id, + 'owner_name': owner_name, + 'created_at': created_at, + 'last_activity_at': last_activity_at, + 'visibility': visibility + } + } diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py new file mode 100644 index 0000000000..24e3e8e218 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/github_scanner.py @@ -0,0 +1,132 @@ +import argparse +import logging +import time +from calendar import timegm +from datetime import datetime +from typing import Optional, List + +import github +from github.Organization import Organization +from github.PaginatedList import PaginatedList +from github.Repository import Repository + +from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING + + +class GitHubScanner(AbstractScanner): + LOGGER = logging.getLogger('git_repo_scanner') + + def __init__(self, url: Optional[str], access_token: Optional[str], organization: str, ignore_repos: List[int], + obey_rate_limit: bool = True) -> None: + super().__init__() + if not organization: + raise argparse.ArgumentError(None, 'Organization required for GitHab connection.') + if url and not access_token: + raise argparse.ArgumentError(None, 'Access token required for GitHab connection.') + + self._url = url + self._access_token = access_token + self._organization = organization + self._ignore_repos = ignore_repos + self._obey_rate_limit = obey_rate_limit + self._gh: Optional[github.Github] = None + + @property + def git_type(self) -> str: + return 'GitHub' + + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + self._setup() + return self._process_repos(start_time, end_time) + + def _process_repos(self, start_time: Optional[datetime], end_time: Optional[datetime]): + findings = [] + org: Organization = self._gh.get_organization(self._organization) + + repos: PaginatedList[Repository] = org.get_repos(type='all', sort='pushed', direction='asc') + + if start_time: + repos = org.get_repos(type='all', sort='pushed', direction='desc') + + for i in range(repos.totalCount): + self._process_repos_page(findings, repos.get_page(i), start_time, end_time) + return findings + + def _process_repos_page(self, + findings: List[FINDING], + repos: List[Repository], + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None): + repo: Repository + for repo in repos: + if repo.id not in self._ignore_repos: + self.LOGGER.info( + f'{len(findings) + 1} - Name: {repo.name} - LastUpdate: {repo.updated_at} - LastPush: {repo.pushed_at}') + + if (start_time or end_time) \ + and not self._check_repo_is_in_time_frame(repo.pushed_at, start_time, end_time): + break + + findings.append(self._create_finding_from_repo(repo)) + self._respect_github_ratelimit() + + def _check_repo_is_in_time_frame(self, + pushed_at: datetime, + start_time: Optional[datetime] = None, + end_time: Optional[datetime] = None): + if start_time: + if pushed_at > start_time: + return True + else: + self.LOGGER.info(f'Reached activity limit! Ignoring all repos with activity since `{start_time}`.') + return False + elif end_time: + if pushed_at < end_time: + return True + else: + self.LOGGER.info(f'Reached activity limit! Ignoring all repos with activity until `{end_time}`.') + return False + + def _respect_github_ratelimit(self): + if self._obey_rate_limit: + api_limit = self._gh.get_rate_limit().core + reset_timestamp = timegm(api_limit.reset.timetuple()) + # add 5 seconds to be sure the rate limit has been reset + seconds_until_reset = reset_timestamp - timegm(time.gmtime()) + 5 + sleep_time = seconds_until_reset / api_limit.remaining + + self.LOGGER.info('Checking Rate-Limit (' + str(self._obey_rate_limit) + ') [remainingApiCalls: ' + str( + api_limit.remaining) + ', seconds_until_reset: ' + str(seconds_until_reset) + ', sleepTime: ' + str( + sleep_time) + ']') + time.sleep(sleep_time) + + def _setup(self): + if self._url: + self._setup_with_url() + else: + self._setup_without_url() + + def _setup_without_url(self): + if self._access_token: + self._gh = github.Github(self._access_token) + else: + self._gh = github.Github() + + def _setup_with_url(self): + if self._access_token: + self._gh = github.Github(base_url=self._url, login_or_token=self._access_token) + else: + raise argparse.ArgumentError(None, 'Access token required for github enterprise authentication.') + + def _create_finding_from_repo(self, repo: Repository) -> FINDING: + return super()._create_finding( + str(repo.id), + repo.html_url, + repo.full_name, + repo.owner.type, + str(repo.owner.id), + repo.owner.name, + repo.created_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + repo.updated_at.strftime("%Y-%m-%dT%H:%M:%SZ"), + 'private' if repo.private else 'public' + ) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py new file mode 100644 index 0000000000..233a376195 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/git_repo_scanner/gitlab_scanner.py @@ -0,0 +1,112 @@ +import argparse +import logging +from datetime import datetime +from typing import List, Optional + +import gitlab +from gitlab.v4.objects import Project, ProjectManager + +from git_repo_scanner.abstract_scanner import AbstractScanner, FINDING + +logger = logging.getLogger('git_repo_scanner') + + +class GitLabScanner(AbstractScanner): + LOGGER = logging.getLogger('git_repo_scanner') + + def __init__(self, url: str, + access_token: str, + group: Optional[int], + ignored_groups: List[int], + ignore_repos: List[int], + obey_rate_limit: bool = True) -> None: + super().__init__() + if not url: + raise argparse.ArgumentError(None, 'URL required for GitLab connection.') + if not access_token: + raise argparse.ArgumentError(None, 'Access token required for GitLab authentication.') + + self._url = url + self._access_token = access_token + self._group = group + self._ignored_groups = ignored_groups + self._ignore_repos = ignore_repos + self._obey_rate_limit = obey_rate_limit + self._gl: Optional[gitlab.Gitlab] = None + + @property + def git_type(self) -> str: + return 'GitLab' + + def process(self, start_time: Optional[datetime] = None, end_time: Optional[datetime] = None) -> List[FINDING]: + self._authenticate() + + projects: List[Project] = self._get_projects(start_time, end_time) + return self._process_projects(projects) + + def _get_projects(self, start_time: Optional[datetime], end_time: Optional[datetime]): + logger.info(f'Get GitLab repositories with last activity between {start_time} and {end_time}.') + + project_manager: ProjectManager = self._gl.projects + options = dict( + all=True, + order_by='last_activity_at', + sort='desc', + obey_rate_limit=self._obey_rate_limit, + max_retries=12 + ) + if start_time is not None: + options['last_activity_after'] = start_time + if end_time is not None: + options['last_activity_before'] = end_time + + if self._group: + options['include_subgroups'] = True + project_manager = self._gl.groups.get(self._group).projects + + return project_manager.list(**options) + + def _process_projects(self, projects: List[Project]) -> List[FINDING]: + project_count = len(projects) + return [ + self._create_finding_from_project(project, i, project_count) + for i, project in enumerate(projects) + if self._is_not_ignored(project) + ] + + def _authenticate(self): + logger.info('Start GitLab authentication') + try: + self._gl = gitlab.Gitlab(self._url, private_token=self._access_token) + self._gl.auth() + except gitlab.exceptions.GitlabAuthenticationError: + self._gl = gitlab.Gitlab(self._url, oauth_token=self._access_token) + self._gl.auth() + + logger.info('GitLab authentication succeeded') + + def _is_not_ignored(self, project: Project) -> bool: + id_project = project.id + kind = project.namespace['kind'] + id_namespace = project.namespace['id'] + if id_project in self._ignore_repos: + return False + if kind == 'group' and id_namespace in self._ignored_groups: + return False + return True + + def _create_finding_from_project(self, project: Project, index: int, total: int) -> FINDING: + logger.info( + f'({index + 1}/{total}) Add finding for repo {project.name} with last activity at ' + f'{datetime.fromisoformat(project.last_activity_at)}') + return super()._create_finding( + project.id, + project.web_url, + project.path_with_namespace, + project.namespace['kind'], + project.namespace['id'], + project.namespace['name'], + project.created_at, + project.last_activity_at, + project.visibility + ) diff --git a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py deleted file mode 100644 index 5724657a06..0000000000 --- a/scanners/git-repo-scanner/scanner/git_repo_scanner_test.py +++ /dev/null @@ -1,178 +0,0 @@ -import datetime -import unittest -import git_repo_scanner -from munch import Munch -from mock import patch -from mock import MagicMock - - -class GitRepoScannerTests(unittest.TestCase): - - def test_process_gitlab_projects_with_no_ignore_list(self): - # given - projects = assemble_projects() - args = get_args() - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) - # then - self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') - self.assertEqual(findings[0]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[1]['name'], 'GitLab Repo', msg='Test finding output') - self.assertEqual(findings[2]['name'], 'GitLab Repo', msg='Test finding output') - - def test_process_gitlab_projects_with_ignore_group(self): - # given - projects = assemble_projects() - args = get_args(ignore_groups=33) - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) - # then - self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg='Test finding output') - - def test_process_gitlab_projects_with_ignore_project(self): - # given - projects = assemble_projects() - args = get_args(ignore_projects=1) - # when - findings = git_repo_scanner.process_gitlab_projects(args, projects) - # then - self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') - self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg='Test finding output') - self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg='Test finding output') - - @patch('github.Github') - @patch('github.Organization') - @patch('github.PaginatedList') - def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): - # given - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args() - # when - findings = git_repo_scanner.process_github_repos(args, github_mock) - # then - org_mock.get_repos.assert_called_with(type='all') - self.assertEqual(6, len(findings), msg='There should be exactly 6 findings') - for finding in findings: - self.assertEqual(finding['name'], 'GitHub Repo', msg='Test finding output') - - @patch('github.Github') - @patch('github.Organization') - @patch('github.PaginatedList') - def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): - # given - repos = assemble_repos() - create_mocks(github_mock, org_mock, pag_mock, repos) - args = get_args(ignore_projects=1, org='org') - # when - findings = git_repo_scanner.process_github_repos(args, github_mock) - # then - github_mock.get_organization.assert_called_with('org') - self.assertEqual(4, len(findings), msg='There should be exactly 4 findings') - - def test_setup_github_with_url_and_no_token_should_exit(self): - # given - args = get_args(url='url') - # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.setup_github(args) - # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') - - def test_parse_github_with_no_org_should_exit(self): - # given - args = get_args() - # when - with self.assertRaises(SystemExit) as cm: - git_repo_scanner.parse_github(args) - # then - self.assertEqual(cm.exception.code, -1, msg='Process should exit') - - -def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): - args = ['--git-type', 'gitlab', - '--file-output', 'out', - '--ignore-repos', str(ignore_projects), - '--ignore-groups', str(ignore_groups)] - if url: - args.append('--url') - args.append(url) - if access_token: - args.append('--access-token') - args.append(access_token) - if org: - args.append('--organization') - args.append(org) - - return git_repo_scanner.get_parser_args(args) - - -def create_mocks(github_mock, org_mock, pag_mock, repos): - pag_mock.totalCount = 2 - pag_mock.get_page = MagicMock(return_value=repos) - org_mock.get_repos = MagicMock(return_value=pag_mock) - github_mock.get_organization = MagicMock(return_value=org_mock) - - -def assemble_projects(): - project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=11, o_kind='group', - o_name='name11') - project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=22, o_kind='user', - o_name='name22') - project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created='10.10.2020', - date_updated='10.11.2020', visibility='private', o_id=33, o_kind='group', - o_name='name33') - return [project1, project2, project3] - - -def assemble_project(p_id, name, url, path, date_created, date_updated, visibility, o_id, o_kind, o_name): - project = Munch() - project.id = p_id - project.name = name - project.web_url = url - project.path_with_namespace = path - project.created_at = date_created - project.last_activity_at = date_updated - project.visibility = visibility - project.namespace = { - 'kind': o_kind, - 'id': o_id, - 'name': o_name - } - return project - - -def assemble_repos(): - date = datetime.datetime(2020, 5, 17) - project1 = assemble_repository(p_id=1, name='name1', url='url1', path='path1', date_created=date, - date_updated=date, visibility=True, o_id=11, o_kind='organization', - o_name='name11') - project2 = assemble_repository(p_id=2, name='name2', url='url2', path='path2', date_created=date, - date_updated=date, visibility=False, o_id=22, o_kind='organization', - o_name='name22') - project3 = assemble_repository(p_id=3, name='name3', url='url3', path='path3', date_created=date, - date_updated=date, visibility=False, o_id=33, o_kind='organization', - o_name='name33') - return [project1, project2, project3] - - -def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, visibility: bool, o_id, - o_kind, o_name): - repo = Munch() - repo.id = p_id - repo.name = name - repo.html_url = url - repo.full_name = path - repo.created_at = date_created - repo.updated_at = date_updated - repo.private = visibility - repo.owner = Munch(type=o_kind, id=o_id, name=o_name) - return repo - - -if __name__ == '__main__': - unittest.main() diff --git a/scanners/git-repo-scanner/scanner/requirements.txt b/scanners/git-repo-scanner/scanner/requirements.txt index 3140bb4c74..510569354b 100644 --- a/scanners/git-repo-scanner/scanner/requirements.txt +++ b/scanners/git-repo-scanner/scanner/requirements.txt @@ -1,4 +1,4 @@ -PyGithub == 1.53 -python-gitlab == 2.5.0 -munch == 2.5.0 -mock == 4.0.2 +PyGithub == 1.54.1 +python-gitlab == 2.6.0 +pytimeparse == 1.1.8 +pytz == 2021.1 diff --git a/scanners/git-repo-scanner/scanner/tests/__init__.py b/scanners/git-repo-scanner/scanner/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py new file mode 100644 index 0000000000..ca91e00c72 --- /dev/null +++ b/scanners/git-repo-scanner/scanner/tests/git_repo_scanner_test.py @@ -0,0 +1,194 @@ +import argparse +import datetime +import unittest +from datetime import timezone +from unittest.mock import MagicMock, Mock +from unittest.mock import patch + +import gitlab +from gitlab.v4.objects import Project, ProjectManager + +from git_repo_scanner.__main__ import get_parser_args +from git_repo_scanner.github_scanner import GitHubScanner +from git_repo_scanner.gitlab_scanner import GitLabScanner + + +class GitRepoScannerTests(unittest.TestCase): + + @property + def wrong_output_msg(self) -> str: + return 'Test finding output' + + def test_process_gitlab_projects_with_no_ignore_list(self): + # given + scanner = GitLabScanner('url', 'token', None, [], []) + projects = assemble_projects() + # when + findings = scanner._process_projects(projects) + # then + self.assertEqual(3, len(findings), msg='There should be exactly 3 findings') + self.assertEqual(findings[0]['name'], 'GitLab Repo', msg=self.wrong_output_msg) + self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) + self.assertEqual(findings[2]['attributes']['web_url'], 'url3', msg=self.wrong_output_msg) + + def test_process_gitlab_projects_with_ignore_group(self): + # given + scanner = GitLabScanner('url', 'token', None, [33], []) + projects = assemble_projects() + # when + findings = scanner._process_projects(projects) + # then + self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') + self.assertEqual(findings[0]['attributes']['web_url'], 'url1', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) + + def test_process_gitlab_projects_with_ignore_project(self): + # given + scanner = GitLabScanner('url', 'token', None, [], [1]) + projects = assemble_projects() + # when + findings = scanner._process_projects(projects) + # then + self.assertEqual(2, len(findings), msg='There should be exactly 2 findings') + self.assertEqual(findings[0]['attributes']['web_url'], 'url2', msg=self.wrong_output_msg) + self.assertEqual(findings[1]['attributes']['web_url'], 'url3', msg=self.wrong_output_msg) + + @patch('github.Github') + @patch('github.Organization') + @patch('github.PaginatedList') + def test_process_github_repos_with_no_ignore_list(self, github_mock, org_mock, pag_mock): + # given + scanner = GitHubScanner('url', 'token', 'org', [], False) + repos = assemble_repos() + create_mocks(github_mock, org_mock, pag_mock, repos) + scanner._gh = github_mock + # when + findings = scanner._process_repos(None, None) + # then + org_mock.get_repos.assert_called_with(type='all', sort='pushed', direction='asc') + self.assertEqual(6, len(findings), msg='There should be exactly 6 findings') + for finding in findings: + self.assertEqual(finding['name'], 'GitHub Repo', msg=self.wrong_output_msg) + + @patch('github.Github') + @patch('github.Organization') + @patch('github.PaginatedList') + def test_process_github_repos_with_ignore_repos(self, github_mock, org_mock, pag_mock): + # given + scanner = GitHubScanner('url', 'token', 'org', [1], False) + repos = assemble_repos() + create_mocks(github_mock, org_mock, pag_mock, repos) + scanner._gh = github_mock + # when + findings = scanner._process_repos(None, None) + # then + github_mock.get_organization.assert_called_with('org') + self.assertEqual(4, len(findings), msg='There should be exactly 4 findings') + + def test_setup_github_with_url_and_no_token_should_exit(self): + # when + with self.assertRaises(argparse.ArgumentError) as cm: + GitHubScanner('url', None, 'org', []) + # then + self.assertEqual(cm.exception.args[1], 'Access token required for GitHab connection.', + msg='Process should exit') + + +def get_args(ignore_groups=0, ignore_projects=0, url=None, access_token=None, org=None): + args = ['--git-type', 'someType', + '--file-output', 'out', + '--obey-rate-limit', False, + '--ignore-repos', str(ignore_projects), + '--ignore-groups', str(ignore_groups)] + if url: + args.append('--url') + args.append(url) + if access_token: + args.append('--access-token') + args.append(access_token) + if org: + args.append('--organization') + args.append(org) + + return get_parser_args(args) + + +def create_mocks(github_mock, org_mock, pag_mock, repos): + pag_mock.totalCount = 2 + pag_mock.get_page = MagicMock(return_value=repos) + org_mock.get_repos = MagicMock(return_value=pag_mock) + github_mock.get_organization = MagicMock(return_value=org_mock) + + +def assemble_projects(): + created = datetime.datetime(2020, 10, 10, tzinfo=timezone.utc).isoformat() + updated = datetime.datetime(2020, 11, 10, tzinfo=timezone.utc).isoformat() + project1 = assemble_project(p_id=1, name='name1', url='url1', path='path1', date_created=created, + date_updated=updated, visibility='private', o_id=11, o_kind='group', + o_name='name11') + project2 = assemble_project(p_id=2, name='name2', url='url2', path='path2', date_created=created, + date_updated=updated, visibility='private', o_id=22, o_kind='user', + o_name='name22') + project3 = assemble_project(p_id=3, name='name3', url='url3', path='path3', date_created=created, + date_updated=updated, visibility='private', o_id=33, o_kind='group', + o_name='name33') + return [project1, project2, project3] + + +def assemble_project(p_id, name, url, path, date_created, date_updated, visibility, o_id, o_kind, o_name): + project = Project(ProjectManager(gitlab), {}) + project.id = p_id + project.name = name + project.web_url = url + project.path_with_namespace = path + project.created_at = date_created + project.last_activity_at = date_updated + project.visibility = visibility + project.namespace = { + 'kind': o_kind, + 'id': o_id, + 'name': o_name + } + return project + + +def assemble_repos(): + date = datetime.datetime(2020, 5, 17, tzinfo=timezone.utc) + project1 = assemble_repository(p_id=1, name='name1', url='url1', path='path1', date_created=date, + date_updated=date, date_pushed=date, visibility=True, o_id=11, o_kind='organization', + o_name='name11') + project2 = assemble_repository(p_id=2, name='name2', url='url2', path='path2', date_created=date, + date_updated=date, date_pushed=date, visibility=False, o_id=22, + o_kind='organization', + o_name='name22') + project3 = assemble_repository(p_id=3, name='name3', url='url3', path='path3', date_created=date, + date_updated=date, date_pushed=date, visibility=False, o_id=33, + o_kind='organization', + o_name='name33') + return [project1, project2, project3] + + +def assemble_repository(p_id, name, url, path, date_created: datetime, date_updated: datetime, date_pushed: datetime, + visibility: bool, o_id, + o_kind, o_name): + + repo = Mock() + owner = Mock() + owner.type = o_kind + owner.id = o_id + owner.name = o_name + repo.id = p_id + repo.name = name + repo.html_url = url + repo.full_name = path + repo.created_at = date_created + repo.pushed_at = date_pushed + repo.updated_at = date_updated + repo.private = visibility + repo.owner = owner + return repo + + +if __name__ == '__main__': + unittest.main() diff --git a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml index 7205b0f9a1..a525f043bd 100644 --- a/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml +++ b/scanners/git-repo-scanner/templates/git-repo-scanner-scan-type.yaml @@ -20,7 +20,8 @@ spec: image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.Version }}" command: - "python" - - "/scripts/git_repo_scanner.py" + - "-m" + - "git_repo_scanner" - "--file-output" - "/home/securecodebox" resources: diff --git a/tests/integration/scanner/git-repo-scanner.test.js b/tests/integration/scanner/git-repo-scanner.test.js new file mode 100644 index 0000000000..2bd74737db --- /dev/null +++ b/tests/integration/scanner/git-repo-scanner.test.js @@ -0,0 +1,19 @@ +const {scan} = require('../helpers'); + +test( + 'gitleaks should find at least 1 repository in the GitHub secureCodeBox organisation', + async () => { + // This integration tests runs about 30min because of the GitHub Public API call rate limit. + // If you want to speed up you need to add an valid access token like: ['--git-type', 'github', '--organization', 'secureCodeBox', '--access-token', '23476VALID2345TOKEN'], + const {count} = await scan( + 'git-repo-scanner-dummy-scan', + 'git-repo-scanner', + ['--git-type', 'github', '--organization', 'secureCodeBox'], + 90 + ); + // There must be >= 28 Repositories found in the GitHub secureCodeBox organisation. + expect(count).toBeGreaterThanOrEqual(28); + }, + 3 * 60 * 1000 +); +