From 1f14cffed38ec27c6eefad01180771603725c846 Mon Sep 17 00:00:00 2001 From: travisgosselin Date: Wed, 7 Dec 2022 15:22:07 -0500 Subject: [PATCH 01/10] Move CodeQL latest version query behind CLI arg --- container/setup.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/container/setup.py b/container/setup.py index c3a0843..b6eb426 100755 --- a/container/setup.py +++ b/container/setup.py @@ -43,10 +43,12 @@ def get_latest_codeql(args): codeql = CodeQL(CODEQL_HOME) current_installed_version = codeql.get_current_local_version() logger.info(f'Current codeql version: {current_installed_version}') - latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title and args.check_latest_cli: - # we got a newer version online, download and install it - codeql.download_and_install_latest_codeql(latest_online_version) + # ensure we only query for the latest codeql cli version if we might actually update it + if args.check_latest_cli: + latest_online_version = codeql.get_latest_codeql_github_version() + if current_installed_version != latest_online_version.title: + # we got a newer version online, download and install it + codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) if args.check_latest_queries: codeql.download_and_install_latest_codeql_queries() From d7766e5c2e2fb94dff21aa4fc307ef91ab7e90e2 Mon Sep 17 00:00:00 2001 From: travisgosselin Date: Wed, 7 Dec 2022 15:28:05 -0500 Subject: [PATCH 02/10] Add GitHub authentication for PyGithub with env var GITHUB_TOKEN --- container/libs/github.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/container/libs/github.py b/container/libs/github.py index 32547f9..d576216 100644 --- a/container/libs/github.py +++ b/container/libs/github.py @@ -1,8 +1,14 @@ -from datetime import datetime, MINYEAR +import os +from datetime import datetime, MINYEAR from github import Github, GitRelease, Repository, GithubException def get_latest_github_repo_version(repo): - client = Github() + # check for a github token that may be used alongside the codeql cli to upload github results + # this will limit rate limting 403 errors on checking codeql versions, as the request will be authenticated if possible. + # by default codeql uses env var "GITHUB_TOKEN" to authenticate + # https://codeql.github.com/docs/codeql-cli/manual/github-upload-results/ + access_token = os.getenv('GITHUB_TOKEN') + client = Github(access_token) if access_token != None else Github() repo = client.get_repo(repo) releases = repo.get_releases() latest_release = get_latest_github_release(releases) From b18642c36ea2d3619752b8c984265ac81b1b759f Mon Sep 17 00:00:00 2001 From: nick Date: Wed, 4 Jan 2023 00:00:23 +0000 Subject: [PATCH 03/10] Fix setup.py to only make the call to the github API to get the latest online version if args.check_latest_cli is set. --- container/setup.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/container/setup.py b/container/setup.py index c3a0843..f5eddc5 100755 --- a/container/setup.py +++ b/container/setup.py @@ -39,14 +39,15 @@ def setup(): get_latest_codeql(args) logger.info("End setup...") def get_latest_codeql(args): - # what version do we have? codeql = CodeQL(CODEQL_HOME) - current_installed_version = codeql.get_current_local_version() - logger.info(f'Current codeql version: {current_installed_version}') - latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title and args.check_latest_cli: - # we got a newer version online, download and install it - codeql.download_and_install_latest_codeql(latest_online_version) + # what version do we have? + if args.check_latest_cli: + current_installed_version = codeql.get_current_local_version() + logger.info(f'Current codeql version: {current_installed_version}') + latest_online_version = codeql.get_latest_codeql_github_version() + if current_installed_version != latest_online_version.title: + # we got a newer version online, download and install it + codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) if args.check_latest_queries: codeql.download_and_install_latest_codeql_queries() From f632b807e94bc39d61e8d4745b22c0afd998ca68 Mon Sep 17 00:00:00 2001 From: Chris Sowley Date: Mon, 13 Mar 2023 13:37:10 -0400 Subject: [PATCH 04/10] Adding TypeScript Support and Removing `github/codeql-go` References (#50) * install nodejs * remove references to moved codeql-go-repo * remove go-repo directory --- Dockerfile | 57 ++++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 32bb783..6313e14 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,32 +12,33 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - software-properties-common \ - vim \ - curl \ - wget \ - git \ - build-essential \ - unzip \ - apt-transport-https \ - python3.8 \ - python3-venv \ - python3-pip \ - python3-setuptools \ - python3-dev \ - gnupg \ - g++ \ - make \ - gcc \ - apt-utils \ - rsync \ - file \ - dos2unix \ - gettext && \ - apt-get clean && \ - rm -f /usr/bin/python /usr/bin/pip && \ - ln -s /usr/bin/python3.8 /usr/bin/python && \ - ln -s /usr/bin/pip3 /usr/bin/pip + software-properties-common \ + nodejs \ + vim \ + curl \ + wget \ + git \ + build-essential \ + unzip \ + apt-transport-https \ + python3.8 \ + python3-venv \ + python3-pip \ + python3-setuptools \ + python3-dev \ + gnupg \ + g++ \ + make \ + gcc \ + apt-utils \ + rsync \ + file \ + dos2unix \ + gettext && \ + apt-get clean && \ + rm -f /usr/bin/python /usr/bin/pip && \ + ln -s /usr/bin/python3.8 /usr/bin/python && \ + ln -s /usr/bin/pip3 /usr/bin/pip # Install .NET Core and Java for tools/builds RUN cd /tmp && \ @@ -63,14 +64,11 @@ RUN pip3 install --upgrade pip \ RUN python3 /usr/local/startup_scripts/get-latest-codeql-version.py > /tmp/codeql_version RUN mkdir -p \ ${CODEQL_HOME}/codeql-repo \ - ${CODEQL_HOME}/codeql-go-repo \ /opt/codeql # get the latest codeql queries and record the HEAD RUN git clone --depth 1 https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \ git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit -RUN git clone --depth 1 https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \ - git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \ wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \ @@ -81,7 +79,6 @@ ENV PATH="${CODEQL_HOME}/codeql:${PATH}" # Pre-compile our queries to save time later RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql-suites/*.qls --additional-packs=. -RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-go-repo/ql/src/codeql-suites/*.qls --additional-packs=. ENV PYTHONIOENCODING=utf-8 From 073695d07151238cd9e23f74c3a7f47b468b0388 Mon Sep 17 00:00:00 2001 From: Suraj Jacob <28795567+jacobmsft@users.noreply.github.com> Date: Tue, 14 Mar 2023 14:36:31 -0700 Subject: [PATCH 05/10] Upgrades (#51) increment ubuntu container version move from pygithub to ghapi skip the chown step --- Dockerfile | 66 ++++++++++++-------------- container/get-latest-codeql-version.py | 8 +--- container/libs/codeql.py | 8 ++-- container/libs/github.py | 18 +++---- container/requirements.txt | 3 +- container/setup.py | 2 +- container/startup.py | 1 - 7 files changed, 50 insertions(+), 56 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6313e14..3022669 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS codeql_base +FROM ubuntu:22.04 AS codeql_base LABEL maintainer="Github codeql team" # tzdata install needs to be non-interactive @@ -12,33 +12,31 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ apt-get update && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ - software-properties-common \ - nodejs \ - vim \ - curl \ - wget \ - git \ - build-essential \ - unzip \ - apt-transport-https \ - python3.8 \ - python3-venv \ - python3-pip \ - python3-setuptools \ - python3-dev \ - gnupg \ - g++ \ - make \ - gcc \ - apt-utils \ - rsync \ - file \ - dos2unix \ - gettext && \ - apt-get clean && \ - rm -f /usr/bin/python /usr/bin/pip && \ - ln -s /usr/bin/python3.8 /usr/bin/python && \ - ln -s /usr/bin/pip3 /usr/bin/pip + software-properties-common \ + nodejs \ + vim \ + curl \ + wget \ + git \ + build-essential \ + unzip \ + apt-transport-https \ + python3.10 \ + python3-venv \ + python3-pip \ + python3-setuptools \ + python3-dev \ + python-is-python3 \ + gnupg \ + g++ \ + make \ + gcc \ + apt-utils \ + rsync \ + file \ + dos2unix \ + gettext && \ + apt-get clean # Install .NET Core and Java for tools/builds RUN cd /tmp && \ @@ -48,15 +46,13 @@ RUN cd /tmp && \ apt-get install -y default-jdk apt-transport-https && \ apt-get update && \ rm packages-microsoft-prod.deb -RUN apt-get install -y dotnet-sdk-3.1 +RUN apt-get install -y dotnet-sdk-6.0 # Clone our setup and run scripts -#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts RUN mkdir -p /usr/local/startup_scripts -RUN ls -al /usr/local/startup_scripts COPY container /usr/local/startup_scripts/ -RUN pip3 install --upgrade pip \ - && pip3 install -r /usr/local/startup_scripts/requirements.txt + +RUN pip3 install -r /usr/local/startup_scripts/requirements.txt # Install latest codeQL @@ -83,8 +79,8 @@ RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql- ENV PYTHONIOENCODING=utf-8 # Change ownership of all files and directories within CODEQL_HOME to the codeql user -RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} +#RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} USER ${USERNAME} -ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] +ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] \ No newline at end of file diff --git a/container/get-latest-codeql-version.py b/container/get-latest-codeql-version.py index d2972bd..8229c05 100644 --- a/container/get-latest-codeql-version.py +++ b/container/get-latest-codeql-version.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # get the parent directory of the script, to link libs - -import os -import sys - from libs.github import get_latest_github_repo_version def main(): - latest_release = get_latest_github_repo_version("github/codeql-cli-binaries") - print(latest_release.title) + latest_release = get_latest_github_repo_version("github", "codeql-cli-binaries") + print(latest_release.tag_name) main() diff --git a/container/libs/codeql.py b/container/libs/codeql.py index 7b6fba0..006c3bf 100644 --- a/container/libs/codeql.py +++ b/container/libs/codeql.py @@ -34,15 +34,15 @@ def download_and_install_latest_codeql(self, github_version): download_url = None download_path = None if os_name == 'posix': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-linux64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-linux64.zip' download_path = f'{self.TEMP_DIR}/codeql_linux.zip' elif os_name == 'nt': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-win64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-win64.zip' download_path = f'{self.TEMP_DIR}/codeql_windows.zip' else: exit(self.ERROR_UNKNOWN_OS) - logger.info(f'Downloading codeql-cli version {github_version.title}...') + logger.info(f'Downloading codeql-cli version {github_version.tag_name}...') check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8") self.install_codeql_cli(download_path) #rm /tmp/codeql_linux.zip @@ -77,7 +77,7 @@ def get_current_local_version(self): return version def get_latest_codeql_github_version(self): - return get_latest_github_repo_version("github/codeql-cli-binaries") + return get_latest_github_repo_version("github", "codeql-cli-binaries") def install_codeql_cli(self, download_path): logger.info("Installing codeql-cli...") diff --git a/container/libs/github.py b/container/libs/github.py index d576216..cd3cfd6 100644 --- a/container/libs/github.py +++ b/container/libs/github.py @@ -1,24 +1,26 @@ import os from datetime import datetime, MINYEAR -from github import Github, GitRelease, Repository, GithubException +from ghapi.all import GhApi +from datetime import datetime, timezone +from dateutil import parser -def get_latest_github_repo_version(repo): +def get_latest_github_repo_version(owner, repository): # check for a github token that may be used alongside the codeql cli to upload github results # this will limit rate limting 403 errors on checking codeql versions, as the request will be authenticated if possible. # by default codeql uses env var "GITHUB_TOKEN" to authenticate # https://codeql.github.com/docs/codeql-cli/manual/github-upload-results/ access_token = os.getenv('GITHUB_TOKEN') - client = Github(access_token) if access_token != None else Github() - repo = client.get_repo(repo) - releases = repo.get_releases() + api = GhApi(owner=owner, repo=repository, token=access_token) if access_token != None else GhApi(owner=owner, repo=repository) + releases = api.repos.list_releases() latest_release = get_latest_github_release(releases) return latest_release def get_latest_github_release(releases): latest_release = None - latest_date = datetime(MINYEAR, 1, 1) + latest_date = datetime(MINYEAR, 1, 1).replace(tzinfo=timezone.utc) for release in releases: - if release.created_at > latest_date: - latest_date = release.created_at + release_date = parser.parse(release.created_at) + if release_date > latest_date: + latest_date = release_date latest_release = release return latest_release diff --git a/container/requirements.txt b/container/requirements.txt index f15ff22..ce3acfc 100644 --- a/container/requirements.txt +++ b/container/requirements.txt @@ -1 +1,2 @@ -PyGithub==1.43.7 \ No newline at end of file +ghapi==1.0.3 +python-dateutil==2.8.2 \ No newline at end of file diff --git a/container/setup.py b/container/setup.py index e3fc8d1..56a148f 100644 --- a/container/setup.py +++ b/container/setup.py @@ -46,7 +46,7 @@ def get_latest_codeql(args): # ensure we only query for the latest codeql cli version if we might actually update it if args.check_latest_cli: latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title: + if current_installed_version != latest_online_version.tag_name: # we got a newer version online, download and install it codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) diff --git a/container/startup.py b/container/startup.py index 94319d7..dc05277 100755 --- a/container/startup.py +++ b/container/startup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys from time import sleep from libs.utils import get_env_variable, check_output_wrapper, get_logger from libs.codeql import * From c76a745740449c84f240c158fb26bddcbb4d46b2 Mon Sep 17 00:00:00 2001 From: David Alcantar <50344095+daalcant@users.noreply.github.com> Date: Sun, 23 Apr 2023 16:05:05 -0700 Subject: [PATCH 06/10] Create pipelines --- Pipelines/codeql-container-pr.yml | 18 ++++++++++++++++++ Pipelines/codeql-container-release.yml | 18 ++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 Pipelines/codeql-container-pr.yml create mode 100644 Pipelines/codeql-container-release.yml diff --git a/Pipelines/codeql-container-pr.yml b/Pipelines/codeql-container-pr.yml new file mode 100644 index 0000000..15c3e88 --- /dev/null +++ b/Pipelines/codeql-container-pr.yml @@ -0,0 +1,18 @@ +# Azure Pipelines +# https://aka.ms/yaml + +name: CodeQL-Container_PR_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +trigger: none +pr: +- main + +resources: + repositories: + - repository: build + type: git + name: SecurityEngineering/codeql-container-build + ref: da/pipeline-update + #ref: refs/tags/v1.0.0 + +jobs: +- template: codeql-container-build-pr.yml@build diff --git a/Pipelines/codeql-container-release.yml b/Pipelines/codeql-container-release.yml new file mode 100644 index 0000000..5dbcb81 --- /dev/null +++ b/Pipelines/codeql-container-release.yml @@ -0,0 +1,18 @@ +# Azure Pipelines +# https://aka.ms/yaml + +name: CodeQL-Container_Release_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +trigger: +- main +pr: none + +resources: + repositories: + - repository: build + type: git + name: SecurityEngineering/codeql-container-build + ref: da/pipeline-update + #ref: refs/tags/v1.0.0 + +jobs: +- template: codeql-container-build-release.yml@build From b49ce936bf0cf4a97b41426c9ca71cc786fea760 Mon Sep 17 00:00:00 2001 From: David Alcantar <50344095+daalcant@users.noreply.github.com> Date: Thu, 27 Apr 2023 15:37:42 -0700 Subject: [PATCH 07/10] Update codeql-container-pr.yml --- Pipelines/codeql-container-pr.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pipelines/codeql-container-pr.yml b/Pipelines/codeql-container-pr.yml index 15c3e88..bc5109d 100644 --- a/Pipelines/codeql-container-pr.yml +++ b/Pipelines/codeql-container-pr.yml @@ -14,5 +14,8 @@ resources: ref: da/pipeline-update #ref: refs/tags/v1.0.0 +variables: + buildTimeStamp: # will be set by script + jobs: - template: codeql-container-build-pr.yml@build From bdcd584f8320ec27f95e559bbbd7f76c4d454e93 Mon Sep 17 00:00:00 2001 From: David Alcantar <50344095+daalcant@users.noreply.github.com> Date: Thu, 27 Apr 2023 15:54:44 -0700 Subject: [PATCH 08/10] Update codeql-container-release.yml --- Pipelines/codeql-container-release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Pipelines/codeql-container-release.yml b/Pipelines/codeql-container-release.yml index 5dbcb81..5dc7c06 100644 --- a/Pipelines/codeql-container-release.yml +++ b/Pipelines/codeql-container-release.yml @@ -14,5 +14,8 @@ resources: ref: da/pipeline-update #ref: refs/tags/v1.0.0 +variables: + buildTimeStamp: # will be set by script + jobs: - template: codeql-container-build-release.yml@build From 2483f40e3494f3c4bd0a8acc50896a5c72ebf7cc Mon Sep 17 00:00:00 2001 From: David Alcantar <50344095+daalcant@users.noreply.github.com> Date: Thu, 27 Apr 2023 21:07:38 -0700 Subject: [PATCH 09/10] Pin templates --- Pipelines/codeql-container-pr.yml | 3 +-- Pipelines/codeql-container-release.yml | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Pipelines/codeql-container-pr.yml b/Pipelines/codeql-container-pr.yml index bc5109d..fb1338c 100644 --- a/Pipelines/codeql-container-pr.yml +++ b/Pipelines/codeql-container-pr.yml @@ -11,8 +11,7 @@ resources: - repository: build type: git name: SecurityEngineering/codeql-container-build - ref: da/pipeline-update - #ref: refs/tags/v1.0.0 + ref: refs/tags/v1.0.0 variables: buildTimeStamp: # will be set by script diff --git a/Pipelines/codeql-container-release.yml b/Pipelines/codeql-container-release.yml index 5dc7c06..9fb1504 100644 --- a/Pipelines/codeql-container-release.yml +++ b/Pipelines/codeql-container-release.yml @@ -11,8 +11,7 @@ resources: - repository: build type: git name: SecurityEngineering/codeql-container-build - ref: da/pipeline-update - #ref: refs/tags/v1.0.0 + ref: refs/tags/v1.0.0 variables: buildTimeStamp: # will be set by script From 3c0d042d3b2d72fdf63ba0bbabda3cac458cac8f Mon Sep 17 00:00:00 2001 From: David Alcantar <50344095+daalcant@users.noreply.github.com> Date: Fri, 28 Apr 2023 16:08:14 -0700 Subject: [PATCH 10/10] Update codeql-container-pr.yml --- Pipelines/codeql-container-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Pipelines/codeql-container-pr.yml b/Pipelines/codeql-container-pr.yml index fb1338c..05347d9 100644 --- a/Pipelines/codeql-container-pr.yml +++ b/Pipelines/codeql-container-pr.yml @@ -3,8 +3,7 @@ name: CodeQL-Container_PR_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) trigger: none -pr: -- main +pr: none resources: repositories: