diff --git a/Dockerfile b/Dockerfile index 32bb783..3022669 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:20.04 AS codeql_base +FROM ubuntu:22.04 AS codeql_base LABEL maintainer="Github codeql team" # tzdata install needs to be non-interactive @@ -13,6 +13,7 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ software-properties-common \ + nodejs \ vim \ curl \ wget \ @@ -20,11 +21,12 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ build-essential \ unzip \ apt-transport-https \ - python3.8 \ + python3.10 \ python3-venv \ python3-pip \ python3-setuptools \ python3-dev \ + python-is-python3 \ gnupg \ g++ \ make \ @@ -34,10 +36,7 @@ RUN adduser --home ${CODEQL_HOME} ${USERNAME} && \ file \ dos2unix \ gettext && \ - apt-get clean && \ - rm -f /usr/bin/python /usr/bin/pip && \ - ln -s /usr/bin/python3.8 /usr/bin/python && \ - ln -s /usr/bin/pip3 /usr/bin/pip + apt-get clean # Install .NET Core and Java for tools/builds RUN cd /tmp && \ @@ -47,15 +46,13 @@ RUN cd /tmp && \ apt-get install -y default-jdk apt-transport-https && \ apt-get update && \ rm packages-microsoft-prod.deb -RUN apt-get install -y dotnet-sdk-3.1 +RUN apt-get install -y dotnet-sdk-6.0 # Clone our setup and run scripts -#RUN git clone https://github.com/microsoft/codeql-container /usr/local/startup_scripts RUN mkdir -p /usr/local/startup_scripts -RUN ls -al /usr/local/startup_scripts COPY container /usr/local/startup_scripts/ -RUN pip3 install --upgrade pip \ - && pip3 install -r /usr/local/startup_scripts/requirements.txt + +RUN pip3 install -r /usr/local/startup_scripts/requirements.txt # Install latest codeQL @@ -63,14 +60,11 @@ RUN pip3 install --upgrade pip \ RUN python3 /usr/local/startup_scripts/get-latest-codeql-version.py > /tmp/codeql_version RUN mkdir -p \ ${CODEQL_HOME}/codeql-repo \ - ${CODEQL_HOME}/codeql-go-repo \ /opt/codeql # get the latest codeql queries and record the HEAD RUN git clone --depth 1 https://github.com/github/codeql ${CODEQL_HOME}/codeql-repo && \ git --git-dir ${CODEQL_HOME}/codeql-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-repo-last-commit -RUN git clone --depth 1 https://github.com/github/codeql-go ${CODEQL_HOME}/codeql-go-repo && \ - git --git-dir ${CODEQL_HOME}/codeql-go-repo/.git log --pretty=reference -1 > /opt/codeql/codeql-go-repo-last-commit RUN CODEQL_VERSION=$(cat /tmp/codeql_version) && \ wget -q https://github.com/github/codeql-cli-binaries/releases/download/${CODEQL_VERSION}/codeql-linux64.zip -O /tmp/codeql_linux.zip && \ @@ -81,13 +75,12 @@ ENV PATH="${CODEQL_HOME}/codeql:${PATH}" # Pre-compile our queries to save time later RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-repo/*/ql/src/codeql-suites/*.qls --additional-packs=. -RUN codeql query compile --threads=0 ${CODEQL_HOME}/codeql-go-repo/ql/src/codeql-suites/*.qls --additional-packs=. ENV PYTHONIOENCODING=utf-8 # Change ownership of all files and directories within CODEQL_HOME to the codeql user -RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} +#RUN chown -R ${USERNAME}:${USERNAME} ${CODEQL_HOME} USER ${USERNAME} -ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] +ENTRYPOINT ["python3", "/usr/local/startup_scripts/startup.py"] \ No newline at end of file diff --git a/Pipelines/codeql-container-pr.yml b/Pipelines/codeql-container-pr.yml new file mode 100644 index 0000000..05347d9 --- /dev/null +++ b/Pipelines/codeql-container-pr.yml @@ -0,0 +1,19 @@ +# Azure Pipelines +# https://aka.ms/yaml + +name: CodeQL-Container_PR_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +trigger: none +pr: none + +resources: + repositories: + - repository: build + type: git + name: SecurityEngineering/codeql-container-build + ref: refs/tags/v1.0.0 + +variables: + buildTimeStamp: # will be set by script + +jobs: +- template: codeql-container-build-pr.yml@build diff --git a/Pipelines/codeql-container-release.yml b/Pipelines/codeql-container-release.yml new file mode 100644 index 0000000..9fb1504 --- /dev/null +++ b/Pipelines/codeql-container-release.yml @@ -0,0 +1,20 @@ +# Azure Pipelines +# https://aka.ms/yaml + +name: CodeQL-Container_Release_$(SourceBranchName)_$(Date:yyyyMMdd)$(Rev:.r) +trigger: +- main +pr: none + +resources: + repositories: + - repository: build + type: git + name: SecurityEngineering/codeql-container-build + ref: refs/tags/v1.0.0 + +variables: + buildTimeStamp: # will be set by script + +jobs: +- template: codeql-container-build-release.yml@build diff --git a/container/get-latest-codeql-version.py b/container/get-latest-codeql-version.py index d2972bd..8229c05 100644 --- a/container/get-latest-codeql-version.py +++ b/container/get-latest-codeql-version.py @@ -1,13 +1,9 @@ #!/usr/bin/env python3 # get the parent directory of the script, to link libs - -import os -import sys - from libs.github import get_latest_github_repo_version def main(): - latest_release = get_latest_github_repo_version("github/codeql-cli-binaries") - print(latest_release.title) + latest_release = get_latest_github_repo_version("github", "codeql-cli-binaries") + print(latest_release.tag_name) main() diff --git a/container/libs/codeql.py b/container/libs/codeql.py index 7b6fba0..006c3bf 100644 --- a/container/libs/codeql.py +++ b/container/libs/codeql.py @@ -34,15 +34,15 @@ def download_and_install_latest_codeql(self, github_version): download_url = None download_path = None if os_name == 'posix': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-linux64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-linux64.zip' download_path = f'{self.TEMP_DIR}/codeql_linux.zip' elif os_name == 'nt': - download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.title}/codeql-win64.zip' + download_url = f'{self.CODEQL_GITHUB_URL}/releases/download/{github_version.tag_name}/codeql-win64.zip' download_path = f'{self.TEMP_DIR}/codeql_windows.zip' else: exit(self.ERROR_UNKNOWN_OS) - logger.info(f'Downloading codeql-cli version {github_version.title}...') + logger.info(f'Downloading codeql-cli version {github_version.tag_name}...') check_output_wrapper(f"wget -q {download_url} -O {download_path}", shell=True).decode("utf-8") self.install_codeql_cli(download_path) #rm /tmp/codeql_linux.zip @@ -77,7 +77,7 @@ def get_current_local_version(self): return version def get_latest_codeql_github_version(self): - return get_latest_github_repo_version("github/codeql-cli-binaries") + return get_latest_github_repo_version("github", "codeql-cli-binaries") def install_codeql_cli(self, download_path): logger.info("Installing codeql-cli...") diff --git a/container/libs/github.py b/container/libs/github.py index 32547f9..cd3cfd6 100644 --- a/container/libs/github.py +++ b/container/libs/github.py @@ -1,18 +1,26 @@ -from datetime import datetime, MINYEAR -from github import Github, GitRelease, Repository, GithubException +import os +from datetime import datetime, MINYEAR +from ghapi.all import GhApi +from datetime import datetime, timezone +from dateutil import parser -def get_latest_github_repo_version(repo): - client = Github() - repo = client.get_repo(repo) - releases = repo.get_releases() +def get_latest_github_repo_version(owner, repository): + # check for a github token that may be used alongside the codeql cli to upload github results + # this will limit rate limting 403 errors on checking codeql versions, as the request will be authenticated if possible. + # by default codeql uses env var "GITHUB_TOKEN" to authenticate + # https://codeql.github.com/docs/codeql-cli/manual/github-upload-results/ + access_token = os.getenv('GITHUB_TOKEN') + api = GhApi(owner=owner, repo=repository, token=access_token) if access_token != None else GhApi(owner=owner, repo=repository) + releases = api.repos.list_releases() latest_release = get_latest_github_release(releases) return latest_release def get_latest_github_release(releases): latest_release = None - latest_date = datetime(MINYEAR, 1, 1) + latest_date = datetime(MINYEAR, 1, 1).replace(tzinfo=timezone.utc) for release in releases: - if release.created_at > latest_date: - latest_date = release.created_at + release_date = parser.parse(release.created_at) + if release_date > latest_date: + latest_date = release_date latest_release = release return latest_release diff --git a/container/requirements.txt b/container/requirements.txt index f15ff22..ce3acfc 100644 --- a/container/requirements.txt +++ b/container/requirements.txt @@ -1 +1,2 @@ -PyGithub==1.43.7 \ No newline at end of file +ghapi==1.0.3 +python-dateutil==2.8.2 \ No newline at end of file diff --git a/container/setup.py b/container/setup.py old mode 100755 new mode 100644 index c3a0843..56a148f --- a/container/setup.py +++ b/container/setup.py @@ -38,15 +38,17 @@ def setup(): # check version and download the latest version get_latest_codeql(args) logger.info("End setup...") + def get_latest_codeql(args): - # what version do we have? codeql = CodeQL(CODEQL_HOME) current_installed_version = codeql.get_current_local_version() logger.info(f'Current codeql version: {current_installed_version}') - latest_online_version = codeql.get_latest_codeql_github_version() - if current_installed_version != latest_online_version.title and args.check_latest_cli: - # we got a newer version online, download and install it - codeql.download_and_install_latest_codeql(latest_online_version) + # ensure we only query for the latest codeql cli version if we might actually update it + if args.check_latest_cli: + latest_online_version = codeql.get_latest_codeql_github_version() + if current_installed_version != latest_online_version.tag_name: + # we got a newer version online, download and install it + codeql.download_and_install_latest_codeql(latest_online_version) # get the latest queries regardless (TODO: Optimize by storing and checking the last commit hash?) if args.check_latest_queries: codeql.download_and_install_latest_codeql_queries() diff --git a/container/startup.py b/container/startup.py index 94319d7..dc05277 100755 --- a/container/startup.py +++ b/container/startup.py @@ -1,6 +1,5 @@ #!/usr/bin/env python3 import os -import sys from time import sleep from libs.utils import get_env_variable, check_output_wrapper, get_logger from libs.codeql import *