Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 31 additions & 31 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -641,45 +641,44 @@ jobs:
run: |
"$BUILD_DIR/cross-python/bin/python3" -m test test_sysconfig test_site test_embed

# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
cifuzz:
name: CIFuzz
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
runs-on: ubuntu-latest
timeout-minutes: 60
needs: build-context
if: needs.build-context.outputs.run-ci-fuzz == 'true'
if: >-
(
needs.build-context.outputs.run-ci-fuzz == 'true'
|| needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
)
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
permissions:
security-events: write
strategy:
fail-fast: false
matrix:
sanitizer: [address, undefined, memory]
steps:
- name: Build fuzzers (${{ matrix.sanitizer }})
id: build
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: cpython3
sanitizer: ${{ matrix.sanitizer }}
- name: Run fuzzers (${{ matrix.sanitizer }})
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
fuzz-seconds: 600
oss-fuzz-project-name: cpython3
output-sarif: true
sanitizer: ${{ matrix.sanitizer }}
- name: Upload crash
if: failure() && steps.build.outcome == 'success'
uses: actions/upload-artifact@v6
with:
name: ${{ matrix.sanitizer }}-artifacts
path: ./out/artifacts
- name: Upload SARIF
if: always() && steps.build.outcome == 'success'
uses: github/codeql-action/upload-sarif@v4
with:
sarif_file: cifuzz-sarif/results.sarif
checkout_path: cifuzz-sarif
sanitizer:
- address
- undefined
- memory
oss-fuzz-project-name:
- cpython3
- python3-libraries
exclude:
- oss-fuzz-project-name: >-
${{
needs.build-context.outputs.run-ci-fuzz == 'true'
&& ''
|| 'cpython3'
}}
- oss-fuzz-project-name: >-
${{
needs.build-context.outputs.run-ci-fuzz-stdlib == 'true'
&& ''
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
|| 'python3-libraries'
}}
uses: ./.github/workflows/reusable-cifuzz.yml
with:
oss-fuzz-project-name: ${{ matrix.oss-fuzz-project-name }}
sanitizer: ${{ matrix.sanitizer }}
timeout-minutes: 60
Comment thread
sethmlarson marked this conversation as resolved.
Outdated

all-required-green: # This job does nothing and is only used for the branch protection
name: All required checks pass
Expand Down Expand Up @@ -735,6 +734,7 @@ jobs:
}}
${{ !fromJSON(needs.build-context.outputs.run-windows-tests) && 'build-windows,' || '' }}
${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz) && 'cifuzz,' || '' }}
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
${{ !fromJSON(needs.build-context.outputs.run-ci-fuzz-stdlib) && 'cifuzz,' || '' }}
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
${{ !fromJSON(needs.build-context.outputs.run-macos) && 'build-macos,' || '' }}
${{
!fromJSON(needs.build-context.outputs.run-ubuntu)
Expand Down
51 changes: 51 additions & 0 deletions .github/workflows/reusable-cifuzz.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
# CIFuzz job based on https://google.github.io/oss-fuzz/getting-started/continuous-integration/
name: Reusable CIFuzz

on:
workflow_call:
inputs:
oss-fuzz-project-name:
description: OSS-Fuzz project name
required: true
type: string
sanitizer:
description: OSS-Fuzz sanitizer
required: true
type: string
timeout-minutes:
description: Timeout in minutes for the action
required: false
default: 60
type: number
Comment thread
sethmlarson marked this conversation as resolved.
Outdated

jobs:
cifuzz:
name: CIFuzz
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
runs-on: ubuntu-latest
timeout-minutes: ${{ inputs.timeout-minutes }}
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
steps:
- name: Build fuzzers (${{ inputs.sanitizer }})
id: build
uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master
with:
oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }}
sanitizer: ${{ inputs.sanitizer }}
- name: Run fuzzers (${{ inputs.sanitizer }})
uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master
with:
fuzz-seconds: 600
oss-fuzz-project-name: ${{ inputs.oss-fuzz-project-name }}
output-sarif: true
sanitizer: ${{ inputs.sanitizer }}
- name: Upload crash
if: failure() && steps.build.outcome == 'success'
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.sanitizer }}-artifacts
path: ./out/artifacts
- name: Upload SARIF
if: always() && steps.build.outcome == 'success'
uses: github/codeql-action/upload-sarif@v4
with:
sarif_file: cifuzz-sarif/results.sarif
checkout_path: cifuzz-sarif
6 changes: 5 additions & 1 deletion .github/workflows/reusable-context.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,11 @@ on: # yamllint disable-line rule:truthy
description: Whether to run the Android tests
value: ${{ jobs.compute-changes.outputs.run-android }} # bool
run-ci-fuzz:
description: Whether to run the CIFuzz job
description: Whether to run the CIFuzz job for 'cpython' fuzzer
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz }} # bool
run-ci-fuzz-stdlib:
description: Whether to run the CIFuzz job for 'python3-libraries' fuzzer
value: ${{ jobs.compute-changes.outputs.run-ci-fuzz-stdlib }} # bool
run-docs:
description: Whether to build the docs
value: ${{ jobs.compute-changes.outputs.run-docs }} # bool
Expand Down Expand Up @@ -56,6 +59,7 @@ jobs:
outputs:
run-android: ${{ steps.changes.outputs.run-android }}
run-ci-fuzz: ${{ steps.changes.outputs.run-ci-fuzz }}
run-ci-fuzz-stdlib: ${{ steps.changes.outputs.run-ci-fuzz-stdlib }}
run-docs: ${{ steps.changes.outputs.run-docs }}
run-ios: ${{ steps.changes.outputs.run-ios }}
run-macos: ${{ steps.changes.outputs.run-macos }}
Expand Down
85 changes: 72 additions & 13 deletions Tools/build/compute-changes.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import os
import subprocess
from dataclasses import dataclass
from dataclasses import dataclass, fields
from pathlib import Path

TYPE_CHECKING = False
Expand Down Expand Up @@ -52,11 +52,59 @@
MACOS_DIRS = frozenset({"Mac"})
WASI_DIRS = frozenset({Path("Tools", "wasm")})

LIBRARY_FUZZER_PATHS = frozenset({
# All C/CPP fuzzers.
Path("configure"),
Path(".github/workflows/reusable-cifuzz.yml"),
# ast
Path("Lib/ast.py"),
Path("Python/ast.c"),
# configparser
Path("Lib/configparser.py"),
# csv
Path("Lib/csv.py"),
Path("Modules/_csv.c"),
# decode
Path("Lib/encodings/"),
Path("Modules/_codecsmodule.c"),
Path("Modules/cjkcodecs/"),
Path("Modules/unicodedata*"),
# difflib
Path("Lib/difflib.py"),
# email
Path("Lib/email/"),
# html
Path("Lib/html/"),
Path("Lib/_markupbase.py"),
# http.client
Path("Lib/http/client.py"),
# json
Path("Lib/json/"),
Path("Modules/_json.c"),
# plist
Path("Lib/plistlib.py"),
# re
Path("Lib/re/"),
Path("Modules/_sre/"),
# tarfile
Path("Lib/tarfile.py"),
# tomllib
Path("Modules/tomllib/"),
# xml
Path("Lib/xml/"),
Comment thread
sethmlarson marked this conversation as resolved.
Path("Lib/_markupbase.py"),
Path("Modules/expat/"),
Path("Modules/pyexpat.c"),
# zipfile
Path("Lib/zipfile/"),
})


@dataclass(kw_only=True, slots=True)
class Outputs:
run_android: bool = False
run_ci_fuzz: bool = False
run_ci_fuzz_stdlib: bool = False
run_docs: bool = False
run_ios: bool = False
run_macos: bool = False
Expand Down Expand Up @@ -96,6 +144,11 @@ def compute_changes() -> None:
else:
print("Branch too old for CIFuzz tests; or no C files were changed")

if outputs.run_ci_fuzz_stdlib:
print("Run CIFuzz tests for libraries")
else:
print("Branch too old for CIFuzz tests; or no library files were changed")
Comment thread
sethmlarson marked this conversation as resolved.
Outdated

if outputs.run_docs:
print("Build documentation")

Expand Down Expand Up @@ -146,9 +199,18 @@ def get_file_platform(file: Path) -> str | None:
return None


def is_fuzzable_library_file(file: Path) -> bool:
return any(
(file.is_relative_to(needs_fuzz) and needs_fuzz.is_dir())
or (file == needs_fuzz and file.is_file())
for needs_fuzz in LIBRARY_FUZZER_PATHS
)


def process_changed_files(changed_files: Set[Path]) -> Outputs:
run_tests = False
run_ci_fuzz = False
run_ci_fuzz_stdlib = False
run_docs = False
run_windows_tests = False
run_windows_msi = False
Expand All @@ -162,8 +224,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
doc_file = file.suffix in SUFFIXES_DOCUMENTATION or doc_or_misc

if file.parent == GITHUB_WORKFLOWS_PATH:
if file.name == "build.yml":
run_tests = run_ci_fuzz = True
if file.name == "build.yml" or file.name == "reusable-cifuzz.yml":
Comment thread
sethmlarson marked this conversation as resolved.
Outdated
run_tests = run_ci_fuzz = run_ci_fuzz_stdlib = True
has_platform_specific_change = False
if file.name == "reusable-docs.yml":
run_docs = True
Expand Down Expand Up @@ -194,6 +256,8 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
("Modules", "_xxtestfuzz"),
}:
run_ci_fuzz = True
if not run_ci_fuzz_stdlib and is_fuzzable_library_file(file):
run_ci_fuzz_stdlib = True

# Check for changed documentation-related files
if doc_file:
Expand Down Expand Up @@ -227,6 +291,7 @@ def process_changed_files(changed_files: Set[Path]) -> Outputs:
return Outputs(
run_android=run_android,
run_ci_fuzz=run_ci_fuzz,
run_ci_fuzz_stdlib=run_ci_fuzz_stdlib,
run_docs=run_docs,
run_ios=run_ios,
run_macos=run_macos,
Expand Down Expand Up @@ -261,16 +326,10 @@ def write_github_output(outputs: Outputs) -> None:
return

with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as f:
f.write(f"run-android={bool_lower(outputs.run_android)}\n")
f.write(f"run-ci-fuzz={bool_lower(outputs.run_ci_fuzz)}\n")
f.write(f"run-docs={bool_lower(outputs.run_docs)}\n")
f.write(f"run-ios={bool_lower(outputs.run_ios)}\n")
f.write(f"run-macos={bool_lower(outputs.run_macos)}\n")
f.write(f"run-tests={bool_lower(outputs.run_tests)}\n")
f.write(f"run-ubuntu={bool_lower(outputs.run_ubuntu)}\n")
f.write(f"run-wasi={bool_lower(outputs.run_wasi)}\n")
f.write(f"run-windows-msi={bool_lower(outputs.run_windows_msi)}\n")
f.write(f"run-windows-tests={bool_lower(outputs.run_windows_tests)}\n")
for field in fields(outputs):
name = field.name.replace("_", "-")
val = bool_lower(getattr(outputs, field.name))
f.write(f"{name}={val}\n")


def bool_lower(value: bool, /) -> str:
Expand Down
Loading