Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 2 additions & 0 deletions .cspell.dict/cpython.txt
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ prec
preds
preinitialized
pybuilddir
pycfunction
pycore
PYGC
pyinner
Expand Down Expand Up @@ -311,3 +312,4 @@ xibufferview
xstat
XXPRIME
ENOTCAPABLE
unmarshalled
1 change: 1 addition & 0 deletions .cspell.dict/python-more.txt
Original file line number Diff line number Diff line change
Expand Up @@ -306,3 +306,4 @@ xmlcharrefreplace
xoptions
xopts
yieldfrom
issubnormal
2 changes: 2 additions & 0 deletions .cspell.dict/rustpython.txt
Original file line number Diff line number Diff line change
Expand Up @@ -40,3 +40,5 @@ rustix
struc
unscanned
zelf
embedders
nocapture
87 changes: 76 additions & 11 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -573,30 +573,57 @@ jobs:

- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0

- name: Verify the CPython reference environment
run: python -c "import bz2, ctypes, lzma, pathlib, platform, sqlite3, ssl; expected = pathlib.Path('.python-version').read_text().strip(); actual = platform.python_version(); assert actual == expected, (actual, expected)"

- name: Install macOS dependencies
uses: ./.github/actions/install-macos-deps
with:
openssl: true

# Keep features in sync with update-caches.yml CARGO_ARGS.
- name: build rustpython
run: cargo build --locked --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }}

- name: run snippets
run: python -m pip install -r requirements.txt && pytest -v
working-directory: ./extra_tests

- name: Detect available cores
id: cores
shell: bash
run: |
cores=$(python -c 'print(__import__("os").process_cpu_count())')
echo "cores=${cores}" >> "$GITHUB_OUTPUT"

- name: Run CPython tests
- name: Build production RustPython
id: build_default
run: cargo build --locked --release --verbose --features=threading,jit ${{ env.CARGO_ARGS }}

- name: Record Windows timestamp compatibility
if: runner.os == 'Windows'
shell: bash
run: |
python - <<'PYTIME'
import json, subprocess, sys
code = """
import datetime, json, time
rows = []
for timestamp in (-1, -2147483649, -11644473600, -11644473601, 2**40):
for name in ('localtime', 'gmtime'):
try:
value = getattr(time, name)(timestamp)
result = [list(value), value.tm_zone, value.tm_gmtoff]
except Exception as error:
result = [type(error).__name__, str(error), getattr(error, 'errno', None)]
rows.append([timestamp, name, result])
print(json.dumps(rows))
"""
for executable in (sys.executable, 'target/release/rustpython.exe'):
result = subprocess.check_output([executable, '-c', code], text=True)
print(json.dumps({'executable': executable, 'timestamps': json.loads(result)}))
PYTIME

- name: run snippets
run: python -m pip install -r requirements.txt && python -m pytest -v
working-directory: ./extra_tests

- name: Run CPython tests (production)
run: |
# shellcheck disable=SC2086
target/release/rustpython -u -m test --slow-ci -j "$CORES" $EXTRA_TEST_ARGS $SKIPS
target/release/rustpython -u -m test --slow-ci -j "$CORES" $EXTRA_TEST_ARGS $SKIPS -x test_frame test_inspect test_monitoring
timeout-minutes: ${{ matrix.timeout }}
shell: bash
env:
Expand Down Expand Up @@ -669,6 +696,28 @@ jobs:
shell: bash
run: python -I scripts/whats_left.py --rustpython target/release/rustpython

# Run all production checks before rebuilding the same binary with C API.
# Keep these features in sync with update-caches.yml CARGO_ARGS.
- name: Build RustPython with C API
id: build_capi
if: ${{ !cancelled() && steps.build_default.outcome == 'success' }}
run: cargo build --locked --release --verbose --features=threading,jit,capi ${{ env.CARGO_ARGS }}
env:
PYO3_CONFIG_FILE: ${{ github.workspace }}/crates/capi/pyo3-rustpython.config
PYO3_NO_PYTHON: "1"

# A failed production check must not suppress this cohort or become green.
- name: Run CPython tests (C API)
if: ${{ !cancelled() && steps.build_capi.outcome == 'success' }}
run: |
target/release/rustpython -u -m test --slow-ci -j "$CORES" $EXTRA_TEST_ARGS test_frame test_inspect test_monitoring
timeout-minutes: ${{ matrix.timeout }}
shell: bash
env:
RUSTPYTHON_SKIP_ENV_POLLUTERS: true
CORES: ${{ steps.cores.outputs.cores }}
EXTRA_TEST_ARGS: ${{ join(matrix.extra_test_args, ' ') }}

clippy:
# Named after the matrix entry rather than left to be named for it: a
# generated name lists every value in the entry, so adding or removing one
Expand Down Expand Up @@ -723,7 +772,12 @@ jobs:
continue-on-error: true

- name: Clippy
run: cargo clippy --locked --keep-going ${{ env.CARGO_ARGS_MOST_FEATS }} --workspace --all-targets --bins --examples --tests ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings
run: cargo clippy --locked --keep-going ${{ env.CARGO_ARGS_MOST_FEATS }} --workspace --exclude rustpython-capi --all-targets --bins --examples --tests ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings

# Load the C-API crate's RustPython-specific PyO3 configuration.
- name: Clippy C API
working-directory: crates/capi
run: cargo clippy --locked --all-targets -- -Dwarnings

# macOS/Windows clippy uses the same OS-surface filter as cargo check.
# Force with the run:cross-check label. Check names stay clippy (<os>).
Expand Down Expand Up @@ -786,6 +840,9 @@ jobs:

- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0

- name: Verify the CPython reference environment
run: python -c "import bz2, ctypes, lzma, pathlib, platform, sqlite3, ssl; expected = pathlib.Path('.python-version').read_text().strip(); actual = platform.python_version(); assert actual == expected, (actual, expected)"

- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt
Expand Down Expand Up @@ -1053,10 +1110,13 @@ jobs:
builtin_str
builtin_thread
cli_int_max_str_digits
import_lazy_native
protocol_index_argument
recursion
stdlib_ast
stdlib_asyncio
stdlib_asyncio_reinitialization_315
stdlib_bytearray_take_bytes_threads_315
stdlib_collections_deque
stdlib_contextvars
stdlib_ctypes
Expand All @@ -1068,17 +1128,20 @@ jobs:
stdlib_io_blocking_buffer
stdlib_itertools
stdlib_json
stdlib_lock_precedence_315
stdlib_mmap
stdlib_multiprocessing_condition
stdlib_multiprocessing_fork_sharer
stdlib_multiprocessing_sharedctypes
stdlib_os
stdlib_pwd
stdlib_queue_signal
stdlib_rlock_constructor_315
stdlib_select
stdlib_socket
stdlib_sqlite
stdlib_ssl
stdlib_ssl_introspection
stdlib_ssl_short_recv
stdlib_struct
stdlib_subinterpreters
Expand All @@ -1095,6 +1158,8 @@ jobs:
stdlib_types
stdlib_typing
stdlib_urllib_https_misaligned_recv
stdlib_venv_cycles
stdlib_weakref_gc_threads_315
stdlib_zoneinfo_cache
syntax_async
syntax_async_comprehension
Expand Down
35 changes: 35 additions & 0 deletions .github/workflows/cron-ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,32 @@ env:
CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,ssl-rustls-aws-lc,jit,host_env,doc,ucd_3_2_0

jobs:
manager-coverage-regression:
name: Verify manager XML-RPC under coverage
runs-on: ubuntu-latest
permissions:
contents: read
# Leave room for the instrumented build and the normal 600-second test limit.
timeout-minutes: 30
if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- uses: dtolnay/rust-toolchain@stable

- uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21
with:
tool: cargo-llvm-cov

# Keep this strict check independent of the full coverage collection below.
- name: Run the canonical manager XML-RPC regression
run: >-
cargo llvm-cov --no-report run --locked --
-m test -v -u all --timeout 600 --match '*test_remote'
test_multiprocessing_forkserver.test_manager

# codecov collects code coverage data from the rust tests, python snippets and python test suite.
# This is done using cargo-llvm-cov, which is a wrapper around llvm-cov.
codecov:
Expand Down Expand Up @@ -112,6 +138,9 @@ jobs:

- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0

- name: Verify the CPython reference environment
run: python -c "import bz2, ctypes, lzma, pathlib, platform, sqlite3, ssl; expected = pathlib.Path('.python-version').read_text().strip(); actual = platform.python_version(); assert actual == expected, (actual, expected)"

- name: build rustpython
run: cargo build --release --verbose --features sqlite

Expand Down Expand Up @@ -172,6 +201,12 @@ jobs:

- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0

- name: Verify the CPython reference environment
run: python -c "import bz2, ctypes, lzma, pathlib, platform, sqlite3, ssl; expected = pathlib.Path('.python-version').read_text().strip(); actual = platform.python_version(); assert actual == expected, (actual, expected)"

- name: Make the reference libpython available to benchmark executables
run: echo "LD_LIBRARY_PATH=$(python -c 'import sysconfig; print(sysconfig.get_config_var("LIBDIR"))')${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" >> "$GITHUB_ENV"

- run: cargo install cargo-criterion

- name: build benchmarks
Expand Down
80 changes: 64 additions & 16 deletions .github/workflows/lib-deps-check.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -111,32 +111,80 @@ jobs:

- name: Run deps check
if: steps.changed-files.outputs.modules != ''
id: deps-check
run: |
# Run deps for all modules at once
echo "deps_output<<EOF" >> "$GITHUB_OUTPUT"
output=$(python scripts/update_lib deps "${MODULES}" --depth 2 2>&1 || true)
echo "$output" >> "$GITHUB_OUTPUT"
echo "EOF" >> "$GITHUB_OUTPUT"
python scripts/update_lib deps "${MODULES}" --depth 2 > "$RUNNER_TEMP/lib-deps-report.md" 2>&1 || true
env:
MODULES: ${{ steps.changed-files.outputs.modules }}

- name: Prepare dependency report
if: steps.changed-files.outputs.modules != ''
id: deps-comment
shell: python
run: |
from os import environ
from pathlib import Path
report_path = Path(environ["RUNNER_TEMP"]) / "lib-deps-report.md"
report = report_path.read_text(encoding="utf-8")
if report:
heading = "## 📦 Library Dependencies\n\n"
body = (
heading
+ "The following Lib/ modules were modified. Here are their dependencies:\n\n"
+ report
+ "\n\n**Legend:**\n"
+ "- `[+]` path exists in CPython\n"
+ "- `[x]` up-to-date, `[ ]` outdated\n"
)
report_path.write_text(body, encoding="utf-8")
body_size = len(body.encode("utf-8"))
run_url = (
f"{environ['GITHUB_SERVER_URL']}/{environ['GITHUB_REPOSITORY']}"
f"/actions/runs/{environ['GITHUB_RUN_ID']}"
)
# Job summaries have a 1 MiB limit; keep larger reports as artifacts.
needs_artifact = body_size > 1024 * 1024
summary = body
if needs_artifact:
summary = (
heading
+ "The dependency report exceeds the job summary limit. "
+ "Download the full report from this run's `lib-deps-report` artifact.\n"
)
with open(environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as fd:
fd.write(summary)
# Leave room for the action's marker, counting UTF-8 bytes, not characters.
comment = body
if body_size >= 60000:
comment = (
heading
+ "The dependency report is too large for a pull request comment. "
+ f"[View the full report in the workflow run]({run_url}#summary).\n"
)
comment_path = Path(environ["RUNNER_TEMP"]) / "lib-deps-comment.md"
comment_path.write_text(comment, encoding="utf-8")
with open(environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as fd:
fd.write("has_report=true\n")
fd.write(f"needs_artifact={str(needs_artifact).lower()}\n")
- name: Upload oversized dependency report
if: steps.deps-comment.outputs.needs_artifact == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: lib-deps-report
path: ${{ runner.temp }}/lib-deps-report.md
if-no-files-found: error

- name: Post comment
if: steps.deps-check.outputs.deps_output != ''
if: steps.deps-comment.outputs.has_report == 'true'
uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
with:
header: lib-deps-check
number: ${{ github.event.pull_request.number }}
message: |
## 📦 Library Dependencies
The following Lib/ modules were modified. Here are their dependencies:
${{ steps.deps-check.outputs.deps_output }}
**Legend:**
- `[+]` path exists in CPython
- `[x]` up-to-date, `[ ]` outdated
path: ${{ runner.temp }}/lib-deps-comment.md

- name: Remove comment if no Lib changes
if: steps.changed-files.outputs.modules == ''
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/update-caches.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ concurrency:
cancel-in-progress: false

env:
PYO3_CONFIG_FILE: ${{ github.workspace }}/crates/capi/pyo3-rustpython.config
PYO3_NO_PYTHON: "1"
CARGO_INCREMENTAL: 0
CARGO_TERM_COLOR: always
CARGO_PROFILE_TEST_DEBUG: 0
CARGO_PROFILE_DEV_DEBUG: 0
CARGO_PROFILE_RELEASE_DEBUG: 0
# Keep feature list in sync with CI's release build in .github/workflows/ci.yaml.
CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit,doc --exclude rustpython_wasm --exclude rustpython-venvlauncher
CARGO_ARGS: --workspace --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls-aws-lc,host_env,threading,jit,capi,doc --exclude rustpython_wasm --exclude rustpython-venvlauncher

jobs:
build-caches:
Expand Down
3 changes: 3 additions & 0 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ repos:
entry: rustfmt
language: system
types: [rust]
# Keep the audited upstream source bytes and explicit backports intact.
exclude: '^vendor/(rustpython-ruff_python_parser|xml)/'
priority: 0

- id: generate-rs-opcode-metadata
Expand Down Expand Up @@ -67,6 +69,7 @@ repos:
hooks:
- id: cspell
types: [rust]
exclude: '^vendor/(rustpython-ruff_python_parser|xml)/'
additional_dependencies:
- '@cspell/dict-en_us'
- '@cspell/dict-cpp'
Expand Down
Loading