diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml index 67fa10d..c21e07b 100644 --- a/.github/workflows/codspeed.yml +++ b/.github/workflows/codspeed.yml @@ -8,23 +8,23 @@ on: env: PYTHON_VERSION: "3.12" + SHARDS: 4 jobs: - benchmarks-instrumentation: + benchmarks: strategy: matrix: - include: - - mode: "instrumentation" - runs-on: ubuntu-24.04 - - mode: "walltime" - runs-on: codspeed-macro + shard: [1, 2, 3, 4] + mode: ["instrumentation", "walltime"] - name: Run ${{ matrix.mode }} benchmarks - runs-on: ${{ matrix.runs-on }} + name: "Run ${{ matrix.mode }} benchmarks (Shard #${{ matrix.shard }})" + runs-on: ${{ matrix.mode == 'instrumentation' && 'ubuntu-24.04' || 'codspeed-macro' }} steps: - uses: actions/checkout@v4 with: submodules: "recursive" + - name: Install required-version defined in uv.toml + uses: astral-sh/setup-uv@v5 - uses: actions/setup-python@v2 with: python-version: ${{ env.PYTHON_VERSION }} @@ -32,10 +32,18 @@ jobs: run: | sudo apt-get update sudo apt-get install valgrind -y - pip install . + uv sync --dev sudo apt-get remove valgrind -y - name: Run benchmarks uses: CodSpeedHQ/action@main with: - run: pytest tests/benchmarks/ --codspeed + mode: ${{ matrix.mode }} + run: uv run pytest tests/benchmarks/ --codspeed --test-group=${{ matrix.shard }} --test-group-count=${{ env.SHARDS }} token: ${{ secrets.CODSPEED_TOKEN }} + + all-checks: + runs-on: ubuntu-latest + steps: + - run: echo "All CI checks passed." + needs: + - benchmarks diff --git a/CHANGELOG.md b/CHANGELOG.md index 188f08e..2d580e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,26 +5,207 @@ +## [4.1.0] - 2025-10-06 + +### ๐Ÿ› Bug Fixes +- Use ParamSpec in instrument definition instead of tuple and dict by @art049 +- Use ParamSpec in the plugin call definition by @flying-sheep + +### ๐Ÿ“š Documentation +- Add pull_request trigger comment by @adriencaccia + +### โš™๏ธ Internals +- Exclude beta tags in changelog by @art049 +- Bump instrument-hooks (#87) by @not-matthias in [#87](https://github.com/CodSpeedHQ/pytest-codspeed/pull/87) +- Switch to sharded upload (#75) by @art049 in [#75](https://github.com/CodSpeedHQ/pytest-codspeed/pull/75) +- Fix changelog generation by @art049 + + +# Changelog + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +### โš™๏ธ Internals + +- Fix changelog generation by @art049 + ## [4.0.0] - 2025-07-10 ### ๐Ÿš€ Features + - Update readme by @art049 + +### โš™๏ธ Internals + +- Remove pre-releases from git-cliff changelog by @art049 +- Link to the documentation by @art049 +- Improve reliability of perf trampoline compatibility checks by @art049 + +## [4.0.0-beta1] - 2025-06-10 + +### ๐Ÿ› Bug Fixes + +- Reenable walltime instrument hooks by @art049 in [#82](https://github.com/CodSpeedHQ/pytest-codspeed/pull/82) + +## [4.0.0-beta] - 2025-06-06 + +### ๐Ÿš€ Features + - Support pytest-benchmark's pedantic API by @art049 in [#81](https://github.com/CodSpeedHQ/pytest-codspeed/pull/81) - Make sure the benchmark fixture can only be called once per bench by @art049 - Support marker attributes to customize the walltime execution by @art049 in [#80](https://github.com/CodSpeedHQ/pytest-codspeed/pull/80) - Use instrument hooks by @not-matthias - Add instrument-hooks native module by @not-matthias + +### ๐Ÿ› Bug Fixes + +- Fix native library typing by @art049 + +### ๐Ÿงช Testing + +- Add benches from the documentation's getting started by @art049 in [#71](https://github.com/CodSpeedHQ/pytest-codspeed/pull/71) +- Add simple python benches by @art049 + +### โš™๏ธ Internals + +- Bump ruff by @art049 +- Update release workflow to include submodules by @art049 in [#79](https://github.com/CodSpeedHQ/pytest-codspeed/pull/79) +- Remove valgrind wrapper by @not-matthias +- Update apt before installing packages by @art049 + +## [3.2.0] - 2025-01-31 + +### ๐Ÿš€ Features + - Increase the min round time to a bigger value (+/- 1ms) by @art049 - Add benchmarks-walltime job to run additional performance benchmarks by @art049 in [#65](https://github.com/CodSpeedHQ/pytest-codspeed/pull/65) - Fix the random seed while measuring with instruments by @art049 in [#48](https://github.com/CodSpeedHQ/pytest-codspeed/pull/48) + +### ๐Ÿ› Bug Fixes + +- Use time per iteration instead of total round time in stats by @art049 + +### ๐Ÿ—๏ธ Refactor + +- Replace hardcoded outlier factor for improved readability by @art049 in [#67](https://github.com/CodSpeedHQ/pytest-codspeed/pull/67) + +### โš™๏ธ Internals + +- Fix self-dependency by @adriencaccia in [#66](https://github.com/CodSpeedHQ/pytest-codspeed/pull/66) +- Fix uv version in CI by @adriencaccia + +## [3.1.2] - 2025-01-09 + +### ๐Ÿ› Bug Fixes + +- Update package_data to include header and source files for valgrind wrapper by @art049 in [#64](https://github.com/CodSpeedHQ/pytest-codspeed/pull/64) + +## [3.1.1] - 2025-01-07 + +### โš™๏ธ Internals + +- Fix tag num with bumpver by @art049 in [#61](https://github.com/CodSpeedHQ/pytest-codspeed/pull/61) +- Update uv lock before release by @art049 +- Add a py3-none-any fallback wheel by @art049 + +## [3.1.0] - 2024-12-09 + +### ๐Ÿ—๏ธ Refactor + +- Remove the scripted semver generation by @art049 + +### โš™๏ธ Internals + +- Fix typo in cibuildwheel config by @art049 in [#57](https://github.com/CodSpeedHQ/pytest-codspeed/pull/57) + +## [3.1.0-beta] - 2024-12-06 + +### ๐Ÿš€ Features + - Check buildability and fallback when build doesn't work by @art049 - Compile the callgrind wrapper at build time by @art049 + +### ๐Ÿ› Bug Fixes + +- Allow build on arm64 by @art049 + +### โš™๏ธ Internals + +- Build wheels with cibuildwheel by @art049 +- Allow forcing integrated tests by @art049 +- Fix release script by @art049 +- Use bumpver to manage versions by @art049 +- Add a changelog by @art049 +- Force native extension build in CI by @art049 +- Updated matrix release workflow by @art049 +- Use a common python version in the codspeed job by @art049 +- Fix the codspeed workflow by @art049 +- Use uv in CI by @art049 +- Commit uv lock file by @art049 + +## [3.0.0] - 2024-10-29 + +### ๐Ÿ› Bug Fixes + +- Fix compatibility with pytest-benchmark 5.0.0 by @art049 in [#54](https://github.com/CodSpeedHQ/pytest-codspeed/pull/54) + +### โš™๏ธ Internals + +- Drop support for python3.8 by @art049 +- Expose type information (#53) by @Dreamsorcerer in [#53](https://github.com/CodSpeedHQ/pytest-codspeed/pull/53) +- Run the CI with ubuntu 24.04 by @art049 +- Improve naming in workflow examples by @art049 +- Bump actions/checkout to v4 (#47) by @fargito in [#47](https://github.com/CodSpeedHQ/pytest-codspeed/pull/47) + +## [3.0.0b4] - 2024-09-27 + +### ๐Ÿš€ Features + - Send more outlier data by @art049 + +### ๐Ÿ› Bug Fixes + +- Fix display of parametrized tests by @art049 +- Reenable gc logic by @art049 + +### ๐Ÿงช Testing + +- Add benches for various syscalls by @art049 + +## [3.0.0b3] - 2024-09-26 + +### ๐Ÿš€ Features + - Also save the lower and upper fences in the json data by @art049 in [#46](https://github.com/CodSpeedHQ/pytest-codspeed/pull/46) + +### ๐Ÿงช Testing + +- Refactor the algorithm benches using parametrization and add benches on bit_manipulation by @art049 + +## [3.0.0b2] - 2024-09-24 + +### ๐Ÿš€ Features + - Also save the q1 and q3 in the json data by @art049 in [#45](https://github.com/CodSpeedHQ/pytest-codspeed/pull/45) - Add the --codspeed-max-time flag by @art049 + +## [3.0.0b1] - 2024-09-20 + +### ๐Ÿš€ Features + - Send the semver version to cospeed instead of the PEP440 one by @art049 in [#44](https://github.com/CodSpeedHQ/pytest-codspeed/pull/44) - Also store the semver version by @art049 + +### ๐Ÿงช Testing + +- Add benches for TheAlgorithms/backtracking by @art049 in [#43](https://github.com/CodSpeedHQ/pytest-codspeed/pull/43) + +## [3.0.0b0] - 2024-09-18 + +### ๐Ÿš€ Features + - Improve table style when displaying results by @art049 in [#41](https://github.com/CodSpeedHQ/pytest-codspeed/pull/41) - Add the total bench time to the collected stats by @art049 - Add configuration and split tests between instruments by @art049 @@ -32,110 +213,107 @@ - Implement the walltime instrument by @art049 - Add bench of various python noop by @art049 - Avoid overriding pytest's default protocol (#32) by @kenodegard in [#32](https://github.com/CodSpeedHQ/pytest-codspeed/pull/32) -- Support pytest 8.1.1 by @art049 -- Avoid concurrent wrapper builds by @art049 -- Add a test for pytest-xdist compatibility by @art049 -- Release the package from the CI with trusted provider by @art049 -- Add a return type to the benchmark fixture by @art049 in [#13](https://github.com/CodSpeedHQ/pytest-codspeed/pull/13) -- Add support for returning values (#12) by @patrick91 in [#12](https://github.com/CodSpeedHQ/pytest-codspeed/pull/12) -- Warmup performance map generation by @art049 -- Add some details about the callgraph generation status in the header by @art049 -- Test that perf maps are generated by @art049 -- Add a local test matrix with hatch by @art049 -- Test that benchmark selection with -k works by @art049 -- Add support for CPython3.12 and perf trampoline by @art049 -- Add introspection benchmarks by @art049 in [#9](https://github.com/CodSpeedHQ/pytest-codspeed/pull/9) -- Add library metadata in the profile output by @art049 in [#5](https://github.com/CodSpeedHQ/pytest-codspeed/pull/5) -- Allow running along with pytest-benchmarks by @art049 ### ๐Ÿ› Bug Fixes -- Reenable walltime instrument hooks by @art049 in [#82](https://github.com/CodSpeedHQ/pytest-codspeed/pull/82) -- Fix native library typing by @art049 -- Use time per iteration instead of total round time in stats by @art049 -- Update package_data to include header and source files for valgrind wrapper by @art049 in [#64](https://github.com/CodSpeedHQ/pytest-codspeed/pull/64) -- Allow build on arm64 by @art049 -- Fix compatibility with pytest-benchmark 5.0.0 by @art049 in [#54](https://github.com/CodSpeedHQ/pytest-codspeed/pull/54) -- Fix display of parametrized tests by @art049 -- Reenable gc logic by @art049 + - Use importlib_metadata to keep backward compatibility by @art049 - Properly decide the mode depending on our env variable spec by @art049 - Disable pytest-speed when installed and codspeed is enabled by @art049 -- Loosen runtime requirements (#21) by @edgarrmondragon in [#21](https://github.com/CodSpeedHQ/pytest-codspeed/pull/21) -- Fix xdist test output assertion by @art049 -- Fix relative git path when using working-directory by @art049 in [#15](https://github.com/CodSpeedHQ/pytest-codspeed/pull/15) -- Fix typo in release.yml (#14) by @art049 in [#14](https://github.com/CodSpeedHQ/pytest-codspeed/pull/14) -- Fix setuptools installation with python3.12 by @art049 -- Support benchmark.extra_info parameters on the fixture by @art049 in [#10](https://github.com/CodSpeedHQ/pytest-codspeed/pull/10) -- Filter out pytest-benchmark warnings in the tests by @art049 -- Support kwargs with the benchmark fixture by @art049 in [#4](https://github.com/CodSpeedHQ/pytest-codspeed/pull/4) -- Avoid wrapping the callable to maintain existing results by @art049 -- Disable automatic garbage collection to increase stability by @art049 in [#2](https://github.com/CodSpeedHQ/pytest-codspeed/pull/2) -- Update readme by @art049 -- Fix the release script by @art049 -- Make the release script executable by @art049 -- Match the test output in any order by @art049 ### ๐Ÿ—๏ธ Refactor -- Replace hardcoded outlier factor for improved readability by @art049 in [#67](https://github.com/CodSpeedHQ/pytest-codspeed/pull/67) -- Remove the scripted semver generation by @art049 + - Differentiate the mode from the underlying instrument by @art049 - Move the instrumentation wrapper directly in the instrument by @art049 - Change Instrumentation to CPUInstrumentation by @art049 - Create an abstraction for each instrument by @art049 -- Use the pytest_run_protocol hook for better exec control by @art049 -- Manage compatibility env in the conftest by @art049 ### ๐Ÿ“š Documentation + - Update action version in the CI workflow configuration (#39) by @frgfm in [#39](https://github.com/CodSpeedHQ/pytest-codspeed/pull/39) - Bump action versions in README by @adriencaccia ### ๐Ÿงช Testing -- Add benches from the documentation's getting started by @art049 in [#71](https://github.com/CodSpeedHQ/pytest-codspeed/pull/71) -- Add simple python benches by @art049 -- Add benches for various syscalls by @art049 -- Refactor the algorithm benches using parametrization and add benches on bit_manipulation by @art049 -- Add benches for TheAlgorithms/backtracking by @art049 in [#43](https://github.com/CodSpeedHQ/pytest-codspeed/pull/43) + - Add benches for TheAlgorithms/audio_filters by @art049 in [#42](https://github.com/CodSpeedHQ/pytest-codspeed/pull/42) ### โš™๏ธ Internals -- Remove pre-releases from git-cliff changelog -- Link to the documentation by @art049 -- Improve reliability of perf trampoline compatibility checks by @art049 -- Bump ruff by @art049 -- Update release workflow to include submodules by @art049 in [#79](https://github.com/CodSpeedHQ/pytest-codspeed/pull/79) -- Remove valgrind wrapper by @not-matthias -- Update apt before installing packages by @art049 -- Fix self-dependency by @adriencaccia in [#66](https://github.com/CodSpeedHQ/pytest-codspeed/pull/66) -- Fix uv version in CI by @adriencaccia -- Fix tag num with bumpver by @art049 in [#61](https://github.com/CodSpeedHQ/pytest-codspeed/pull/61) -- Update uv lock before release by @art049 -- Add a py3-none-any fallback wheel by @art049 -- Fix typo in cibuildwheel config by @art049 in [#57](https://github.com/CodSpeedHQ/pytest-codspeed/pull/57) -- Build wheels with cibuildwheel by @art049 -- Allow forcing integrated tests by @art049 -- Fix release script by @art049 -- Use bumpver to manage versions by @art049 -- Add a changelog by @art049 -- Force native extension build in CI by @art049 -- Updated matrix release workflow by @art049 -- Use a common python version in the codspeed job by @art049 -- Fix the codspeed workflow by @art049 -- Use uv in CI by @art049 -- Commit uv lock file by @art049 -- Drop support for python3.8 by @art049 -- Expose type information (#53) by @Dreamsorcerer in [#53](https://github.com/CodSpeedHQ/pytest-codspeed/pull/53) -- Run the CI with ubuntu 24.04 by @art049 -- Improve naming in workflow examples by @art049 -- Bump actions/checkout to v4 (#47) by @fargito in [#47](https://github.com/CodSpeedHQ/pytest-codspeed/pull/47) + - Add a test on the walltime instrument by @art049 - Fix utils test using a fake git repo by @art049 - Update readme by @art049 - Support python 3.13 and drop 3.7 by @art049 in [#40](https://github.com/CodSpeedHQ/pytest-codspeed/pull/40) - Add TCH, FA, and UP to ruff lints (#29) by @kenodegard in [#29](https://github.com/CodSpeedHQ/pytest-codspeed/pull/29) + +## [2.2.1] - 2024-03-19 + +### ๐Ÿš€ Features + +- Support pytest 8.1.1 by @art049 + +### ๐Ÿ› Bug Fixes + +- Loosen runtime requirements (#21) by @edgarrmondragon in [#21](https://github.com/CodSpeedHQ/pytest-codspeed/pull/21) + +### โš™๏ธ Internals + - Add all-checks job to CI workflow by @art049 in [#28](https://github.com/CodSpeedHQ/pytest-codspeed/pull/28) - Switch from black to ruff format by @art049 - Update action version in README.md by @adriencaccia - Add codspeed badge to the readme by @art049 + +## [2.2.0] - 2023-09-01 + +### ๐Ÿš€ Features + +- Avoid concurrent wrapper builds by @art049 +- Add a test for pytest-xdist compatibility by @art049 + +### ๐Ÿ› Bug Fixes + +- Fix xdist test output assertion by @art049 + +## [2.1.0] - 2023-07-27 + +### ๐Ÿ› Bug Fixes + +- Fix relative git path when using working-directory by @art049 in [#15](https://github.com/CodSpeedHQ/pytest-codspeed/pull/15) +- Fix typo in release.yml (#14) by @art049 in [#14](https://github.com/CodSpeedHQ/pytest-codspeed/pull/14) + +## [2.0.1] - 2023-07-22 + +### ๐Ÿš€ Features + +- Release the package from the CI with trusted provider by @art049 +- Add a return type to the benchmark fixture by @art049 in [#13](https://github.com/CodSpeedHQ/pytest-codspeed/pull/13) +- Add support for returning values (#12) by @patrick91 in [#12](https://github.com/CodSpeedHQ/pytest-codspeed/pull/12) + +### ๐Ÿ› Bug Fixes + +- Fix setuptools installation with python3.12 by @art049 + +## [2.0.0] - 2023-07-04 + +### ๐Ÿš€ Features + +- Warmup performance map generation by @art049 +- Add some details about the callgraph generation status in the header by @art049 +- Test that perf maps are generated by @art049 +- Add a local test matrix with hatch by @art049 +- Test that benchmark selection with -k works by @art049 +- Add support for CPython3.12 and perf trampoline by @art049 +- Add introspection benchmarks by @art049 in [#9](https://github.com/CodSpeedHQ/pytest-codspeed/pull/9) + +### ๐Ÿ› Bug Fixes + +- Support benchmark.extra_info parameters on the fixture by @art049 in [#10](https://github.com/CodSpeedHQ/pytest-codspeed/pull/10) +- Filter out pytest-benchmark warnings in the tests by @art049 + +### ๐Ÿ—๏ธ Refactor + +- Use the pytest_run_protocol hook for better exec control by @art049 + +### โš™๏ธ Internals + - Separate the benchmark workflow by @art049 in [#8](https://github.com/CodSpeedHQ/pytest-codspeed/pull/8) - Bump version to 1.3.0 to trigger the callgraph generation by @art049 - Reuse same test code in the tests by @art049 @@ -145,12 +323,79 @@ - Restructure dev dependencies by @art049 - Replace isort by ruff by @art049 in [#11](https://github.com/CodSpeedHQ/pytest-codspeed/pull/11) - Add discord badge in the readme by @art049 + +## [1.2.2] - 2022-12-02 + +### ๐Ÿš€ Features + +- Add library metadata in the profile output by @art049 in [#5](https://github.com/CodSpeedHQ/pytest-codspeed/pull/5) + +## [1.2.1] - 2022-11-28 + +### ๐Ÿ› Bug Fixes + +- Support kwargs with the benchmark fixture by @art049 in [#4](https://github.com/CodSpeedHQ/pytest-codspeed/pull/4) + +## [1.2.0] - 2022-11-22 + +### ๐Ÿ› Bug Fixes + +- Avoid wrapping the callable to maintain existing results by @art049 +- Disable automatic garbage collection to increase stability by @art049 in [#2](https://github.com/CodSpeedHQ/pytest-codspeed/pull/2) +- Update readme by @art049 + +### โš™๏ธ Internals + - Update readme by @art049 + +## [1.1.0] - 2022-11-10 + +### ๐Ÿš€ Features + +- Allow running along with pytest-benchmarks by @art049 + +### ๐Ÿ› Bug Fixes + +- Fix the release script by @art049 +- Make the release script executable by @art049 +- Match the test output in any order by @art049 + +### ๐Ÿ—๏ธ Refactor + +- Manage compatibility env in the conftest by @art049 + +### โš™๏ธ Internals + - Add a pytest-benchmark compatibility test by @art049 in [#1](https://github.com/CodSpeedHQ/pytest-codspeed/pull/1) - Add more details on the pytest run by @art049 - Continue running on matrix item error by @art049 - Add a CI configuration with pytest-benchmark installed by @art049 +## [1.0.1] - 2022-11-05 +[unreleased]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v4.0.0..HEAD +[4.0.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v4.0.0-beta1..v4.0.0 +[4.0.0-beta1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v4.0.0-beta..v4.0.0-beta1 +[4.0.0-beta]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.2.0..v4.0.0-beta +[3.2.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.1.2..v3.2.0 +[3.1.2]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.1.1..v3.1.2 +[3.1.1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.1.0..v3.1.1 +[3.1.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.1.0-beta..v3.1.0 +[3.1.0-beta]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0..v3.1.0-beta +[3.0.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0b4..v3.0.0 +[3.0.0b4]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0b3..v3.0.0b4 +[3.0.0b3]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0b2..v3.0.0b3 +[3.0.0b2]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0b1..v3.0.0b2 +[3.0.0b1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v3.0.0b0..v3.0.0b1 +[3.0.0b0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v2.2.1..v3.0.0b0 +[2.2.1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v2.2.0..v2.2.1 +[2.2.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v2.1.0..v2.2.0 +[2.1.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v2.0.1..v2.1.0 +[2.0.1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v2.0.0..v2.0.1 +[2.0.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v1.2.2..v2.0.0 +[1.2.2]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v1.2.1..v1.2.2 +[1.2.1]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v1.2.0..v1.2.1 +[1.2.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v1.1.0..v1.2.0 +[1.1.0]: https://github.com/CodSpeedHQ/pytest-codspeed/compare/v1.0.4..v1.1.0 diff --git a/README.md b/README.md index d16e445..59aa1bb 100644 --- a/README.md +++ b/README.md @@ -92,7 +92,7 @@ on: push: branches: - "main" # or "master" - pull_request: + pull_request: # required to have reports on PRs # `workflow_dispatch` allows CodSpeed to trigger backtest # performance analysis in order to generate initial data. workflow_dispatch: @@ -111,8 +111,9 @@ jobs: run: pip install -r requirements.txt - name: Run benchmarks - uses: CodSpeedHQ/action@v3 + uses: CodSpeedHQ/action@v4 with: + mode: instrumentation # or `walltime` token: ${{ secrets.CODSPEED_TOKEN }} run: pytest tests/ --codspeed ``` diff --git a/cliff.toml b/cliff.toml index 5e9bba6..8c0ca49 100644 --- a/cliff.toml +++ b/cliff.toml @@ -72,8 +72,6 @@ postprocessors = [ # output = "test.md" [git] -# ignore pre-release tags -ignore_tags = "rc|alpha|beta" # parse the commits based on https://www.conventionalcommits.org conventional_commits = true # filter out the commits that are not conventional @@ -88,6 +86,7 @@ commit_preprocessors = [ # If the spelling is incorrect, it will be automatically fixed. #{ pattern = '.*', replace_command = 'typos --write-changes -' }, ] +ignore_tags = ".*beta.*" # regex for parsing and grouping commits commit_parsers = [ { message = "^feat", group = "๐Ÿš€ Features" }, diff --git a/pyproject.toml b/pyproject.toml index 7d1ed88..2295312 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,19 +37,24 @@ dependencies = [ ] [project.optional-dependencies] -lint = ["mypy ~= 1.11.2", "ruff ~= 0.11.12"] compat = [ "pytest-benchmark ~= 5.0.0", "pytest-xdist ~= 3.6.1", # "pytest-speed>=0.3.5", ] -test = ["pytest ~= 7.0", "pytest-cov ~= 4.0.0"] [tool.uv.sources] pytest-codspeed = { workspace = true } [dependency-groups] -dev = ["pytest-codspeed"] +dev = [ + "pytest-codspeed", + "mypy ~= 1.11.2", + "ruff ~= 0.11.12", + "pytest ~= 7.0", + "pytest-cov ~= 4.0.0", + "pytest-test-groups>=1.1.0", +] [project.entry-points] pytest11 = { codspeed = "pytest_codspeed.plugin" } @@ -66,7 +71,7 @@ version = { attr = "pytest_codspeed.__version__" } [tool.bumpver] -current_version = "4.0.0" +current_version = "4.1.0" version_pattern = "MAJOR.MINOR.PATCH[-TAG[NUM]]" commit_message = "Release v{new_version} ๐Ÿš€" tag_message = "Release v{new_version} ๐Ÿš€" diff --git a/scripts/pre-release.sh b/scripts/pre-release.sh index aedfe1d..99525ad 100755 --- a/scripts/pre-release.sh +++ b/scripts/pre-release.sh @@ -13,7 +13,7 @@ else echo "GITHUB_TOKEN is not set. Trying to fetch it from gh" GITHUB_TOKEN=$(gh auth token) fi - git cliff -o CHANGELOG.md --tag $VERSION + git cliff --unreleased --tag $VERSION --prepend CHANGELOG.md git add CHANGELOG.md fi diff --git a/src/pytest_codspeed/__init__.py b/src/pytest_codspeed/__init__.py index 27e4f87..3644fe5 100644 --- a/src/pytest_codspeed/__init__.py +++ b/src/pytest_codspeed/__init__.py @@ -1,6 +1,6 @@ -__version__ = "4.0.0" +__version__ = "4.1.0" # We also have the semver version since __version__ is not semver compliant -__semver_version__ = "4.0.0" +__semver_version__ = "4.1.0" from .plugin import BenchmarkFixture diff --git a/src/pytest_codspeed/instruments/__init__.py b/src/pytest_codspeed/instruments/__init__.py index fb264f1..71936db 100644 --- a/src/pytest_codspeed/instruments/__init__.py +++ b/src/pytest_codspeed/instruments/__init__.py @@ -8,11 +8,13 @@ from typing import Any, Callable, ClassVar, TypeVar import pytest + from typing_extensions import ParamSpec from pytest_codspeed.config import BenchmarkMarkerOptions, PedanticOptions from pytest_codspeed.plugin import CodSpeedConfig T = TypeVar("T") + P = ParamSpec("P") class Instrument(metaclass=ABCMeta): @@ -30,9 +32,9 @@ def measure( marker_options: BenchmarkMarkerOptions, name: str, uri: str, - fn: Callable[..., T], - *args: tuple, - **kwargs: dict[str, Any], + fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> T: ... @abstractmethod diff --git a/src/pytest_codspeed/instruments/hooks/__init__.py b/src/pytest_codspeed/instruments/hooks/__init__.py index 98e32fa..19aea91 100644 --- a/src/pytest_codspeed/instruments/hooks/__init__.py +++ b/src/pytest_codspeed/instruments/hooks/__init__.py @@ -63,7 +63,7 @@ def set_executed_benchmark(self, uri: str, pid: int | None = None) -> None: if pid is None: pid = os.getpid() - ret = self.lib.instrument_hooks_executed_benchmark( + ret = self.lib.instrument_hooks_set_executed_benchmark( self.instance, pid, uri.encode("ascii") ) if ret != 0: diff --git a/src/pytest_codspeed/instruments/hooks/build.py b/src/pytest_codspeed/instruments/hooks/build.py index 99071d1..992ee01 100644 --- a/src/pytest_codspeed/instruments/hooks/build.py +++ b/src/pytest_codspeed/instruments/hooks/build.py @@ -6,10 +6,35 @@ includes_dir = Path(__file__).parent.joinpath("instrument-hooks/includes") header_text = (includes_dir / "core.h").read_text() -filtered_header = "\n".join( - line for line in header_text.splitlines() if not line.strip().startswith("#") -) -ffibuilder.cdef(filtered_header) + + +# Manually copied from `instrument-hooks/includes/core.h` to avoid parsing issues +ffibuilder.cdef(""" +typedef uint64_t *InstrumentHooks; + +InstrumentHooks *instrument_hooks_init(void); +void instrument_hooks_deinit(InstrumentHooks *); + +bool instrument_hooks_is_instrumented(InstrumentHooks *); +uint8_t instrument_hooks_start_benchmark(InstrumentHooks *); +uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *); +uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid, + const char *uri); +uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name, + const char *version); + +#define MARKER_TYPE_SAMPLE_START 0 +#define MARKER_TYPE_SAMPLE_END 1 +#define MARKER_TYPE_BENCHMARK_START 2 +#define MARKER_TYPE_BENCHMARK_END 3 + +uint8_t instrument_hooks_add_marker(InstrumentHooks *, uint32_t pid, + uint8_t marker_type, uint64_t timestamp); +uint64_t instrument_hooks_current_timestamp(void); + +void callgrind_start_instrumentation(); +void callgrind_stop_instrumentation(); +""") ffibuilder.set_source( "pytest_codspeed.instruments.hooks.dist_instrument_hooks", diff --git a/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi b/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi index 3ae9ec5..37d1a79 100644 --- a/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi +++ b/src/pytest_codspeed/instruments/hooks/dist_instrument_hooks.pyi @@ -12,7 +12,7 @@ class lib: @staticmethod def instrument_hooks_stop_benchmark(hooks: InstrumentHooksPointer) -> int: ... @staticmethod - def instrument_hooks_executed_benchmark( + def instrument_hooks_set_executed_benchmark( hooks: InstrumentHooksPointer, pid: int, uri: bytes ) -> int: ... @staticmethod @@ -20,6 +20,12 @@ class lib: hooks: InstrumentHooksPointer, name: bytes, version: bytes ) -> int: ... @staticmethod + def instrument_hooks_add_marker( + hooks: InstrumentHooksPointer, pid: int, marker_type: int, timestamp: int + ) -> int: ... + @staticmethod + def instrument_hooks_current_timestamp() -> int: ... + @staticmethod def callgrind_start_instrumentation() -> int: ... @staticmethod def callgrind_stop_instrumentation() -> int: ... diff --git a/src/pytest_codspeed/instruments/hooks/instrument-hooks b/src/pytest_codspeed/instruments/hooks/instrument-hooks index b003e50..b1e401a 160000 --- a/src/pytest_codspeed/instruments/hooks/instrument-hooks +++ b/src/pytest_codspeed/instruments/hooks/instrument-hooks @@ -1 +1 @@ -Subproject commit b003e5024d61cfb784d6ac6f3ffd7d61bf7b9ec9 +Subproject commit b1e401a4d031ad308edb22ed59a52253a1ebe924 diff --git a/src/pytest_codspeed/instruments/valgrind.py b/src/pytest_codspeed/instruments/valgrind.py index 71b1542..dcb5eea 100644 --- a/src/pytest_codspeed/instruments/valgrind.py +++ b/src/pytest_codspeed/instruments/valgrind.py @@ -14,7 +14,7 @@ from pytest import Session from pytest_codspeed.config import PedanticOptions - from pytest_codspeed.instruments import T + from pytest_codspeed.instruments import P, T from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig @@ -52,9 +52,9 @@ def measure( marker_options: BenchmarkMarkerOptions, name: str, uri: str, - fn: Callable[..., T], - *args: tuple, - **kwargs: dict[str, Any], + fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> T: self.benchmark_count += 1 diff --git a/src/pytest_codspeed/instruments/walltime.py b/src/pytest_codspeed/instruments/walltime.py index 2423240..e2c5815 100644 --- a/src/pytest_codspeed/instruments/walltime.py +++ b/src/pytest_codspeed/instruments/walltime.py @@ -24,7 +24,7 @@ from pytest import Session from pytest_codspeed.config import PedanticOptions - from pytest_codspeed.instruments import T + from pytest_codspeed.instruments import P, T from pytest_codspeed.plugin import BenchmarkMarkerOptions, CodSpeedConfig DEFAULT_WARMUP_TIME_NS = 1_000_000_000 @@ -187,9 +187,9 @@ def measure( marker_options: BenchmarkMarkerOptions, name: str, uri: str, - fn: Callable[..., T], - *args: tuple, - **kwargs: dict[str, Any], + fn: Callable[P, T], + *args: P.args, + **kwargs: P.kwargs, ) -> T: benchmark_config = BenchmarkConfig.from_codspeed_config_and_marker_data( self.config, marker_options diff --git a/src/pytest_codspeed/plugin.py b/src/pytest_codspeed/plugin.py index 24e0401..014d7d7 100644 --- a/src/pytest_codspeed/plugin.py +++ b/src/pytest_codspeed/plugin.py @@ -30,11 +30,12 @@ from . import __version__ if TYPE_CHECKING: - from typing import Any, Callable, TypeVar + from typing import Any, Callable, ParamSpec, TypeVar from pytest_codspeed.instruments import Instrument T = TypeVar("T") + P = ParamSpec("P") @pytest.hookimpl(trylast=True) @@ -264,10 +265,10 @@ def wrap_runtest( plugin: CodSpeedPlugin, node: pytest.Item, config: pytest.Config, - fn: Callable[..., T], -) -> Callable[..., T]: + fn: Callable[P, T], +) -> Callable[P, T]: @functools.wraps(fn) - def wrapped(*args: tuple, **kwargs: dict[str, Any]) -> T: + def wrapped(*args: P.args, **kwargs: P.kwargs) -> T: return _measure(plugin, node, config, None, fn, args, kwargs) return wrapped @@ -328,9 +329,7 @@ def __init__(self, request: pytest.FixtureRequest): self._plugin = get_plugin(self._config) self._called = False - def __call__( - self, target: Callable[..., T], *args: tuple, **kwargs: dict[str, Any] - ) -> T: + def __call__(self, target: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: if self._called: raise RuntimeError("The benchmark fixture can only be used once per test") self._called = True diff --git a/uv.lock b/uv.lock index 0a56fed..cc69240 100644 --- a/uv.lock +++ b/uv.lock @@ -346,36 +346,36 @@ compat = [ { name = "pytest-benchmark" }, { name = "pytest-xdist" }, ] -lint = [ - { name = "mypy" }, - { name = "ruff" }, -] -test = [ - { name = "pytest" }, - { name = "pytest-cov" }, -] [package.dev-dependencies] dev = [ + { name = "mypy" }, + { name = "pytest" }, { name = "pytest-codspeed" }, + { name = "pytest-cov" }, + { name = "pytest-test-groups" }, + { name = "ruff" }, ] [package.metadata] requires-dist = [ { name = "cffi", specifier = ">=1.17.1" }, { name = "importlib-metadata", marker = "python_full_version < '3.10'", specifier = ">=8.5.0" }, - { name = "mypy", marker = "extra == 'lint'", specifier = "~=1.11.2" }, { name = "pytest", specifier = ">=3.8" }, - { name = "pytest", marker = "extra == 'test'", specifier = "~=7.0" }, { name = "pytest-benchmark", marker = "extra == 'compat'", specifier = "~=5.0.0" }, - { name = "pytest-cov", marker = "extra == 'test'", specifier = "~=4.0.0" }, { name = "pytest-xdist", marker = "extra == 'compat'", specifier = "~=3.6.1" }, { name = "rich", specifier = ">=13.8.1" }, - { name = "ruff", marker = "extra == 'lint'", specifier = "~=0.11.12" }, ] [package.metadata.requires-dev] -dev = [{ name = "pytest-codspeed", editable = "." }] +dev = [ + { name = "mypy", specifier = "~=1.11.2" }, + { name = "pytest", specifier = "~=7.0" }, + { name = "pytest-codspeed", editable = "." }, + { name = "pytest-cov", specifier = "~=4.0.0" }, + { name = "pytest-test-groups", specifier = ">=1.1.0" }, + { name = "ruff", specifier = "~=0.11.12" }, +] [[package]] name = "pytest-cov" @@ -390,6 +390,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fe/1f/9ec0ddd33bd2b37d6ec50bb39155bca4fe7085fa78b3b434c05459a860e3/pytest_cov-4.0.0-py3-none-any.whl", hash = "sha256:2feb1b751d66a8bd934e5edfa2e961d11309dc37b73b0eabe73b5945fee20f6b", size = 21554 }, ] +[[package]] +name = "pytest-test-groups" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/5a/c7874fe15e03d86a1109a3274b57a2473edb8a1dda4a4d27f25d848b6ff5/pytest_test_groups-1.2.1.tar.gz", hash = "sha256:67576b295522fc144b3a42fa1801f50ae962389e984b48bab4336686d09032f1", size = 8137 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/ff/7ff0ca5e8051931bf7fb65e31f085f7c0577615bf3a4776fb583cb471800/pytest_test_groups-1.2.1-py3-none-any.whl", hash = "sha256:8c7a016448f9ad347fb69a62f417f0a2358ecbf129fe44bc44ee991918a0bb73", size = 5278 }, +] + [[package]] name = "pytest-xdist" version = "3.6.1"