diff --git a/.coveragerc b/.coveragerc index b810471417f..e5a68198b84 100644 --- a/.coveragerc +++ b/.coveragerc @@ -10,6 +10,9 @@ include = *\Lib\site-packages\pytest.py parallel = 1 branch = 1 +# The sysmon core (default since Python 3.14) is much slower. +# Perhaps: https://github.com/coveragepy/coveragepy/issues/2082 +core = ctrace [paths] source = src/ @@ -25,9 +28,11 @@ exclude_lines = ^\s*raise NotImplementedError\b ^\s*return NotImplemented\b ^\s*assert False(,|$) + ^\s*case unreachable: ^\s*assert_never\( ^\s*if TYPE_CHECKING: ^\s*@overload( |$) + ^\s*def .+: \.\.\.$ ^\s*@pytest\.mark\.xfail diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index d6aac5c425d..4f4cdb6c564 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -5,7 +5,7 @@ # # To "install" it: # -# git config --local blame.ignoreRevsFile .gitblameignore +# git config --local blame.ignoreRevsFile .git-blame-ignore-revs # run black 703e4b11ba76171eccd3f13e723c47b810ded7ef diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 5f2d1cf09c8..88049407b45 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1,5 +1,7 @@ # info: # * https://help.github.com/en/articles/displaying-a-sponsor-button-in-your-repository # * https://tidelift.com/subscription/how-to-connect-tidelift-with-github +github: pytest-dev tidelift: pypi/pytest open_collective: pytest +thanks_dev: u/gh/pytest-dev diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index f5ea4d39764..c44ef2d8210 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -25,84 +25,117 @@ jobs: attestations: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false - name: Build and Check Package - uses: hynek/build-and-inspect-python-package@v2.6.0 + uses: hynek/build-and-inspect-python-package@efb823f52190ad02594531168b7a2d5790e66516 with: attest-build-provenance-github: 'true' - deploy: - if: github.repository == 'pytest-dev/pytest' + generate-gh-release-notes: needs: [package] runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install --upgrade tox + + - name: Generate release notes + env: + VERSION: ${{ github.event.inputs.version }} + run: | + tox -e generate-gh-release-notes -- "$VERSION" gh-release-notes.md + + - name: Upload release notes + uses: actions/upload-artifact@v4 + with: + name: release-notes + path: gh-release-notes.md + retention-days: 1 + + publish-to-pypi: + if: github.repository == 'pytest-dev/pytest' + # Need generate-gh-release-notes only for ordering. + # Don't want to release to PyPI if generating GitHub release notes fails. + needs: [package, generate-gh-release-notes] + runs-on: ubuntu-latest environment: deploy timeout-minutes: 30 permissions: id-token: write - contents: write steps: - - uses: actions/checkout@v4 - - name: Download Package - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: Packages path: dist - name: Publish package to PyPI - uses: pypa/gh-action-pypi-publish@v1.9.0 + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e + with: + attestations: true + + push-tag: + needs: [publish-to-pypi] + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: true - name: Push tag + env: + VERSION: ${{ github.event.inputs.version }} run: | git config user.name "pytest bot" git config user.email "pytestbot@gmail.com" - git tag --annotate --message=v${{ github.event.inputs.version }} ${{ github.event.inputs.version }} ${{ github.sha }} - git push origin ${{ github.event.inputs.version }} - - release-notes: + git tag --annotate --message=v"$VERSION" "$VERSION" ${{ github.sha }} + git push origin "$VERSION" - # todo: generate the content in the build job - # the goal being of using a github action script to push the release data - # after success instead of creating a complete python/tox env - needs: [deploy] + create-github-release: + needs: [push-tag, generate-gh-release-notes] runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 10 permissions: contents: write steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - persist-credentials: false - - name: Download Package - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: Packages path: dist - - name: Set up Python - uses: actions/setup-python@v5 + - name: Download release notes + uses: actions/download-artifact@v6 with: - python-version: "3.11" - - - name: Install tox - run: | - python -m pip install --upgrade pip - pip install --upgrade tox - - - name: Generate release notes - run: | - sudo apt-get install pandoc - tox -e generate-gh-release-notes -- ${{ github.event.inputs.version }} scripts/latest-release-notes.md + name: release-notes + path: . - name: Publish GitHub Release - uses: softprops/action-gh-release@v2 - with: - body_path: scripts/latest-release-notes.md - files: dist/* - tag_name: ${{ github.event.inputs.version }} + env: + VERSION: ${{ github.event.inputs.version }} + GH_REPO: ${{ github.repository }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create --notes-file gh-release-notes.md --verify-tag "$VERSION" dist/* diff --git a/.github/workflows/doc-check-links.yml b/.github/workflows/doc-check-links.yml new file mode 100644 index 00000000000..497ec73500a --- /dev/null +++ b/.github/workflows/doc-check-links.yml @@ -0,0 +1,37 @@ +name: Doc Check Links + +on: + schedule: + # At 00:00 on Sunday. + # https://crontab.guru + - cron: '0 0 * * 0' + workflow_dispatch: + +# Set permissions at the job level. +permissions: {} + +jobs: + doc-check-links: + if: github.repository_owner == 'pytest-dev' + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Python + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: pip + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install tox + + - name: Run sphinx linkcheck via tox + run: tox -e docs-checklinks diff --git a/.github/workflows/prepare-release-pr.yml b/.github/workflows/prepare-release-pr.yml index 1bb23fab844..9dcfea7bae5 100644 --- a/.github/workflows/prepare-release-pr.yml +++ b/.github/workflows/prepare-release-pr.yml @@ -27,14 +27,16 @@ jobs: pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 + # persist-credentials is needed in order for us to push the release branch. + persist-credentials: true - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: "3.8" + python-version: "3.13" - name: Install dependencies run: | @@ -43,10 +45,18 @@ jobs: - name: Prepare release PR (minor/patch release) if: github.event.inputs.major == 'no' + env: + BRANCH: ${{ github.event.inputs.branch }} + PRERELEASE: ${{ github.event.inputs.prerelease }} + GH_TOKEN: ${{ github.token }} run: | - tox -e prepare-release-pr -- ${{ github.event.inputs.branch }} ${{ github.token }} --prerelease='${{ github.event.inputs.prerelease }}' + tox -e prepare-release-pr -- "$BRANCH" --prerelease="$PRERELEASE" - name: Prepare release PR (major release) if: github.event.inputs.major == 'yes' + env: + BRANCH: ${{ github.event.inputs.branch }} + PRERELEASE: ${{ github.event.inputs.prerelease }} + GH_TOKEN: ${{ github.token }} run: | - tox -e prepare-release-pr -- ${{ github.event.inputs.branch }} ${{ github.token }} --major --prerelease='${{ github.event.inputs.prerelease }}' + tox -e prepare-release-pr -- "$BRANCH" --major --prerelease="$PRERELEASE" diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 82f9a1f2579..aeac36cea60 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -10,7 +10,7 @@ jobs: permissions: issues: write steps: - - uses: actions/stale@v9 + - uses: actions/stale@v10 with: debug-only: false days-before-issue-stale: 14 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9158d6bcc72..b7f0634d08d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,8 @@ on: - reopened # default - ready_for_review # used in PRs created from the release workflow + workflow_dispatch: # allows manual triggering of the workflow + env: PYTEST_ADDOPTS: "--color=yes" @@ -35,12 +37,12 @@ jobs: package: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false - name: Build and Check Package - uses: hynek/build-and-inspect-python-package@v2.6.0 + uses: hynek/build-and-inspect-python-package@efb823f52190ad02594531168b7a2d5790e66516 build: needs: [package] @@ -54,176 +56,218 @@ jobs: fail-fast: false matrix: name: [ - "windows-py38", - "windows-py38-pluggy", - "windows-py39", - "windows-py310", + "windows-py310-unittest-asynctest", + "windows-py310-unittest-twisted24", + "windows-py310-unittest-twisted25", + "windows-py310-pluggy", + "windows-py310-xdist", "windows-py311", "windows-py312", "windows-py313", + "windows-py314", - "ubuntu-py38", - "ubuntu-py38-pluggy", - "ubuntu-py38-freeze", - "ubuntu-py39", - "ubuntu-py310", + "ubuntu-py310-unittest-asynctest", + "ubuntu-py310-unittest-twisted24", + "ubuntu-py310-unittest-twisted25", + "ubuntu-py310-lsof-numpy-pexpect", + "ubuntu-py310-pluggy", + "ubuntu-py310-freeze", + "ubuntu-py310-xdist", "ubuntu-py311", "ubuntu-py312", - "ubuntu-py313", - "ubuntu-pypy3", + "ubuntu-py313-pexpect", + "ubuntu-py314", + "ubuntu-pypy3-xdist", - "macos-py38", - "macos-py39", "macos-py310", "macos-py312", "macos-py313", + "macos-py314", "doctesting", "plugins", ] include: - - name: "windows-py38" - python: "3.8" + # Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage. + - name: "windows-py310-unittest-asynctest" + python: "3.10" os: windows-latest - tox_env: "py38-unittestextras" + tox_env: "py310-asynctest" use_coverage: true - - name: "windows-py38-pluggy" - python: "3.8" + + - name: "windows-py310-unittest-twisted24" + python: "3.10" os: windows-latest - tox_env: "py38-pluggymain-pylib-xdist" - - name: "windows-py39" - python: "3.9" + tox_env: "py310-twisted24" + use_coverage: true + + - name: "windows-py310-unittest-twisted25" + python: "3.10" os: windows-latest - tox_env: "py39-xdist" - - name: "windows-py310" + tox_env: "py310-twisted25" + use_coverage: true + + - name: "windows-py310-pluggy" + python: "3.10" + os: windows-latest + tox_env: "py310-pluggymain-pylib-xdist" + xfail: true + + - name: "windows-py310-xdist" python: "3.10" os: windows-latest tox_env: "py310-xdist" + - name: "windows-py311" python: "3.11" os: windows-latest tox_env: "py311" + - name: "windows-py312" python: "3.12" os: windows-latest tox_env: "py312" + - name: "windows-py313" - python: "3.13-dev" + python: "3.13" os: windows-latest tox_env: "py313" + xfail: true - - name: "ubuntu-py38" - python: "3.8" + - name: "windows-py314" + python: "3.14" + os: windows-latest + tox_env: "py314" + use_coverage: true + + # Use separate jobs for different unittest flavors (twisted, asynctest) to ensure proper coverage. + - name: "ubuntu-py310-unittest-asynctest" + python: "3.10" os: ubuntu-latest - tox_env: "py38-lsof-numpy-pexpect" + tox_env: "py310-asynctest" use_coverage: true - - name: "ubuntu-py38-pluggy" - python: "3.8" + + - name: "ubuntu-py310-unittest-twisted24" + python: "3.10" os: ubuntu-latest - tox_env: "py38-pluggymain-pylib-xdist" - - name: "ubuntu-py38-freeze" - python: "3.8" + tox_env: "py310-twisted24" + use_coverage: true + + - name: "ubuntu-py310-unittest-twisted25" + python: "3.10" os: ubuntu-latest - tox_env: "py38-freeze" - - name: "ubuntu-py39" - python: "3.9" + tox_env: "py310-twisted25" + use_coverage: true + + - name: "ubuntu-py310-lsof-numpy-pexpect" + python: "3.10" os: ubuntu-latest - tox_env: "py39-xdist" - - name: "ubuntu-py310" + tox_env: "py310-lsof-numpy-pexpect" + use_coverage: true + + - name: "ubuntu-py310-pluggy" + python: "3.10" + os: ubuntu-latest + tox_env: "py310-pluggymain-pylib-xdist" + xfail: true + + - name: "ubuntu-py310-freeze" + python: "3.10" + os: ubuntu-latest + tox_env: "py310-freeze" + xfail: true + + - name: "ubuntu-py310-xdist" python: "3.10" os: ubuntu-latest tox_env: "py310-xdist" + - name: "ubuntu-py311" python: "3.11" os: ubuntu-latest tox_env: "py311" use_coverage: true + - name: "ubuntu-py312" python: "3.12" os: ubuntu-latest tox_env: "py312" use_coverage: true - - name: "ubuntu-py313" - python: "3.13-dev" + + - name: "ubuntu-py313-pexpect" + python: "3.13" os: ubuntu-latest - tox_env: "py313" + tox_env: "py313-pexpect" use_coverage: true - - name: "ubuntu-pypy3" - python: "pypy-3.9" + xfail: true + + - name: "ubuntu-py314" + python: "3.14" + os: ubuntu-latest + tox_env: "py314" + use_coverage: true + + - name: "ubuntu-pypy3-xdist" + python: "pypy-3.10" os: ubuntu-latest tox_env: "pypy3-xdist" - - name: "macos-py38" - python: "3.8" - os: macos-latest - tox_env: "py38-xdist" - - name: "macos-py39" - python: "3.9" - os: macos-latest - tox_env: "py39-xdist" - use_coverage: true + - name: "macos-py310" python: "3.10" os: macos-latest tox_env: "py310-xdist" + xfail: true + - name: "macos-py312" python: "3.12" os: macos-latest tox_env: "py312-xdist" + - name: "macos-py313" - python: "3.13-dev" + python: "3.13" os: macos-latest tox_env: "py313-xdist" + xfail: true + + - name: "macos-py314" + python: "3.14" + os: macos-latest + tox_env: "py314-xdist" - name: "plugins" python: "3.12" os: ubuntu-latest tox_env: "plugins" + - name: "doctesting" - python: "3.8" + python: "3.10" os: ubuntu-latest tox_env: "doctesting" use_coverage: true - continue-on-error: >- - ${{ - contains( - fromJSON( - '[ - "windows-py38-pluggy", - "windows-py313", - "ubuntu-py38-pluggy", - "ubuntu-py38-freeze", - "ubuntu-py313", - "macos-py38", - "macos-py313" - ]' - ), - matrix.name - ) - && true - || false - }} + continue-on-error: ${{ matrix.xfail && true || false }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 persist-credentials: false - name: Download Package - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: Packages path: dist - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.python }} - check-latest: ${{ endsWith(matrix.python, '-dev') }} + check-latest: true + allow-prereleases: true - name: Install dependencies run: | @@ -233,11 +277,15 @@ jobs: - name: Test without coverage if: "! matrix.use_coverage" shell: bash + env: + _PYTEST_TOX_POSARGS_JUNIT: --junitxml=junit.xml run: tox run -e ${{ matrix.tox_env }} --installpkg `find dist/*.tar.gz` - name: Test with coverage if: "matrix.use_coverage" shell: bash + env: + _PYTEST_TOX_POSARGS_JUNIT: --junitxml=junit.xml run: tox run -e ${{ matrix.tox_env }}-coverage --installpkg `find dist/*.tar.gz` - name: Generate coverage report @@ -246,12 +294,20 @@ jobs: - name: Upload coverage to Codecov if: "matrix.use_coverage" - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 with: fail_ci_if_error: false files: ./coverage.xml verbose: true + - name: Upload JUnit report to Codecov + uses: codecov/codecov-action@5a1091511ad55cbe89839c7260b706298ca349f7 + with: + fail_ci_if_error: false + files: junit.xml + report_type: test_results + verbose: true + check: # This job does nothing and is only used for the branch protection if: always() @@ -262,6 +318,6 @@ jobs: steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@223e4bb7a751b91f43eda76992bcfbf23b8b0302 + uses: re-actors/alls-green@a638d6464689bbb24c325bb3fe9404d63a913030 with: jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/update-plugin-list.yml b/.github/workflows/update-plugin-list.yml index ade8452afd5..b396d6e19d4 100644 --- a/.github/workflows/update-plugin-list.yml +++ b/.github/workflows/update-plugin-list.yml @@ -20,15 +20,16 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 + persist-credentials: false - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: "3.11" - cache: pip + python-version: "3.13" + - name: requests-cache uses: actions/cache@v4 with: @@ -36,18 +37,17 @@ jobs: key: plugins-http-cache-${{ github.run_id }} # Can use time based key as well restore-keys: plugins-http-cache- - - name: Install dependencies + - name: Install tox run: | python -m pip install --upgrade pip - pip install packaging requests tabulate[widechars] tqdm requests-cache platformdirs - + pip install --upgrade tox - name: Update Plugin List - run: python scripts/update-plugin-list.py + run: tox -e update-plugin-list - name: Create Pull Request id: pr - uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e with: commit-message: '[automated] Update plugin list' author: 'pytest bot ' @@ -61,8 +61,9 @@ jobs: - name: Instruct the maintainers to trigger CI by undrafting the PR env: GITHUB_TOKEN: ${{ github.token }} + PULL_REQUEST_NUMBER: ${{ steps.pr.outputs.pull-request-number }} run: >- gh pr comment --body 'Please mark the PR as ready for review to trigger PR checks.' --repo '${{ github.repository }}' - '${{ steps.pr.outputs.pull-request-number }}' + "$PULL_REQUEST_NUMBER" diff --git a/.gitignore b/.gitignore index c4557b33a1c..d0e8dc54ba1 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,7 @@ coverage.xml .vscode __pycache__/ .python-version +.claude/settings.local.json # generated by pip pip-wheel-metadata/ diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000..682334c7430 --- /dev/null +++ b/.mailmap @@ -0,0 +1,2 @@ +Freya Bruhin +Freya Bruhin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 419addd95be..2f9f56256b6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,52 +1,85 @@ +minimum_pre_commit_version: "4.4.0" repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: "v0.5.2" + rev: "v0.14.3" hooks: - - id: ruff + - id: ruff-check args: ["--fix"] - id: ruff-format - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v6.0.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-yaml +- repo: https://github.com/woodruffw/zizmor-pre-commit + rev: v1.16.2 + hooks: + - id: zizmor - repo: https://github.com/adamchainz/blacken-docs - rev: 1.18.0 + rev: 1.20.0 hooks: - id: blacken-docs additional_dependencies: [black==24.1.1] +- repo: https://github.com/codespell-project/codespell + rev: v2.4.1 + hooks: + - id: codespell + args: ["--toml=pyproject.toml"] + additional_dependencies: + - tomli - repo: https://github.com/pre-commit/pygrep-hooks rev: v1.10.0 hooks: - id: python-use-type-annotations - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.1 + rev: v1.18.2 hooks: - id: mypy files: ^(src/|testing/|scripts/) - args: [] additional_dependencies: - iniconfig>=1.1.0 - attrs>=19.2.0 - pluggy>=1.5.0 - packaging - tomli - - types-pkg_resources + - types-setuptools + - types-tabulate + # for mypy running on python>=3.11 since exceptiongroup is only a dependency + # on <3.11 + - exceptiongroup>=1.0.0rc8 +- repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.407 + hooks: + - id: pyright + files: ^(src/|scripts/) + additional_dependencies: + - iniconfig>=1.1.0 + - attrs>=19.2.0 + - pluggy>=1.5.0 + - packaging + - tomli + - types-setuptools - types-tabulate # for mypy running on python>=3.11 since exceptiongroup is only a dependency # on <3.11 - exceptiongroup>=1.0.0rc8 + # Manual because passing pyright is a work in progress. + stages: [manual] - repo: https://github.com/tox-dev/pyproject-fmt - rev: "2.1.4" + rev: "v2.11.0" hooks: - id: pyproject-fmt # https://pyproject-fmt.readthedocs.io/en/latest/#calculating-max-supported-python-version additional_dependencies: ["tox>=4.9"] - repo: https://github.com/asottile/pyupgrade - rev: v3.16.0 + rev: v3.21.0 hooks: - id: pyupgrade + args: + - "--py310-plus" + # Manual because ruff does what pyupgrade does and the two are not out of sync + # often enough to make launching pyupgrade everytime worth it stages: [manual] - repo: local hooks: @@ -55,14 +88,15 @@ repos: entry: pylint language: system types: [python] - args: ["-rn", "-sn", "--fail-on=I"] + args: ["-rn", "-sn", "--fail-on=I", "--enable-all-extentions"] + require_serial: true stages: [manual] - id: rst name: rst - entry: rst-lint --encoding utf-8 + entry: rst-lint files: ^(RELEASING.rst|README.rst|TIDELIFT.rst)$ language: python - additional_dependencies: [pygments, restructuredtext_lint] + additional_dependencies: [pygments, restructuredtext_lint>=2.0.0] - id: changelogs-rst name: changelog filenames language: fail diff --git a/.readthedocs.yaml b/.readthedocs.yaml index f7370f1bb98..6380b34adec 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -17,7 +17,7 @@ build: os: ubuntu-24.04 tools: python: >- - 3.12 + 3.13 apt_packages: - inkscape jobs: diff --git a/AUTHORS b/AUTHORS index 9b6cb6a9d23..e8140292aa4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -14,6 +14,7 @@ Ahn Ki-Wook Akhilesh Ramakrishnan Akiomi Kamakura Alan Velasco +Alejandro Villate Alessio Izzo Alex Jones Alex Lambson @@ -24,6 +25,7 @@ Alice Purcell Allan Feldman Aly Sivji Amir Elkess +Ammar Askar Anatoly Bubenkoff Anders Hovmöller Andras Mitzki @@ -31,17 +33,20 @@ Andras Tim Andrea Cimatoribus Andreas Motl Andreas Zeidler +Andrew Pikul Andrew Shapton Andrey Paramonov Andrzej Klajnert Andrzej Ostrowski Andy Freeland Anita Hammer +Anna Tasiopoulou Anthon van der Neut Anthony Shaw Anthony Sottile Anton Grinevich Anton Lodder +Anton Zhilin Antony Lee Arel Cordero Arias Emmanuel @@ -51,9 +56,11 @@ Aron Coyle Aron Curzon Arthur Richard Ashish Kurmi +Ashley Whetter Aviral Verma Aviv Palivoda Babak Keyvani +Bahram Farahmand Barney Gale Ben Brown Ben Gartner @@ -77,6 +84,7 @@ Carlos Jenkins Ceridwen Charles Cloud Charles Machalow +Charles-Meldhine Madi Mnemoi (cmnemoi) Charnjit SiNGH (CCSJ) Cheuk Ting Ho Chris Mahoney @@ -85,6 +93,7 @@ Chris NeJame Chris Rose Chris Wheeler Christian Boelsen +Christian Clauss Christian Fetzer Christian Neumüller Christian Theunert @@ -93,6 +102,7 @@ Christine Mecklenborg Christoph Buelter Christopher Dignam Christopher Gilling +Christopher Head Claire Cecil Claudio Madotto Clément M.T. Robert @@ -100,6 +110,7 @@ Cornelius Riemenschneider CrazyMerlyn Cristian Vera Cyrus Maden +Daara Shaw Damian Skrzypczak Daniel Grana Daniel Hahler @@ -115,15 +126,19 @@ Dave Hunt David Díaz-Barquero David Mohr David Paul Röthlisberger +David Peled David Szotten David Vierra Daw-Ran Liou Debi Mishra +Denis Cherednichenko Denis Kirisov Denivy Braiam Rück +Deysha Rivera Dheeraj C K Dhiren Serai Diego Russo +Dima Gerasimov Dmitry Dygalo Dmitry Pribysh Dominic Mortlock @@ -137,6 +152,7 @@ Eero Vaher Eli Boyarski Elizaveta Shashkova Éloi Rivard +Emil Hjelm Endre Galaczi Eric Hunsberger Eric Liu @@ -145,6 +161,7 @@ Eric Yuan Erik Aronesty Erik Hasse Erik M. Bray +Ethan Wass Evan Kepner Evgeny Seliverstov Fabian Sturm @@ -155,10 +172,11 @@ faph Felix Hofstätter Felix Nieuwenhuizen Feng Ma -Florian Bruhin Florian Dahlitz Floris Bruynooghe +Frank Hoffmann Fraser Stark +Freya Bruhin Gabriel Landau Gabriel Reis Garvit Shubham @@ -187,6 +205,7 @@ Ilya Konstantinov Ionuț Turturică Isaac Virshup Israel Fruchter +Israël Hallé Itxaso Aizpurua Iwan Briquemont Jaap Broekhuizen @@ -204,6 +223,7 @@ Jeff Rackauckas Jeff Widman Jenni Rinker Jens Tröger +Jiajun Xu John Eddie Ayson John Litborn John Towler @@ -211,12 +231,14 @@ Jon Parise Jon Sonesen Jonas Obrist Jordan Guymon +Jordan Macdonald Jordan Moldow Jordan Speicher Joseph Hunkeler Joseph Sawaya Josh Karpel Joshua Bronson +Julian Valentin Jurko Gospodnetić Justice Ndou Justyna Janczyszyn @@ -235,6 +257,7 @@ Kevin Hierro Carrasco Kevin J. Foley Kian Eliasi Kian-Meng Ang +Kim Soo Kodi B. Arfer Kojo Idrissa Kostis Anagnostopoulos @@ -242,15 +265,18 @@ Kristoffer Nordström Kyle Altendorf Lawrence Mitchell Lee Kamentsky +Leonardus Chen Lev Maximov Levon Saldamli Lewis Cowles +Liam DeVoe Llandy Riveron Del Risco Loic Esteve lovetheguitar Lukas Bednar Luke Murphy Maciek Fijalkowski +Maggie Chung Maho Maik Figura Mandeep Bhutani @@ -258,9 +284,11 @@ Manuel Krebber Marc Mueller Marc Schlaich Marcelo Duarte Trevisani +Marcin Augustynów Marcin Bachry Marc Bresson Marco Gorelli +Marcos Boger Mark Abramowitz Mark Dickinson Mark Vong @@ -285,6 +313,7 @@ Michael Goerz Michael Krebs Michael Seifert Michael Vogt +Michael Reznik Michal Wajszczuk Michał Górny Michał Zięba @@ -295,10 +324,13 @@ Mike Hoyle (hoylemd) Mike Lundy Milan Lesnek Miro Hrončok +Mulat Mekonen mrbean-bremen Nathan Goldbaum +Nathan Rousseau Nathaniel Compton Nathaniel Waisbrot +Nauman Ahmed Ned Batchelder Neil Martin Neven Mundar @@ -312,10 +344,13 @@ Nikolay Kondratyev Nipunn Koorapati Oleg Pidsadnyi Oleg Sushchenko +Oleksandr Zavertniev Olga Matoula Oliver Bestwalter +Olivier Grisel Omar Kohl Omer Hadari +Omri Golan Ondřej Súkup Oscar Benjamin Parth Patel @@ -325,8 +360,10 @@ Paul Müller Paul Reece Pauli Virtanen Pavel Karateev +Pavel Zhukov Paweł Adamczak Pedro Algarvio +Peter Gessler Petter Strandmark Philipp Loose Pierre Sassoulas @@ -345,12 +382,16 @@ Ralf Schmitt Ralph Giles Ram Rachum Ran Benita +Randy Döring Raphael Castaneda Raphael Pierzina Rafal Semik +Reza Mousavi Raquel Alegre Ravi Chandra Reagan Lee +Reilly Brogan +Rob Arrow Robert Holt Roberto Aldera Roberto Polli @@ -368,6 +409,7 @@ Sadra Barikbin Saiprasad Kale Samuel Colvin Samuel Dion-Girardeau +Samuel Gaist Samuel Jirovec Samuel Searles-Bryant Samuel Therrien (Avasam) @@ -381,6 +423,7 @@ Serhii Mozghovyi Seth Junot Shantanu Jain Sharad Nair +Shaygan Hooshyari Shubham Adep Simon Blanchard Simon Gomizelj @@ -396,11 +439,13 @@ Stefanie Molin Stefano Taschini Steffen Allner Stephan Obermann +Sven Sven-Hendrik Haase Sviatoslav Sydorenko Sylvain Marié Tadek Teleżyński Takafumi Arakaki +Takumi Otani Taneli Hukkinen Tanvi Mehta Tanya Agarwal @@ -411,11 +456,14 @@ Ted Xiao Terje Runde Thomas Grainger Thomas Hisch +Tianyu Dongfang Tim Hoffmann Tim Strazny TJ Bruno Tobias Diez +Tobias Petersen Tom Dalton +Tom Most Tom Viner Tomáš Gavenčiak Tomer Keren @@ -443,6 +491,7 @@ Volodymyr Kochetkov Volodymyr Piskun Wei Lin Wil Cooley +Will Riley William Lee Wim Glenn Wouter van Ackooy @@ -457,6 +506,7 @@ Yusuke Kadowaki Yutian Li Yuval Shimon Zac Hatfield-Dodds +Zac Palmer Laporte Zach Snicker Zachary Kneupper Zachary OBrien diff --git a/CITATION b/CITATION index d4e9d8ec7a1..ac7c5d6f312 100644 --- a/CITATION +++ b/CITATION @@ -1,16 +1,28 @@ NOTE: Change "x.y" by the version you use. If you are unsure about which version -you are using run: `pip show pytest`. +you are using run: `pip show pytest`. Do not include the patch number (i.e., z in x.y.z) Text: [pytest] pytest x.y, 2004 Krekel et al., https://github.com/pytest-dev/pytest +BibLaTeX: + +@software{pytest, + title = {pytest x.y}, + author = {Holger Krekel and Bruno Oliveira and Ronny Pfannschmidt and Floris Bruynooghe and Brianna Laugher and Freya Bruhin}, + year = {2004}, + version = {x.y}, + url = {https://github.com/pytest-dev/pytest}, + note = {Contributors: Holger Krekel and Bruno Oliveira and Ronny Pfannschmidt and Floris Bruynooghe and Brianna Laugher and Freya Bruhin and others} +} + BibTeX: -@misc{pytestx.y, - title = {pytest x.y}, - author = {Krekel, Holger and Oliveira, Bruno and Pfannschmidt, Ronny and Bruynooghe, Floris and Laugher, Brianna and Bruhin, Florian}, - year = {2004}, - url = {https://github.com/pytest-dev/pytest}, +@misc{pytest, + author = {Holger Krekel and Bruno Oliveira and Ronny Pfannschmidt and Floris Bruynooghe and Brianna Laugher and Freya Bruhin}, + title = {pytest x.y}, + year = {2004}, + howpublished = {\url{https://github.com/pytest-dev/pytest}}, + note = {Version x.y. Contributors include Holger Krekel, Bruno Oliveira, Ronny Pfannschmidt, Floris Bruynooghe, Brianna Laugher, Freya Bruhin, and others.} } diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index f0ca304be4e..14d56263449 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -70,7 +70,7 @@ contacted individually: - Brianna Laugher ([@pfctdayelise](https://github.com/pfctdayelise)): brianna@laugher.id.au - Bruno Oliveira ([@nicoddemus](https://github.com/nicoddemus)): nicoddemus@gmail.com -- Florian Bruhin ([@the-compiler](https://github.com/the-compiler)): pytest@the-compiler.org +- Freya Bruhin ([@the-compiler](https://github.com/the-compiler)): pytest@the-compiler.org ## Attribution diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 12e2b18bb52..fb9f7f4d53d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -139,15 +139,14 @@ The objectives of the ``pytest-dev`` organisation are: * Sharing some of the maintenance responsibility (in case a maintainer no longer wishes to maintain a plugin) -You can submit your plugin by subscribing to the `pytest-dev mail list -`_ and writing a -mail pointing to your existing pytest plugin repository which must have +You can submit your plugin by posting a new topic in the `pytest-dev GitHub Discussions +`_ pointing to your existing pytest plugin repository which must have the following: - PyPI presence with packaging metadata that contains a ``pytest-`` prefixed name, version number, authors, short and long description. -- a `tox configuration `_ +- a `tox configuration `_ for running tests using `tox `_. - a ``README`` describing how to use the plugin and on which @@ -198,13 +197,13 @@ Short version #. Follow `PEP-8 `_ for naming. #. Tests are run using ``tox``:: - tox -e linting,py39 + tox -e linting,py313 The test environments above are usually enough to cover most cases locally. #. Write a ``changelog`` entry: ``changelog/2574.bugfix.rst``, use issue id number and one of ``feature``, ``improvement``, ``bugfix``, ``doc``, ``deprecation``, - ``breaking``, ``vendor`` or ``trivial`` for the issue type. + ``breaking``, ``vendor``, ``packaging``, ``contrib``, or ``misc`` for the issue type. #. Unless your change is a trivial or a documentation fix (e.g., a typo or reword of a small section) please @@ -270,24 +269,24 @@ Here is a simple overview, with pytest-specific bits: #. Run all the tests - You need to have Python 3.8 or later available in your system. Now + You need to have a supported Python version available in your system. Now running tests is as simple as issuing this command:: - $ tox -e linting,py39 + $ tox -e linting,py - This command will run tests via the "tox" tool against Python 3.9 - and also perform "lint" coding-style checks. + This command will run tests via the "tox" tool against your default Python + version and also perform "lint" coding-style checks. #. You can now edit your local working copy and run the tests again as necessary. Please follow `PEP-8 `_ for naming. - You can pass different options to ``tox``. For example, to run tests on Python 3.9 and pass options to pytest - (e.g. enter pdb on failure) to pytest you can do:: + You can pass different options to ``tox``. For example, to run tests on Python 3.13 and pass options to pytest + (e.g. enter pdb on failure) you can do:: - $ tox -e py39 -- --pdb + $ tox -e py313 -- --pdb - Or to only run tests in a particular test module on Python 3.9:: + Or to only run tests in a particular test module on Python 3.12:: - $ tox -e py39 -- testing/test_config.py + $ tox -e py312 -- testing/test_config.py When committing, ``pre-commit`` will re-format the files if necessary. @@ -306,8 +305,9 @@ Here is a simple overview, with pytest-specific bits: #. Create a new changelog entry in ``changelog``. The file should be named ``..rst``, where *issueid* is the number of the issue related to the change and *type* is one of - ``feature``, ``improvement``, ``bugfix``, ``doc``, ``deprecation``, ``breaking``, ``vendor`` - or ``trivial``. You may skip creating the changelog entry if the change doesn't affect the + ``feature``, ``improvement``, ``bugfix``, ``doc``, ``deprecation``, ``breaking``, ``vendor``, + ``packaging``, ``contrib``, or ``misc``. + You may skip creating the changelog entry if the change doesn't affect the documented behaviour of pytest. #. Add yourself to ``AUTHORS`` file if not there yet, in alphabetical order. @@ -346,7 +346,7 @@ For example, to ensure a simple test passes you can write: result.assert_outcomes(failed=0, passed=1) -Alternatively, it is possible to make checks based on the actual output of the termal using +Alternatively, it is possible to make checks based on the actual output of the terminal using *glob-like* expressions: .. code-block:: python @@ -380,6 +380,57 @@ pull requests from other contributors yourself after having reviewed them. +Merge/squash guidelines +----------------------- + +When a PR is approved and ready to be integrated to the ``main`` branch, one has the option to *merge* the commits unchanged, or *squash* all the commits into a single commit. + +Here are some guidelines on how to proceed, based on examples of a single PR commit history: + +1. Miscellaneous commits: + + * ``Implement X`` + * ``Fix test_a`` + * ``Add myself to AUTHORS`` + * ``fixup! Fix test_a`` + * ``Update tests/test_integration.py`` + * ``Merge origin/main into PR branch`` + * ``Update tests/test_integration.py`` + + In this case, prefer to use the **Squash** merge strategy: the commit history is a bit messy (not in a derogatory way, often one just commits changes because they know the changes will eventually be squashed together), so squashing everything into a single commit is best. You must clean up the commit message, making sure it contains useful details. + +2. Separate commits related to the same topic: + + * ``Implement X`` + * ``Add myself to AUTHORS`` + * ``Update CHANGELOG for X`` + + In this case, prefer to use the **Squash** merge strategy: while the commit history is not "messy" as in the example above, the individual commits do not bring much value overall, specially when looking at the changes a few months/years down the line. + +3. Separate commits, each with their own topic (refactorings, renames, etc), but still have a larger topic/purpose. + + * ``Refactor class X in preparation for feature Y`` + * ``Remove unused method`` + * ``Implement feature Y`` + + In this case, prefer to use the **Merge** strategy: each commit is valuable on its own, even if they serve a common topic overall. Looking at the history later, it is useful to have the removal of the unused method separately on its own commit, along with more information (such as how it became unused in the first place). + +4. Separate commits, each with their own topic, but without a larger topic/purpose other than improve the code base (using more modern techniques, improve typing, removing clutter, etc). + + * ``Improve internal names in X`` + * ``Add type annotations to Y`` + * ``Remove unnecessary dict access`` + * ``Remove unreachable code due to EOL Python`` + + In this case, prefer to use the **Merge** strategy: each commit is valuable on its own, and the information on each is valuable in the long term. + + +As mentioned, those are overall guidelines, not rules cast in stone. This topic was discussed in `#12633 `_. + + +*Backport PRs* (as those created automatically from a ``backport`` label) should always be **squashed**, as they preserve the original PR author. + + Backporting bug fixes for the next patch release ------------------------------------------------ @@ -428,16 +479,18 @@ above? to do the backport. 2. However, often the merge is done by another maintainer, in which case it is nice of them to do the backport procedure if they have the time. -3. For bugs submitted by non-maintainers, it is expected that a core developer will to do +3. For bugs submitted by non-maintainers, it is expected that a core developer will do the backport, normally the one that merged the PR on ``main``. -4. If a non-maintainers notices a bug which is fixed on ``main`` but has not been backported - (due to maintainers forgetting to apply the *needs backport* label, or just plain missing it), +4. If a non-maintainer notices a bug which is fixed on ``main`` but has not been backported + (due to maintainers forgetting to apply the *needs backport* or *backport x.x.x* labels, or just plain missing it), they are also welcome to open a PR with the backport. The procedure is simple and really helps with the maintenance of the project. All the above are not rules, but merely some guidelines/suggestions on what we should expect about backports. +Backports should be **squashed** (rather than **merged**), as doing so preserves the original PR author correctly. + Handling stale issues/PRs ------------------------- @@ -459,7 +512,7 @@ can always reopen the issue/pull request in their own time later if it makes sen When to close ~~~~~~~~~~~~~ -Here are a few general rules the maintainers use deciding when to close issues/PRs because +Here are a few general rules the maintainers use to decide when to close issues/PRs because of lack of inactivity: * Issues labeled ``question`` or ``needs information``: closed after 14 days inactive. @@ -471,7 +524,7 @@ The above are **not hard rules**, but merely **guidelines**, and can be (and oft Closing pull requests ~~~~~~~~~~~~~~~~~~~~~ -When closing a Pull Request, it needs to be acknowledging the time, effort, and interest demonstrated by the person which submitted it. As mentioned previously, it is not the intent of the team to dismiss a stalled pull request entirely but to merely to clear up our queue, so a message like the one below is warranted when closing a pull request that went stale: +When closing a Pull Request, we should acknowledge the time, effort, and interest demonstrated by the person who submitted it. As mentioned previously, it is not the intent of the team to dismiss a stalled pull request entirely but to merely to clear up our queue, so a message like the one below is warranted when closing a pull request that went stale: Hi , @@ -479,15 +532,15 @@ When closing a Pull Request, it needs to be acknowledging the time, effort, and We noticed it has been awhile since you have updated this PR, however. pytest is a high activity project, with many issues/PRs being opened daily, so it is hard for us maintainers to track which PRs are ready for merging, for review, or need more attention. - So for those reasons we, think it is best to close the PR for now, but with the only intention to clean up our queue, it is by no means a rejection of your changes. We still encourage you to re-open this PR (it is just a click of a button away) when you are ready to get back to it. + So for those reasons, we think it is best to close the PR for now, but with the only intention to clean up our queue, it is by no means a rejection of your changes. We still encourage you to re-open this PR (it is just a click of a button away) when you are ready to get back to it. Again we appreciate your time for working on this, and hope you might get back to this at a later time! -Closing Issues +Closing issues -------------- When a pull request is submitted to fix an issue, add text like ``closes #XYZW`` to the PR description and/or commits (where ``XYZW`` is the issue number). See the `GitHub docs `_ for more information. -When an issue is due to user error (e.g. misunderstanding of a functionality), please politely explain to the user why the issue raised is really a non-issue and ask them to close the issue if they have no further questions. If the original requestor is unresponsive, the issue will be handled as described in the section `Handling stale issues/PRs`_ above. +When an issue is due to user error (e.g. misunderstanding of a functionality), please politely explain to the user why the issue raised is really a non-issue and ask them to close the issue if they have no further questions. If the original requester is unresponsive, the issue will be handled as described in the section `Handling stale issues/PRs`_ above. diff --git a/README.rst b/README.rst index a81e082cdd7..3bc5f06fc81 100644 --- a/README.rst +++ b/README.rst @@ -79,7 +79,7 @@ To execute it:: ========================== 1 failed in 0.04 seconds =========================== -Due to ``pytest``'s detailed assertion introspection, only plain ``assert`` statements are used. See `getting-started `_ for more examples. +Thanks to ``pytest``'s detailed assertion introspection, you can simply use plain ``assert`` statements. See `getting-started `_ for more examples. Features @@ -97,7 +97,7 @@ Features - Can run `unittest `_ (or trial) test suites out of the box -- Python 3.8+ or PyPy3 +- Python 3.10+ or PyPy3 - Rich plugin architecture, with over 1300+ `external plugins `_ and thriving community diff --git a/RELEASING.rst b/RELEASING.rst index 08004a84c00..2b00e658e7a 100644 --- a/RELEASING.rst +++ b/RELEASING.rst @@ -117,7 +117,7 @@ To release a version ``MAJOR.MINOR.PATCH``, follow these steps: #. Create a branch ``release-MAJOR.MINOR.PATCH`` from the ``MAJOR.MINOR.x`` branch. - Ensure your are updated and in a clean working tree. + Ensure your local checkout is up to date and in a clean working tree. #. Using ``tox``, generate docs, changelog, announcements:: @@ -133,10 +133,14 @@ Releasing Both automatic and manual processes described above follow the same steps from this point onward. -#. After all tests pass and the PR has been approved, trigger the ``deploy`` job +#. After all tests pass and the PR has been approved, trigger the ``deploy`` workflow in https://github.com/pytest-dev/pytest/actions/workflows/deploy.yml, using the ``release-MAJOR.MINOR.PATCH`` branch as source. + Using the command-line:: + + $ gh workflow run deploy.yml -R pytest-dev/pytest --ref=release-{VERSION} -f version={VERSION} + This job will require approval from ``pytest-dev/core``, after which it will publish to PyPI and tag the repository. @@ -158,16 +162,16 @@ Both automatic and manual processes described above follow the same steps from t git tag MAJOR.{MINOR+1}.0.dev0 git push upstream MAJOR.{MINOR+1}.0.dev0 -#. For major and minor releases, change the default version in the `Read the Docs Settings `_ to the new branch. - #. Send an email announcement with the contents from:: doc/en/announce/release-.rst To the following mailing lists: - * pytest-dev@python.org (all releases) - * python-announce-list@python.org (all releases) - * testing-in-python@lists.idyll.org (only major/minor releases) + * python-announce-list@python.org + + And announce it with the ``#pytest`` hashtag on: - And announce it on `Twitter `_ with the ``#pytest`` hashtag. + * `Bluesky `_ + * `Fosstodon `_ + * `Twitter/X `_ diff --git a/bench/empty.py b/bench/empty.py index 35abeef4140..346b79d5e33 100644 --- a/bench/empty.py +++ b/bench/empty.py @@ -2,4 +2,4 @@ for i in range(1000): - exec("def test_func_%d(): pass" % i) + exec(f"def test_func_{i}(): pass") diff --git a/changelog/11706.bugfix.rst b/changelog/11706.bugfix.rst deleted file mode 100644 index a86db5ef66a..00000000000 --- a/changelog/11706.bugfix.rst +++ /dev/null @@ -1,4 +0,0 @@ -Fix reporting of teardown errors in higher-scoped fixtures when using `--maxfail` or `--stepwise`. - -Originally added in pytest 8.0.0, but reverted in 8.0.2 due to a regression in pytest-xdist. -This regression was fixed in pytest-xdist 3.6.1. diff --git a/changelog/11771.contrib.rst b/changelog/11771.contrib.rst deleted file mode 100644 index a3c1ed1099e..00000000000 --- a/changelog/11771.contrib.rst +++ /dev/null @@ -1,5 +0,0 @@ -The PyPy runtime version has been updated to 3.9 from 3.8 that introduced -a flaky bug at the garbage collector which was not expected to fix there -as the V3.8 is EoL. - --- by :user:`x612skm` diff --git a/changelog/11797.bugfix.rst b/changelog/11797.bugfix.rst deleted file mode 100644 index 94b72da00fd..00000000000 --- a/changelog/11797.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`pytest.approx` now correctly handles :class:`Sequence `-like objects. diff --git a/changelog/12153.doc.rst b/changelog/12153.doc.rst deleted file mode 100644 index ac36becf9a7..00000000000 --- a/changelog/12153.doc.rst +++ /dev/null @@ -1 +0,0 @@ -Documented using :envvar:`PYTEST_VERSION` to detect if code is running from within a pytest run. diff --git a/changelog/12204.bugfix.rst b/changelog/12204.bugfix.rst deleted file mode 100644 index 099ad70610a..00000000000 --- a/changelog/12204.bugfix.rst +++ /dev/null @@ -1,11 +0,0 @@ -Fixed a regression in pytest 8.0 where tracebacks get longer and longer when multiple -tests fail due to a shared higher-scope fixture which raised -- by :user:`bluetech`. - -Also fixed a similar regression in pytest 5.4 for collectors which raise during setup. - -The fix necessitated internal changes which may affect some plugins: - -* ``FixtureDef.cached_result[2]`` is now a tuple ``(exc, tb)`` - instead of ``exc``. -* ``SetupState.stack`` failures are now a tuple ``(exc, tb)`` - instead of ``exc``. diff --git a/changelog/12231.feature.rst b/changelog/12231.feature.rst deleted file mode 100644 index dad04bc20c1..00000000000 --- a/changelog/12231.feature.rst +++ /dev/null @@ -1,11 +0,0 @@ -Added `--xfail-tb` flag, which turns on traceback output for XFAIL results. - -* If the `--xfail-tb` flag is not sent, tracebacks for XFAIL results are NOT shown. -* The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. -* Note: Even if you have `--xfail-tb` set, you won't see them if `--tb=no`. - -Some history: - -With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. - -This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. With this, the default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. While this is a behavior change, it brings default behavior back to pre-8.0.0 behavior, which ultimately was considered the better course of action. diff --git a/changelog/12264.bugfix.rst b/changelog/12264.bugfix.rst deleted file mode 120000 index e5704e6e819..00000000000 --- a/changelog/12264.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -12204.bugfix.rst \ No newline at end of file diff --git a/changelog/12275.bugfix.rst b/changelog/12275.bugfix.rst deleted file mode 100644 index 2d040a3a063..00000000000 --- a/changelog/12275.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix collection error upon encountering an :mod:`abstract ` class, including abstract `unittest.TestCase` subclasses. diff --git a/changelog/12281.feature.rst b/changelog/12281.feature.rst deleted file mode 100644 index c6e8e3b3098..00000000000 --- a/changelog/12281.feature.rst +++ /dev/null @@ -1,8 +0,0 @@ -Added support for keyword matching in marker expressions. - -Now tests can be selected by marker keyword arguments. -Supported values are :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None`. - -See :ref:`marker examples ` for more information. - --- by :user:`lovetheguitar` diff --git a/changelog/12328.bugfix.rst b/changelog/12328.bugfix.rst deleted file mode 100644 index f334425850b..00000000000 --- a/changelog/12328.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix a regression in pytest 8.0.0 where package-scoped parameterized items were not correctly reordered to minimize setups/teardowns in some cases. diff --git a/changelog/12424.bugfix.rst b/changelog/12424.bugfix.rst deleted file mode 100644 index 7ad1126858b..00000000000 --- a/changelog/12424.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fix crash with `assert testcase is not None` assertion failure when re-running unittest tests using plugins like pytest-rerunfailures. Regressed in 8.2.2. diff --git a/changelog/12467.improvement.rst b/changelog/12467.improvement.rst deleted file mode 100644 index b1e0581ed16..00000000000 --- a/changelog/12467.improvement.rst +++ /dev/null @@ -1,3 +0,0 @@ -Migrated all internal type-annotations to the python3.10+ style by using the `annotations` future import. - --- by :user:`RonnyPfannschmidt` diff --git a/changelog/12469.doc.rst b/changelog/12469.doc.rst deleted file mode 100644 index 2340315353c..00000000000 --- a/changelog/12469.doc.rst +++ /dev/null @@ -1,6 +0,0 @@ -The external plugin mentions in the documentation now avoid mentioning -:std:doc:`setuptools entry-points ` as the concept is -much more generic nowadays. Instead, the terminology of "external", -"installed", or "third-party" plugins (or packages) replaces that. - --- by :user:`webknjaz` diff --git a/changelog/12469.improvement.rst b/changelog/12469.improvement.rst deleted file mode 100644 index a90fb1e6610..00000000000 --- a/changelog/12469.improvement.rst +++ /dev/null @@ -1,4 +0,0 @@ -The console output now uses the "third-party plugins" terminology, -replacing the previously established but confusing and outdated -reference to :std:doc:`setuptools ` --- by :user:`webknjaz`. diff --git a/changelog/12472.bugfix.rst b/changelog/12472.bugfix.rst deleted file mode 100644 index f08e9d1f90b..00000000000 --- a/changelog/12472.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a crash when returning category ``"error"`` or ``"failed"`` with a custom test status from :hook:`pytest_report_teststatus` hook -- :user:`pbrezina`. diff --git a/changelog/12493.contrib.rst b/changelog/12493.contrib.rst deleted file mode 100644 index db3d045697e..00000000000 --- a/changelog/12493.contrib.rst +++ /dev/null @@ -1,13 +0,0 @@ -The change log draft preview integration has been refactored to use a -third party extension ``sphinxcontib-towncrier``. The previous in-repo -script was putting the change log preview file at -:file:`doc/en/_changelog_towncrier_draft.rst`. Said file is no longer -ignored in Git and might show up among untracked files in the -development environments of the contributors. To address that, the -contributors can run the following command that will clean it up: - -.. code-block:: console - - $ git clean -x -i -- doc/en/_changelog_towncrier_draft.rst - --- by :user:`webknjaz` diff --git a/changelog/12498.contrib.rst b/changelog/12498.contrib.rst deleted file mode 100644 index 436c6f0e9ed..00000000000 --- a/changelog/12498.contrib.rst +++ /dev/null @@ -1,5 +0,0 @@ -All the undocumented ``tox`` environments now have descriptions. -They can be listed in one's development environment by invoking -``tox -av`` in a terminal. - --- by :user:`webknjaz` diff --git a/changelog/12501.contrib.rst b/changelog/12501.contrib.rst deleted file mode 100644 index 6f434c287b3..00000000000 --- a/changelog/12501.contrib.rst +++ /dev/null @@ -1,11 +0,0 @@ -The changelog configuration has been updated to introduce more accurate -audience-tailored categories. Previously, there was a ``trivial`` -change log fragment type with an unclear and broad meaning. It was -removed and we now have ``contrib``, ``misc`` and ``packaging`` in -place of it. - -The new change note types target the readers who are downstream -packagers and project contributors. Additionally, the miscellaneous -section is kept for unspecified updates that do not fit anywhere else. - --- by :user:`webknjaz` diff --git a/changelog/12502.contrib.rst b/changelog/12502.contrib.rst deleted file mode 100644 index 940a2d7a120..00000000000 --- a/changelog/12502.contrib.rst +++ /dev/null @@ -1,7 +0,0 @@ -The UX of the GitHub automation making pull requests to update the -plugin list has been updated. Previously, the maintainers had to close -the automatically created pull requests and re-open them to trigger the -CI runs. From now on, they only need to click the `Ready for review` -button instead. - --- by :user:`webknjaz` diff --git a/changelog/12505.bugfix.rst b/changelog/12505.bugfix.rst deleted file mode 100644 index f55a8a17e4b..00000000000 --- a/changelog/12505.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Improve handling of invalid regex patterns in :func:`pytest.raises(match=r'...') ` by providing a clear error message. diff --git a/changelog/12522.contrib.rst b/changelog/12522.contrib.rst deleted file mode 100644 index dd994317165..00000000000 --- a/changelog/12522.contrib.rst +++ /dev/null @@ -1,4 +0,0 @@ -The ``:pull:`` RST role has been replaced with a shorter -``:pr:`` due to starting to use the implementation from -the third-party :pypi:`sphinx-issues` Sphinx extension --- by :user:`webknjaz`. diff --git a/changelog/12531.contrib.rst b/changelog/12531.contrib.rst deleted file mode 100644 index 12083fc320e..00000000000 --- a/changelog/12531.contrib.rst +++ /dev/null @@ -1,6 +0,0 @@ -The coverage reporting configuration has been updated to exclude -pytest's own tests marked as expected to fail from the coverage -report. This has an effect of reducing the influence of flaky -tests on the resulting number. - --- by :user:`webknjaz` diff --git a/changelog/12533.contrib.rst b/changelog/12533.contrib.rst deleted file mode 100644 index 3da7007a0fd..00000000000 --- a/changelog/12533.contrib.rst +++ /dev/null @@ -1,7 +0,0 @@ -The ``extlinks`` Sphinx extension is no longer enabled. The ``:bpo:`` -role it used to declare has been removed with that. BPO itself has -migrated to GitHub some years ago and it is possible to link the -respective issues by using their GitHub issue numbers and the -``:issue:`` role that the ``sphinx-issues`` extension implements. - --- by :user:`webknjaz` diff --git a/changelog/12544.improvement.rst b/changelog/12544.improvement.rst deleted file mode 100644 index 41125f5d939..00000000000 --- a/changelog/12544.improvement.rst +++ /dev/null @@ -1,3 +0,0 @@ -The ``_in_venv()`` function now detects Python virtual environments by -checking for a :file:`pyvenv.cfg` file, ensuring reliable detection on -various platforms -- by :user:`zachsnickers`. diff --git a/changelog/12545.improvement.rst b/changelog/12545.improvement.rst deleted file mode 120000 index 41a1e6bfa49..00000000000 --- a/changelog/12545.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -12544.improvement.rst \ No newline at end of file diff --git a/changelog/12557.contrib.rst b/changelog/12557.contrib.rst deleted file mode 120000 index c036c519093..00000000000 --- a/changelog/12557.contrib.rst +++ /dev/null @@ -1 +0,0 @@ -11771.contrib.rst \ No newline at end of file diff --git a/changelog/12562.contrib.rst b/changelog/12562.contrib.rst deleted file mode 100644 index 0d30495983a..00000000000 --- a/changelog/12562.contrib.rst +++ /dev/null @@ -1,2 +0,0 @@ -Possible typos in using the ``:user:`` RST role is now being linted -through the pre-commit tool integration -- by :user:`webknjaz`. diff --git a/changelog/12567.feature.rst b/changelog/12567.feature.rst deleted file mode 100644 index 3690d7aff68..00000000000 --- a/changelog/12567.feature.rst +++ /dev/null @@ -1,7 +0,0 @@ -Added ``--no-fold-skipped`` command line option - -If this option is set, then skipped tests in short summary are no longer grouped -by reason but all tests are printed individually with correct nodeid in the same -way as other statuses. - --- by :user:`pbrezina` diff --git a/changelog/12577.doc.rst b/changelog/12577.doc.rst deleted file mode 100644 index 0bd427e177d..00000000000 --- a/changelog/12577.doc.rst +++ /dev/null @@ -1,3 +0,0 @@ -`CI` and `BUILD_NUMBER` environment variables role is discribed in -the reference doc. They now also appears when doing `pytest -h` --- by :user:`MarcBresson`. diff --git a/changelog/12580.bugfix.rst b/changelog/12580.bugfix.rst deleted file mode 100644 index 9186ef1a4c9..00000000000 --- a/changelog/12580.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a crash when using the cache class on Windows and the cache directory was created concurrently. diff --git a/changelog/2871.improvement.rst b/changelog/2871.improvement.rst deleted file mode 100644 index 1ba399550c7..00000000000 --- a/changelog/2871.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -Do not truncate arguments to functions in output when running with `-vvv`. diff --git a/changelog/389.improvement.rst b/changelog/389.improvement.rst deleted file mode 100644 index f8e2c19fde0..00000000000 --- a/changelog/389.improvement.rst +++ /dev/null @@ -1,38 +0,0 @@ -The readability of assertion introspection of bound methods has been enhanced --- by :user:`farbodahm`, :user:`webknjaz`, :user:`obestwalter`, :user:`flub` -and :user:`glyphack`. - -Earlier, it was like: - -.. code-block:: console - - =================================== FAILURES =================================== - _____________________________________ test _____________________________________ - - def test(): - > assert Help().fun() == 2 - E assert 1 == 2 - E + where 1 = >() - E + where > = .fun - E + where = Help() - - example.py:7: AssertionError - =========================== 1 failed in 0.03 seconds =========================== - - -And now it's like: - -.. code-block:: console - - =================================== FAILURES =================================== - _____________________________________ test _____________________________________ - - def test(): - > assert Help().fun() == 2 - E assert 1 == 2 - E + where 1 = fun() - E + where fun = .fun - E + where = Help() - - test_local.py:13: AssertionError - =========================== 1 failed in 0.03 seconds =========================== diff --git a/changelog/6962.bugfix.rst b/changelog/6962.bugfix.rst deleted file mode 100644 index 030b6e06392..00000000000 --- a/changelog/6962.bugfix.rst +++ /dev/null @@ -1,2 +0,0 @@ -Parametrization parameters are now compared using `==` instead of `is` (`is` is still used as a fallback if the parameter does not support `==`). -This fixes use of parameters such as lists, which have a different `id` but compare equal, causing fixtures to be re-computed instead of being cached. diff --git a/changelog/7166.bugfix.rst b/changelog/7166.bugfix.rst deleted file mode 100644 index 98e6821f2ff..00000000000 --- a/changelog/7166.bugfix.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed progress percentages (the ``[ 87%]`` at the edge of the screen) sometimes not aligning correctly when running with pytest-xdist ``-n``. diff --git a/changelog/7662.improvement.rst b/changelog/7662.improvement.rst deleted file mode 100644 index b6ae1ba7e4c..00000000000 --- a/changelog/7662.improvement.rst +++ /dev/null @@ -1 +0,0 @@ -Added timezone information to the testsuite timestamp in the JUnit XML report. diff --git a/changelog/README.rst b/changelog/README.rst index fdaa573d427..f1ba2cbd0bd 100644 --- a/changelog/README.rst +++ b/changelog/README.rst @@ -16,12 +16,12 @@ Each file should be named like ``..rst``, where * ``feature``: new user facing features, like new command-line options and new behavior. * ``improvement``: improvement of existing functionality, usually without requiring user intervention (for example, new fields being written in ``--junit-xml``, improved colors in terminal, etc). * ``bugfix``: fixes a bug. -* ``doc``: documentation improvement, like rewording an entire session or adding missing docs. +* ``doc``: documentation improvement, like rewording an entire section or adding missing docs. * ``deprecation``: feature deprecation. * ``breaking``: a change which may break existing suites, such as feature removal or behavior change. * ``vendor``: changes in packages vendored in pytest. * ``packaging``: notes for downstreams about unobvious side effects - and tooling. changes in the test invocation considerations and + and tooling. Changes in the test invocation considerations and runtime assumptions. * ``contrib``: stuff that affects the contributor experience. e.g. Running tests, building the docs, setting up the development diff --git a/codecov.yml b/codecov.yml index 0841ab049ff..c37e5ec4a09 100644 --- a/codecov.yml +++ b/codecov.yml @@ -6,6 +6,8 @@ codecov: coverage: status: - patch: true + patch: + default: + target: 100% # require patches to be 100% project: false comment: false diff --git a/doc/en/announce/index.rst b/doc/en/announce/index.rst index c65eb5f3613..4a5e8b86544 100644 --- a/doc/en/announce/index.rst +++ b/doc/en/announce/index.rst @@ -6,6 +6,19 @@ Release announcements :maxdepth: 2 + release-9.0.3 + release-9.0.2 + release-9.0.1 + release-9.0.0 + release-8.4.2 + release-8.4.1 + release-8.4.0 + release-8.3.5 + release-8.3.4 + release-8.3.3 + release-8.3.2 + release-8.3.1 + release-8.3.0 release-8.2.2 release-8.2.1 release-8.2.0 diff --git a/doc/en/announce/release-2.3.0.rst b/doc/en/announce/release-2.3.0.rst index 6905b77b923..c405073ef40 100644 --- a/doc/en/announce/release-2.3.0.rst +++ b/doc/en/announce/release-2.3.0.rst @@ -6,10 +6,10 @@ and parametrized testing in Python. It is now easier, more efficient and more predictable to re-run the same tests with different fixture instances. Also, you can directly declare the caching "scope" of fixtures so that dependent tests throughout your whole test suite can -re-use database or other expensive fixture objects with ease. Lastly, +reuse database or other expensive fixture objects with ease. Lastly, it's possible for fixture functions (formerly known as funcarg factories) to use other fixtures, allowing for a completely modular and -re-usable fixture design. +reusable fixture design. For detailed info and tutorial-style examples, see: diff --git a/doc/en/announce/release-2.8.2.rst b/doc/en/announce/release-2.8.2.rst index e4726338852..f64ea9bb29a 100644 --- a/doc/en/announce/release-2.8.2.rst +++ b/doc/en/announce/release-2.8.2.rst @@ -17,7 +17,7 @@ Thanks to all who contributed to this release, among them: Bruno Oliveira Demian Brecht - Florian Bruhin + Freya Bruhin Ionel Cristian Mărieș Raphael Pierzina Ronny Pfannschmidt diff --git a/doc/en/announce/release-2.8.3.rst b/doc/en/announce/release-2.8.3.rst index 3f357252bb6..1ea7aac6d74 100644 --- a/doc/en/announce/release-2.8.3.rst +++ b/doc/en/announce/release-2.8.3.rst @@ -16,7 +16,7 @@ As usual, you can upgrade from pypi via:: Thanks to all who contributed to this release, among them: Bruno Oliveira - Florian Bruhin + Freya Bruhin Gabe Hollombe Gabriel Reis Hartmut Goebel diff --git a/doc/en/announce/release-2.8.4.rst b/doc/en/announce/release-2.8.4.rst index adbdecc87ea..0605c986928 100644 --- a/doc/en/announce/release-2.8.4.rst +++ b/doc/en/announce/release-2.8.4.rst @@ -16,7 +16,7 @@ As usual, you can upgrade from pypi via:: Thanks to all who contributed to this release, among them: Bruno Oliveira - Florian Bruhin + Freya Bruhin Jeff Widman Mehdy Khoshnoody Nicholas Chammas @@ -43,10 +43,10 @@ The py.test Development Team non-ascii characters. Thanks Bruno Oliveira for the PR. - fix #1204: another error when collecting with a nasty __getattr__(). - Thanks Florian Bruhin for the PR. + Thanks Freya Bruhin for the PR. - fix the summary printed when no tests did run. - Thanks Florian Bruhin for the PR. + Thanks Freya Bruhin for the PR. - a number of documentation modernizations wrt good practices. Thanks Bruno Oliveira for the PR. diff --git a/doc/en/announce/release-2.8.6.rst b/doc/en/announce/release-2.8.6.rst index 5d6565b16a3..a63c7f1e38d 100644 --- a/doc/en/announce/release-2.8.6.rst +++ b/doc/en/announce/release-2.8.6.rst @@ -18,7 +18,7 @@ Thanks to all who contributed to this release, among them: AMiT Kumar Bruno Oliveira Erik M. Bray - Florian Bruhin + Freya Bruhin Georgy Dyuldin Jeff Widman Kartik Singhal diff --git a/doc/en/announce/release-2.9.0.rst b/doc/en/announce/release-2.9.0.rst index 753bb7bf6f0..9477f0a9ba3 100644 --- a/doc/en/announce/release-2.9.0.rst +++ b/doc/en/announce/release-2.9.0.rst @@ -18,7 +18,7 @@ Thanks to all who contributed to this release, among them: Bruno Oliveira Buck Golemon David Vierra - Florian Bruhin + Freya Bruhin Galaczi Endre Georgy Dyuldin Lukas Bednar diff --git a/doc/en/announce/release-2.9.1.rst b/doc/en/announce/release-2.9.1.rst index 7a46d2ae690..3880218d233 100644 --- a/doc/en/announce/release-2.9.1.rst +++ b/doc/en/announce/release-2.9.1.rst @@ -17,7 +17,7 @@ Thanks to all who contributed to this release, among them: Bruno Oliveira Daniel Hahler Dmitry Malinovsky - Florian Bruhin + Freya Bruhin Floris Bruynooghe Matt Bachmann Ronny Pfannschmidt diff --git a/doc/en/announce/release-2.9.2.rst b/doc/en/announce/release-2.9.2.rst index 3e75af7fe69..3dc00b46729 100644 --- a/doc/en/announce/release-2.9.2.rst +++ b/doc/en/announce/release-2.9.2.rst @@ -17,7 +17,7 @@ Thanks to all who contributed to this release, among them: Adam Chainz Benjamin Dopplinger Bruno Oliveira - Florian Bruhin + Freya Bruhin John Towler Martin Prusse Meng Jue diff --git a/doc/en/announce/release-3.0.0.rst b/doc/en/announce/release-3.0.0.rst index 5de38911482..b201b901eb7 100644 --- a/doc/en/announce/release-3.0.0.rst +++ b/doc/en/announce/release-3.0.0.rst @@ -39,7 +39,7 @@ Thanks to all who contributed to this release, among them: Dmitry Dygalo Edoardo Batini Eli Boyarski - Florian Bruhin + Freya Bruhin Floris Bruynooghe Greg Price Guyzmo diff --git a/doc/en/announce/release-3.0.1.rst b/doc/en/announce/release-3.0.1.rst index 8f5cfe411aa..b36587f983a 100644 --- a/doc/en/announce/release-3.0.1.rst +++ b/doc/en/announce/release-3.0.1.rst @@ -17,7 +17,7 @@ Thanks to all who contributed to this release, among them: Bruno Oliveira Daniel Hahler Dmitry Dygalo - Florian Bruhin + Freya Bruhin Marcin Bachry Ronny Pfannschmidt matthiasha diff --git a/doc/en/announce/release-3.0.2.rst b/doc/en/announce/release-3.0.2.rst index 86ba82ca6e6..9b1f2acd60d 100644 --- a/doc/en/announce/release-3.0.2.rst +++ b/doc/en/announce/release-3.0.2.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Ahn Ki-Wook * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Jordan Guymon * Raphael Pierzina * Ronny Pfannschmidt diff --git a/doc/en/announce/release-3.0.3.rst b/doc/en/announce/release-3.0.3.rst index 89a2e0c744e..05bdf4dcd16 100644 --- a/doc/en/announce/release-3.0.3.rst +++ b/doc/en/announce/release-3.0.3.rst @@ -13,7 +13,7 @@ The changelog is available at http://doc.pytest.org/en/stable/changelog.html. Thanks to all who contributed to this release, among them: * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Floris Bruynooghe * Huayi Zhang * Lev Maximov diff --git a/doc/en/announce/release-3.0.4.rst b/doc/en/announce/release-3.0.4.rst index 72c2d29464d..ba37bba2111 100644 --- a/doc/en/announce/release-3.0.4.rst +++ b/doc/en/announce/release-3.0.4.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Bruno Oliveira * Dan Wandschneider -* Florian Bruhin +* Freya Bruhin * Georgy Dyuldin * Grigorii Eremeev * Jason R. Coombs diff --git a/doc/en/announce/release-3.0.7.rst b/doc/en/announce/release-3.0.7.rst index 4b7e075e76a..782910ae6a4 100644 --- a/doc/en/announce/release-3.0.7.rst +++ b/doc/en/announce/release-3.0.7.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Anthony Sottile * Barney Gale * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Floris Bruynooghe * Ionel Cristian Mărieș * Katerina Koukiou diff --git a/doc/en/announce/release-3.1.0.rst b/doc/en/announce/release-3.1.0.rst index 55277067948..454c04c6430 100644 --- a/doc/en/announce/release-3.1.0.rst +++ b/doc/en/announce/release-3.1.0.rst @@ -27,7 +27,7 @@ Thanks to all who contributed to this release, among them: * David Giese * David Szotten * Dmitri Pribysh -* Florian Bruhin +* Freya Bruhin * Florian Schulze * Floris Bruynooghe * John Towler diff --git a/doc/en/announce/release-3.1.1.rst b/doc/en/announce/release-3.1.1.rst index 135b2fe8443..99fb0d0f801 100644 --- a/doc/en/announce/release-3.1.1.rst +++ b/doc/en/announce/release-3.1.1.rst @@ -12,7 +12,7 @@ The full changelog is available at http://doc.pytest.org/en/stable/changelog.htm Thanks to all who contributed to this release, among them: * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Floris Bruynooghe * Jason R. Coombs * Ronny Pfannschmidt diff --git a/doc/en/announce/release-3.1.2.rst b/doc/en/announce/release-3.1.2.rst index a9b85c4715c..3e988b17e84 100644 --- a/doc/en/announce/release-3.1.2.rst +++ b/doc/en/announce/release-3.1.2.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Andreas Pelme * ApaDoctor * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Ronny Pfannschmidt * Segev Finer diff --git a/doc/en/announce/release-3.2.0.rst b/doc/en/announce/release-3.2.0.rst index edc66a28e78..68694493907 100644 --- a/doc/en/announce/release-3.2.0.rst +++ b/doc/en/announce/release-3.2.0.rst @@ -25,7 +25,7 @@ Thanks to all who contributed to this release, among them: * Andras Tim * Bruno Oliveira * Daniel Hahler -* Florian Bruhin +* Freya Bruhin * Floris Bruynooghe * John Still * Jordan Moldow diff --git a/doc/en/announce/release-3.2.1.rst b/doc/en/announce/release-3.2.1.rst index c40217d311d..a492390fa58 100644 --- a/doc/en/announce/release-3.2.1.rst +++ b/doc/en/announce/release-3.2.1.rst @@ -13,7 +13,7 @@ Thanks to all who contributed to this release, among them: * Alex Gaynor * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Ronny Pfannschmidt * Srinivas Reddy Thatiparthy diff --git a/doc/en/announce/release-3.2.4.rst b/doc/en/announce/release-3.2.4.rst index ff0b35781b1..9bde3afab3b 100644 --- a/doc/en/announce/release-3.2.4.rst +++ b/doc/en/announce/release-3.2.4.rst @@ -15,7 +15,7 @@ Thanks to all who contributed to this release, among them: * Christian Boelsen * Christoph Buchner * Daw-Ran Liou -* Florian Bruhin +* Freya Bruhin * Franck Michea * Leonard Lausen * Matty G diff --git a/doc/en/announce/release-3.3.0.rst b/doc/en/announce/release-3.3.0.rst index 1cbf2c448c8..d54910bea4c 100644 --- a/doc/en/announce/release-3.3.0.rst +++ b/doc/en/announce/release-3.3.0.rst @@ -27,7 +27,7 @@ Thanks to all who contributed to this release, among them: * Daniel Hahler * Dirk Thomas * Dmitry Malinovsky -* Florian Bruhin +* Freya Bruhin * George Y. Kussumoto * Hugo * Jesús Espino diff --git a/doc/en/announce/release-3.3.1.rst b/doc/en/announce/release-3.3.1.rst index 98b6fa6c1ba..a1a0a6d6f45 100644 --- a/doc/en/announce/release-3.3.1.rst +++ b/doc/en/announce/release-3.3.1.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Bruno Oliveira * Daniel Hahler * Eugene Prikazchikov -* Florian Bruhin +* Freya Bruhin * Roland Puntaier * Ronny Pfannschmidt * Sebastian Rahlf diff --git a/doc/en/announce/release-3.3.2.rst b/doc/en/announce/release-3.3.2.rst index 7a2577d1ff8..8c4110cc350 100644 --- a/doc/en/announce/release-3.3.2.rst +++ b/doc/en/announce/release-3.3.2.rst @@ -15,7 +15,7 @@ Thanks to all who contributed to this release, among them: * Antony Lee * Austin * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Floris Bruynooghe * Henk-Jaap Wagenaar * Jurko Gospodnetić diff --git a/doc/en/announce/release-3.4.0.rst b/doc/en/announce/release-3.4.0.rst index 6ab5b124a25..8a8582f7a00 100644 --- a/doc/en/announce/release-3.4.0.rst +++ b/doc/en/announce/release-3.4.0.rst @@ -30,7 +30,7 @@ Thanks to all who contributed to this release, among them: * Brian Maissy * Bruno Oliveira * Cyrus Maden -* Florian Bruhin +* Freya Bruhin * Henk-Jaap Wagenaar * Ian Lesperance * Jon Dufresne diff --git a/doc/en/announce/release-3.4.1.rst b/doc/en/announce/release-3.4.1.rst index d83949453a2..bef05752698 100644 --- a/doc/en/announce/release-3.4.1.rst +++ b/doc/en/announce/release-3.4.1.rst @@ -16,7 +16,7 @@ Thanks to all who contributed to this release, among them: * Andy Freeland * Brian Maissy * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Jason R. Coombs * Marcin Bachry * Pedro Algarvio diff --git a/doc/en/announce/release-3.4.2.rst b/doc/en/announce/release-3.4.2.rst index 07cd9d3a8ba..5ab73986617 100644 --- a/doc/en/announce/release-3.4.2.rst +++ b/doc/en/announce/release-3.4.2.rst @@ -13,7 +13,7 @@ Thanks to all who contributed to this release, among them: * Allan Feldman * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Jason R. Coombs * Kyle Altendorf * Maik Figura diff --git a/doc/en/announce/release-3.5.0.rst b/doc/en/announce/release-3.5.0.rst index 6bc2f3cd0cb..7ce2fe3dfe0 100644 --- a/doc/en/announce/release-3.5.0.rst +++ b/doc/en/announce/release-3.5.0.rst @@ -26,7 +26,7 @@ Thanks to all who contributed to this release, among them: * Bruno Oliveira * Carlos Jenkins * Daniel Hahler -* Florian Bruhin +* Freya Bruhin * Jason R. Coombs * Jeffrey Rackauckas * Jordan Speicher diff --git a/doc/en/announce/release-5.0.0.rst b/doc/en/announce/release-5.0.0.rst index f5e593e9d88..166d4e565c3 100644 --- a/doc/en/announce/release-5.0.0.rst +++ b/doc/en/announce/release-5.0.0.rst @@ -26,7 +26,7 @@ Thanks to all who contributed to this release, among them: * Daniel Hahler * Dirk Thomas * Evan Kepner -* Florian Bruhin +* Freya Bruhin * Hugo * Kevin J. Foley * Pulkit Goyal diff --git a/doc/en/announce/release-5.0.1.rst b/doc/en/announce/release-5.0.1.rst index e16a8f716f1..f0ffb791545 100644 --- a/doc/en/announce/release-5.0.1.rst +++ b/doc/en/announce/release-5.0.1.rst @@ -15,7 +15,7 @@ Thanks to all who contributed to this release, among them: * Andreu Vallbona Plazas * Anthony Sottile * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Michael Moore * Niklas Meinzer * Thomas Grainger diff --git a/doc/en/announce/release-5.1.0.rst b/doc/en/announce/release-5.1.0.rst index 9ab54ff9730..6170023604a 100644 --- a/doc/en/announce/release-5.1.0.rst +++ b/doc/en/announce/release-5.1.0.rst @@ -27,7 +27,7 @@ Thanks to all who contributed to this release, among them: * Bruno Oliveira * Daniel Hahler * David Röthlisberger -* Florian Bruhin +* Freya Bruhin * Ilya Stepin * Jon Dufresne * Kaiqi diff --git a/doc/en/announce/release-5.1.1.rst b/doc/en/announce/release-5.1.1.rst index bb8de48014a..1262e94fd00 100644 --- a/doc/en/announce/release-5.1.1.rst +++ b/doc/en/announce/release-5.1.1.rst @@ -14,7 +14,7 @@ Thanks to all who contributed to this release, among them: * Anthony Sottile * Bruno Oliveira * Daniel Hahler -* Florian Bruhin +* Freya Bruhin * Hugo van Kemenade * Ran Benita * Ronny Pfannschmidt diff --git a/doc/en/announce/release-5.2.1.rst b/doc/en/announce/release-5.2.1.rst index fe42b9bf15f..904e1b59893 100644 --- a/doc/en/announce/release-5.2.1.rst +++ b/doc/en/announce/release-5.2.1.rst @@ -13,7 +13,7 @@ Thanks to all who contributed to this release, among them: * Anthony Sottile * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Hynek Schlawack * Kevin J. Foley * tadashigaki diff --git a/doc/en/announce/release-5.2.2.rst b/doc/en/announce/release-5.2.2.rst index 89fd6a534d4..015baba52e7 100644 --- a/doc/en/announce/release-5.2.2.rst +++ b/doc/en/announce/release-5.2.2.rst @@ -16,7 +16,7 @@ Thanks to all who contributed to this release, among them: * Anthony Sottile * Bruno Oliveira * Daniel Hahler -* Florian Bruhin +* Freya Bruhin * Nattaphoom Chaipreecha * Oliver Bestwalter * Philipp Loose diff --git a/doc/en/announce/release-5.2.3.rst b/doc/en/announce/release-5.2.3.rst index bab174495d9..8c89e04540a 100644 --- a/doc/en/announce/release-5.2.3.rst +++ b/doc/en/announce/release-5.2.3.rst @@ -17,7 +17,7 @@ Thanks to all who contributed to this release, among them: * Daniel Hahler * Daniil Galiev * David Szotten -* Florian Bruhin +* Freya Bruhin * Patrick Harmon * Ran Benita * Zac Hatfield-Dodds diff --git a/doc/en/announce/release-5.3.1.rst b/doc/en/announce/release-5.3.1.rst index d575bb70e3f..5dc82ab7d88 100644 --- a/doc/en/announce/release-5.3.1.rst +++ b/doc/en/announce/release-5.3.1.rst @@ -15,7 +15,7 @@ Thanks to all who contributed to this release, among them: * Bruno Oliveira * Daniel Hahler * Felix Yan -* Florian Bruhin +* Freya Bruhin * Mark Dickinson * Nikolay Kondratyev * Steffen Schroeder diff --git a/doc/en/announce/release-6.0.0rc1.rst b/doc/en/announce/release-6.0.0rc1.rst index 5690b514baf..6f0a745cd00 100644 --- a/doc/en/announce/release-6.0.0rc1.rst +++ b/doc/en/announce/release-6.0.0rc1.rst @@ -25,7 +25,7 @@ Thanks to all who contributed to this release, among them: * David Diaz Barquero * Fabio Zadrozny * Felix Nieuwenhuizen -* Florian Bruhin +* Freya Bruhin * Florian Dahlitz * Gleb Nikonorov * Hugo van Kemenade diff --git a/doc/en/announce/release-6.1.0.rst b/doc/en/announce/release-6.1.0.rst index f4b571ae846..0c787d0bd15 100644 --- a/doc/en/announce/release-6.1.0.rst +++ b/doc/en/announce/release-6.1.0.rst @@ -23,7 +23,7 @@ Thanks to all of the contributors to this release: * C. Titus Brown * Drew Devereux * Faris A Chugthai -* Florian Bruhin +* Freya Bruhin * Hugo van Kemenade * Hynek Schlawack * Joseph Lucas diff --git a/doc/en/announce/release-6.2.0.rst b/doc/en/announce/release-6.2.0.rst index af16b830ddd..8e99d8fcda5 100644 --- a/doc/en/announce/release-6.2.0.rst +++ b/doc/en/announce/release-6.2.0.rst @@ -30,7 +30,7 @@ Thanks to all of the contributors to this release: * Cserna Zsolt * Dominic Mortlock * Emiel van de Laar -* Florian Bruhin +* Freya Bruhin * Garvit Shubham * Gustavo Camargo * Hugo Martins diff --git a/doc/en/announce/release-6.2.4.rst b/doc/en/announce/release-6.2.4.rst index fa2e3e78132..129368e73cd 100644 --- a/doc/en/announce/release-6.2.4.rst +++ b/doc/en/announce/release-6.2.4.rst @@ -14,7 +14,7 @@ Thanks to all of the contributors to this release: * Anthony Sottile * Bruno Oliveira * Christian Maurer -* Florian Bruhin +* Freya Bruhin * Ran Benita diff --git a/doc/en/announce/release-6.2.5.rst b/doc/en/announce/release-6.2.5.rst index bc6b4cf4222..daf9731c800 100644 --- a/doc/en/announce/release-6.2.5.rst +++ b/doc/en/announce/release-6.2.5.rst @@ -15,7 +15,7 @@ Thanks to all of the contributors to this release: * Bruno Oliveira * Brylie Christopher Oxley * Daniel Asztalos -* Florian Bruhin +* Freya Bruhin * Jason Haugen * MapleCCC * Michał Górny diff --git a/doc/en/announce/release-7.0.0.rst b/doc/en/announce/release-7.0.0.rst index 3ce4335564f..934064df745 100644 --- a/doc/en/announce/release-7.0.0.rst +++ b/doc/en/announce/release-7.0.0.rst @@ -34,7 +34,7 @@ Thanks to all of the contributors to this release: * Emmanuel Arias * Emmanuel Meric de Bellefon * Eric Liu -* Florian Bruhin +* Freya Bruhin * GergelyKalmar * Graeme Smecher * Harshna diff --git a/doc/en/announce/release-7.0.0rc1.rst b/doc/en/announce/release-7.0.0rc1.rst index a5bf0ed3c44..dd6ecdd131b 100644 --- a/doc/en/announce/release-7.0.0rc1.rst +++ b/doc/en/announce/release-7.0.0rc1.rst @@ -38,7 +38,7 @@ Thanks to all the contributors to this release: * Emmanuel Arias * Emmanuel Meric de Bellefon * Eric Liu -* Florian Bruhin +* Freya Bruhin * GergelyKalmar * Graeme Smecher * Harshna diff --git a/doc/en/announce/release-7.1.0.rst b/doc/en/announce/release-7.1.0.rst index 3361e1c8a32..f138524c564 100644 --- a/doc/en/announce/release-7.1.0.rst +++ b/doc/en/announce/release-7.1.0.rst @@ -28,7 +28,7 @@ Thanks to all of the contributors to this release: * Elijah DeLee * Emmanuel Arias * Fabian Egli -* Florian Bruhin +* Freya Bruhin * Gabor Szabo * Hasan Ramezani * Hugo van Kemenade diff --git a/doc/en/announce/release-7.2.0.rst b/doc/en/announce/release-7.2.0.rst index eca84aeb669..44cd553ec0f 100644 --- a/doc/en/announce/release-7.2.0.rst +++ b/doc/en/announce/release-7.2.0.rst @@ -33,7 +33,7 @@ Thanks to all of the contributors to this release: * EmptyRabbit * Ezio Melotti * Florian Best -* Florian Bruhin +* Freya Bruhin * Fredrik Berndtsson * Gabriel Landau * Gergely Kalmár diff --git a/doc/en/announce/release-7.3.0.rst b/doc/en/announce/release-7.3.0.rst index 33258dabade..b6d8379d5b5 100644 --- a/doc/en/announce/release-7.3.0.rst +++ b/doc/en/announce/release-7.3.0.rst @@ -42,7 +42,7 @@ Thanks to all of the contributors to this release: * Ezio Melotti * Felix Hofstätter * Florian Best -* Florian Bruhin +* Freya Bruhin * Fredrik Berndtsson * Gabriel Landau * Garvit Shubham diff --git a/doc/en/announce/release-7.4.0.rst b/doc/en/announce/release-7.4.0.rst index 5a0d18267d3..fef2ad6cb3d 100644 --- a/doc/en/announce/release-7.4.0.rst +++ b/doc/en/announce/release-7.4.0.rst @@ -27,7 +27,7 @@ Thanks to all of the contributors to this release: * Bryan Ricker * Chris Mahoney * Facundo Batista -* Florian Bruhin +* Freya Bruhin * Jarrett Keifer * Kenny Y * Miro Hrončok diff --git a/doc/en/announce/release-7.4.1.rst b/doc/en/announce/release-7.4.1.rst index efadcf919e8..4e22d3ead66 100644 --- a/doc/en/announce/release-7.4.1.rst +++ b/doc/en/announce/release-7.4.1.rst @@ -12,7 +12,7 @@ The full changelog is available at https://docs.pytest.org/en/stable/changelog.h Thanks to all of the contributors to this release: * Bruno Oliveira -* Florian Bruhin +* Freya Bruhin * Ran Benita diff --git a/doc/en/announce/release-8.0.0rc1.rst b/doc/en/announce/release-8.0.0rc1.rst index 547c8cbc53b..0cbfc3dad59 100644 --- a/doc/en/announce/release-8.0.0rc1.rst +++ b/doc/en/announce/release-8.0.0rc1.rst @@ -31,7 +31,7 @@ Thanks to all of the contributors to this release: * Christoph Anton Mitterer * DetachHead * Erik Hasse -* Florian Bruhin +* Freya Bruhin * Fraser Stark * Ha Pam * Hugo van Kemenade diff --git a/doc/en/announce/release-8.1.0.rst b/doc/en/announce/release-8.1.0.rst index 62cafdd78bb..6762bd412fe 100644 --- a/doc/en/announce/release-8.1.0.rst +++ b/doc/en/announce/release-8.1.0.rst @@ -28,7 +28,7 @@ Thanks to all of the contributors to this release: * Eric Larson * Fabian Sturm * Faisal Fawad -* Florian Bruhin +* Freya Bruhin * Franck Charras * Joachim B Haga * John Litborn diff --git a/doc/en/announce/release-8.2.0.rst b/doc/en/announce/release-8.2.0.rst index 2a63c8d8722..7aba492d7da 100644 --- a/doc/en/announce/release-8.2.0.rst +++ b/doc/en/announce/release-8.2.0.rst @@ -20,7 +20,7 @@ Thanks to all of the contributors to this release: * Bruno Oliveira * Daniel Miller -* Florian Bruhin +* Freya Bruhin * HolyMagician03-UMich * John Litborn * Levon Saldamli diff --git a/doc/en/announce/release-8.3.0.rst b/doc/en/announce/release-8.3.0.rst new file mode 100644 index 00000000000..0589aedfa89 --- /dev/null +++ b/doc/en/announce/release-8.3.0.rst @@ -0,0 +1,60 @@ +pytest-8.3.0 +======================================= + +The pytest team is proud to announce the 8.3.0 release! + +This release contains new features, improvements, and bug fixes, +the full list of changes is available in the changelog: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* Anita Hammer +* Ben Brown +* Brian Okken +* Bruno Oliveira +* Cornelius Riemenschneider +* Farbod Ahmadian +* Freya Bruhin +* Hynek Schlawack +* James Frost +* Jason R. Coombs +* Jelle Zijlstra +* Josh Soref +* Marc Bresson +* Michael Vogt +* Nathan Goldbaum +* Nicolas Simonds +* Oliver Bestwalter +* Pavel Březina +* Pierre Sassoulas +* Pradyun Gedam +* Ran Benita +* Ronny Pfannschmidt +* SOUBHIK KUMAR MITRA +* Sam Jirovec +* Stavros Ntentos +* Sviatoslav Sydorenko +* Sviatoslav Sydorenko (Святослав Сидоренко) +* Tomasz Kłoczko +* Virendra Patil +* Yutian Li +* Zach Snicker +* dj +* holger krekel +* joseph-sentry +* lovetheguitar +* neutraljump + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.3.1.rst b/doc/en/announce/release-8.3.1.rst new file mode 100644 index 00000000000..0fb9b40d9c7 --- /dev/null +++ b/doc/en/announce/release-8.3.1.rst @@ -0,0 +1,19 @@ +pytest-8.3.1 +======================================= + +pytest 8.3.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Ran Benita + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.3.2.rst b/doc/en/announce/release-8.3.2.rst new file mode 100644 index 00000000000..1e4a071692c --- /dev/null +++ b/doc/en/announce/release-8.3.2.rst @@ -0,0 +1,19 @@ +pytest-8.3.2 +======================================= + +pytest 8.3.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Ran Benita +* Ronny Pfannschmidt + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.3.3.rst b/doc/en/announce/release-8.3.3.rst new file mode 100644 index 00000000000..6e73714d4f9 --- /dev/null +++ b/doc/en/announce/release-8.3.3.rst @@ -0,0 +1,31 @@ +pytest-8.3.3 +======================================= + +pytest 8.3.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Anthony Sottile +* Avasam +* Bruno Oliveira +* Christian Clauss +* Eugene Mwangi +* Freya Bruhin +* GTowers1 +* Nauman Ahmed +* Pierre Sassoulas +* Reagan Lee +* Ronny Pfannschmidt +* Stefaan Lippens +* Sviatoslav Sydorenko (Святослав Сидоренко) +* dongfangtianyu + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.3.4.rst b/doc/en/announce/release-8.3.4.rst new file mode 100644 index 00000000000..3ec21d73f5e --- /dev/null +++ b/doc/en/announce/release-8.3.4.rst @@ -0,0 +1,30 @@ +pytest-8.3.4 +======================================= + +pytest 8.3.4 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. To upgrade:: + + pip install --upgrade pytest + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Freya Bruhin +* Frank Hoffmann +* Jakob van Santen +* Leonardus Chen +* Pierre Sassoulas +* Pradeep Kumar +* Ran Benita +* Serge Smertin +* Stefaan Lippens +* Sviatoslav Sydorenko (Святослав Сидоренко) +* dongfangtianyu +* suspe + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.3.5.rst b/doc/en/announce/release-8.3.5.rst new file mode 100644 index 00000000000..21bae869180 --- /dev/null +++ b/doc/en/announce/release-8.3.5.rst @@ -0,0 +1,26 @@ +pytest-8.3.5 +======================================= + +pytest 8.3.5 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Freya Bruhin +* John Litborn +* Kenny Y +* Ran Benita +* Sadra Barikbin +* Vincent (Wen Yu) Ge +* delta87 +* dongfangtianyu +* mwychung +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.4.0.rst b/doc/en/announce/release-8.4.0.rst new file mode 100644 index 00000000000..f492d45070a --- /dev/null +++ b/doc/en/announce/release-8.4.0.rst @@ -0,0 +1,106 @@ +pytest-8.4.0 +======================================= + +The pytest team is proud to announce the 8.4.0 release! + +This release contains new features, improvements, and bug fixes, +the full list of changes is available in the changelog: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* Adam Johnson +* Ammar Askar +* Andrew Pikul +* Andy Freeland +* Anthony Sottile +* Anton Zhilin +* Arpit Gupta +* Ashley Whetter +* Avasam +* Bahram Farahmand +* Brigitta Sipőcz +* Bruno Oliveira +* Callum Scott +* Christian Clauss +* Christopher Head +* Daara +* Daniel Miller +* Deysha Rivera +* Emil Hjelm +* Eugene Mwangi +* Freya Bruhin +* Frank Hoffmann +* GTowers1 +* Guillaume Gauvrit +* Gupta Arpit +* Harmin Parra Rueda +* Jakob van Santen +* Jason N. White +* Jiajun Xu +* John Litborn +* Julian Valentin +* JulianJvn +* Kenny Y +* Leonardus Chen +* Marcelo Duarte Trevisani +* Marcin Augustynów +* Natalia Mokeeva +* Nathan Rousseau +* Nauman Ahmed +* Nick Murphy +* Oleksandr Zavertniev +* Pavel Zhukov +* Peter Gessler +* Pierre Sassoulas +* Pradeep Kumar +* Ran Benita +* Reagan Lee +* Rob Arrow +* Ronny Pfannschmidt +* Sadra Barikbin +* Sam Bull +* Samuel Bronson +* Sashko +* Serge Smertin +* Shaygan Hooshyari +* Stefaan Lippens +* Stefan Zimmermann +* Stephen McDowell +* Sviatoslav Sydorenko +* Sviatoslav Sydorenko (Святослав Сидоренко) +* Thomas Grainger +* TobiMcNamobi +* Tobias Alex-Petersen +* Tony Narlock +* Vincent (Wen Yu) Ge +* Virendra Patil +* Will Riley +* Yann Dirson +* Zac Hatfield-Dodds +* delta87 +* dongfangtianyu +* eitanwass +* fazeelghafoor +* ikappaki +* jakkdl +* maugu +* moajo +* mwychung +* polkapolka +* suspe +* sven +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.4.1.rst b/doc/en/announce/release-8.4.1.rst new file mode 100644 index 00000000000..07ee26187a7 --- /dev/null +++ b/doc/en/announce/release-8.4.1.rst @@ -0,0 +1,21 @@ +pytest-8.4.1 +======================================= + +pytest 8.4.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Iwithyou2025 +* John Litborn +* Martin Fischer +* Ran Benita +* SarahPythonista + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-8.4.2.rst b/doc/en/announce/release-8.4.2.rst new file mode 100644 index 00000000000..3111e85bd0f --- /dev/null +++ b/doc/en/announce/release-8.4.2.rst @@ -0,0 +1,27 @@ +pytest-8.4.2 +======================================= + +pytest 8.4.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* AD +* Aditi De +* Bruno Oliveira +* Freya Bruhin +* John Litborn +* Liam DeVoe +* Marc Mueller +* NayeemJohn +* Olivier Grisel +* Ran Benita +* bengartner +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-9.0.0.rst b/doc/en/announce/release-9.0.0.rst new file mode 100644 index 00000000000..67d4f95a56d --- /dev/null +++ b/doc/en/announce/release-9.0.0.rst @@ -0,0 +1,69 @@ +pytest-9.0.0 +======================================= + +The pytest team is proud to announce the 9.0.0 release! + +This release contains new features, improvements, bug fixes, and breaking changes, so users +are encouraged to take a look at the CHANGELOG carefully: + + https://docs.pytest.org/en/stable/changelog.html + +For complete documentation, please visit: + + https://docs.pytest.org/en/stable/ + +As usual, you can upgrade from PyPI via: + + pip install -U pytest + +Thanks to all of the contributors to this release: + +* AD +* Aditi De +* Ali Nazzal +* Bruno Oliveira +* Charles-Meldhine Madi Mnemoi +* Clément Robert +* CoretexShadow +* Cornelius Roemer +* Eero Vaher +* Freya Bruhin +* Harsha Sai +* Hossein +* Israël Hallé +* Iwithyou2025 +* James Addison +* John Litborn +* Jordan Macdonald +* Kieran Ryan +* Liam DeVoe +* Marc Mueller +* Marcos Boger +* Michał Górny +* Mulat Mekonen +* NayeemJohn +* Olivier Grisel +* Omri Golan +* Pierre Sassoulas +* Praise Tompane +* Ran Benita +* Reilly Brogan +* Samuel Gaist +* SarahPythonista +* Sorin Sbarnea +* Stu-ops +* Tanuj Rai +* bengartner +* dariomesic +* jakkdl +* karlicoss +* popododo0720 +* sazsu +* slackline +* vyuroshchin +* zapl +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-9.0.1.rst b/doc/en/announce/release-9.0.1.rst new file mode 100644 index 00000000000..46af130e03c --- /dev/null +++ b/doc/en/announce/release-9.0.1.rst @@ -0,0 +1,18 @@ +pytest-9.0.1 +======================================= + +pytest 9.0.1 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Bruno Oliveira +* Ran Benita +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-9.0.2.rst b/doc/en/announce/release-9.0.2.rst new file mode 100644 index 00000000000..f184e1aa4b2 --- /dev/null +++ b/doc/en/announce/release-9.0.2.rst @@ -0,0 +1,22 @@ +pytest-9.0.2 +======================================= + +pytest 9.0.2 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Alex Waygood +* Bruno Oliveira +* Fazeel Usmani +* Freya Bruhin +* Ran Benita +* Tom Most +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/announce/release-9.0.3.rst b/doc/en/announce/release-9.0.3.rst new file mode 100644 index 00000000000..c9540218764 --- /dev/null +++ b/doc/en/announce/release-9.0.3.rst @@ -0,0 +1,38 @@ +pytest-9.0.3 +======================================= + +pytest 9.0.3 has just been released to PyPI. + +This is a bug-fix release, being a drop-in replacement. + +The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. + +Thanks to all of the contributors to this release: + +* Aditya Giri +* Alejandro Villate +* Bruno Oliveira +* Bubble-Interface +* Charles-Meldhine Madi Mnemoi +* DavidAG +* Denis Cherednichenko +* Dr Alex Mitre +* Freya +* Freya Bruhin +* Hugo van Kemenade +* John Litborn +* Liam DeVoe +* Lily Wu +* Maxime Grenu +* Ran Benita +* Randy Döring +* Ronald Eddy Jr +* Samuel Newbold +* Tejas Verma +* Vladimir +* jxramos +* 🇺🇦 Sviatoslav Sydorenko (Святослав Сидоренко) + + +Happy testing, +The pytest Development Team diff --git a/doc/en/backwards-compatibility.rst b/doc/en/backwards-compatibility.rst index c0feb833ce1..a7ee2253d67 100644 --- a/doc/en/backwards-compatibility.rst +++ b/doc/en/backwards-compatibility.rst @@ -53,14 +53,14 @@ History ========= -Focus primary on smooth transition - stance (pre 6.0) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Focus primarily on smooth transition - stance (pre 6.0) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Keeping backwards compatibility has a very high priority in the pytest project. Although we have deprecated functionality over the years, most of it is still supported. All deprecations in pytest were done because simpler or more efficient ways of accomplishing the same tasks have emerged, making the old way of doing things unnecessary. With the pytest 3.0 release, we introduced a clear communication scheme for when we will actually remove the old busted joint and politely ask you to use the new hotness instead, while giving you enough time to adjust your tests or raise concerns if there are valid reasons to keep deprecated functionality around. -To communicate changes, we issue deprecation warnings using a custom warning hierarchy (see :ref:`internal-warnings`). These warnings may be suppressed using the standard means: ``-W`` command-line flag or ``filterwarnings`` ini options (see :ref:`warnings`), but we suggest to use these sparingly and temporarily, and heed the warnings when possible. +To communicate changes, we issue deprecation warnings using a custom warning hierarchy (see :ref:`internal-warnings`). These warnings may be suppressed using the standard means: :option:`-W` command-line flag or :confval:`filterwarnings` configuration option (see :ref:`warnings`), but we suggest to use these sparingly and temporarily, and heed the warnings when possible. We will only start the removal of deprecated functionality in major releases (e.g. if we deprecate something in 3.0, we will start to remove it in 4.0), and keep it around for at least two minor releases (e.g. if we deprecate something in 3.9 and 4.0 is the next release, we start to remove it in 5.0, not in 4.0). @@ -83,8 +83,10 @@ Released pytest versions support all Python versions that are actively maintaine ============== =================== pytest version min. Python version ============== =================== -8.0+ 3.8+ -7.1+ 3.7+ +9.0+ 3.10+ +8.4 3.9+ +8.0 - 8.3 3.8+ +7.1 - 7.4 3.7+ 6.2 - 7.0 3.6+ 5.0 - 6.1 3.5+ 3.3 - 4.6 2.7, 3.4+ diff --git a/doc/en/broken-dep-constraints.txt b/doc/en/broken-dep-constraints.txt new file mode 100644 index 00000000000..1488e06fa23 --- /dev/null +++ b/doc/en/broken-dep-constraints.txt @@ -0,0 +1,2 @@ +# This file contains transitive dependencies that need to be pinned for some reason. +# Eventually this file will be empty, but in this case keep it around for future use. diff --git a/doc/en/builtin.rst b/doc/en/builtin.rst index 8dfffb0828a..9d38b329454 100644 --- a/doc/en/builtin.rst +++ b/doc/en/builtin.rst @@ -12,17 +12,17 @@ For information on plugin hooks and objects, see :ref:`plugins`. For information on the ``pytest.mark`` mechanism, see :ref:`mark`. -For information about fixtures, see :ref:`fixtures`. To see a complete list of available fixtures (add ``-v`` to also see fixtures with leading ``_``), type : +For information about fixtures, see :ref:`fixtures`. To see a complete list of available fixtures (add :option:`-v` to also see fixtures with leading ``_``), type : .. code-block:: pytest $ pytest --fixtures -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collected 0 items - cache -- .../_pytest/cacheprovider.py:560 + cache -- .../_pytest/cacheprovider.py:566 Return a cache object that can persist state between testing sessions. cache.get(key, default) @@ -33,7 +33,48 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a Values can be any object handled by the json stdlib module. - capsysbinary -- .../_pytest/capture.py:1003 + capsys -- .../_pytest/capture.py:1000 + Enable text capturing of writes to ``sys.stdout`` and ``sys.stderr``. + + The captured output is made available via ``capsys.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + + Returns an instance of :class:`CaptureFixture[str] `. + + Example: + + .. code-block:: python + + def test_output(capsys): + print("hello") + captured = capsys.readouterr() + assert captured.out == "hello\n" + + capteesys -- .../_pytest/capture.py:1028 + Enable simultaneous text capturing and pass-through of writes + to ``sys.stdout`` and ``sys.stderr`` as defined by ``--capture=``. + + + The captured output is made available via ``capteesys.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + + The output is also passed-through, allowing it to be "live-printed", + reported, or both as defined by ``--capture=``. + + Returns an instance of :class:`CaptureFixture[str] `. + + Example: + + .. code-block:: python + + def test_output(capteesys): + print("hello") + captured = capteesys.readouterr() + assert captured.out == "hello\n" + + capsysbinary -- .../_pytest/capture.py:1063 Enable bytes capturing of writes to ``sys.stdout`` and ``sys.stderr``. The captured output is made available via ``capsysbinary.readouterr()`` @@ -43,6 +84,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a Returns an instance of :class:`CaptureFixture[bytes] `. Example: + .. code-block:: python def test_output(capsysbinary): @@ -50,7 +92,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a captured = capsysbinary.readouterr() assert captured.out == b"hello\n" - capfd -- .../_pytest/capture.py:1030 + capfd -- .../_pytest/capture.py:1091 Enable text capturing of writes to file descriptors ``1`` and ``2``. The captured output is made available via ``capfd.readouterr()`` method @@ -60,6 +102,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a Returns an instance of :class:`CaptureFixture[str] `. Example: + .. code-block:: python def test_system_echo(capfd): @@ -67,7 +110,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a captured = capfd.readouterr() assert captured.out == "hello\n" - capfdbinary -- .../_pytest/capture.py:1057 + capfdbinary -- .../_pytest/capture.py:1119 Enable bytes capturing of writes to file descriptors ``1`` and ``2``. The captured output is made available via ``capfd.readouterr()`` method @@ -77,6 +120,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a Returns an instance of :class:`CaptureFixture[bytes] `. Example: + .. code-block:: python def test_system_echo(capfdbinary): @@ -84,24 +128,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a captured = capfdbinary.readouterr() assert captured.out == b"hello\n" - capsys -- .../_pytest/capture.py:976 - Enable text capturing of writes to ``sys.stdout`` and ``sys.stderr``. - - The captured output is made available via ``capsys.readouterr()`` method - calls, which return a ``(out, err)`` namedtuple. - ``out`` and ``err`` will be ``text`` objects. - - Returns an instance of :class:`CaptureFixture[str] `. - - Example: - .. code-block:: python - - def test_output(capsys): - print("hello") - captured = capsys.readouterr() - assert captured.out == "hello\n" - - doctest_namespace [session scope] -- .../_pytest/doctest.py:738 + doctest_namespace [session scope] -- .../_pytest/doctest.py:722 Fixture that returns a :py:class:`dict` that will be injected into the namespace of doctests. @@ -115,17 +142,17 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a For more details: :ref:`doctest_namespace`. - pytestconfig [session scope] -- .../_pytest/fixtures.py:1338 + pytestconfig [session scope] -- .../_pytest/fixtures.py:1431 Session-scoped fixture that returns the session's :class:`pytest.Config` object. Example:: def test_foo(pytestconfig): - if pytestconfig.getoption("verbose") > 0: + if pytestconfig.get_verbosity() > 0: ... - record_property -- .../_pytest/junitxml.py:284 + record_property -- .../_pytest/junitxml.py:277 Add extra properties to the calling test. User properties become part of the test report and are available to the @@ -139,13 +166,13 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a def test_function(record_property): record_property("example_key", 1) - record_xml_attribute -- .../_pytest/junitxml.py:307 + record_xml_attribute -- .../_pytest/junitxml.py:300 Add extra xml attributes to the tag for the calling test. The fixture is callable with ``name, value``. The value is automatically XML-encoded. - record_testsuite_property [session scope] -- .../_pytest/junitxml.py:345 + record_testsuite_property [session scope] -- .../_pytest/junitxml.py:338 Record a new ```` tag as child of the root ````. This is suitable to writing global information regarding the entire test @@ -170,20 +197,15 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a `pytest-xdist `__ plugin. See :issue:`7767` for details. - tmpdir_factory [session scope] -- .../_pytest/legacypath.py:303 + tmpdir_factory [session scope] -- .../_pytest/legacypath.py:298 Return a :class:`pytest.TempdirFactory` instance for the test session. - tmpdir -- .../_pytest/legacypath.py:310 - Return a temporary directory path object which is unique to each test - function invocation, created as a sub directory of the base temporary - directory. - - By default, a new base temporary directory is created each test session, - and old bases are removed after 3 sessions, to aid in debugging. If - ``--basetemp`` is used then it is cleared each session. See - :ref:`temporary directory location and retention`. - - The returned object is a `legacy_path`_ object. + tmpdir -- .../_pytest/legacypath.py:305 + Return a temporary directory (as `legacy_path`_ object) + which is unique to each test function invocation. + The temporary directory is created as a subdirectory + of the base temporary directory, with configurable retention, + as discussed in :ref:`temporary directory location and retention`. .. note:: These days, it is preferred to use ``tmp_path``. @@ -192,7 +214,7 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a .. _legacy_path: https://py.readthedocs.io/en/latest/path.html - caplog -- .../_pytest/logging.py:602 + caplog -- .../_pytest/logging.py:596 Access and control log capturing. Captured logs are available through the following properties/methods:: @@ -227,28 +249,23 @@ For information about fixtures, see :ref:`fixtures`. To see a complete list of a To undo modifications done by the fixture in a contained scope, use :meth:`context() `. - recwarn -- .../_pytest/recwarn.py:32 + recwarn -- .../_pytest/recwarn.py:34 Return a :class:`WarningsRecorder` instance that records all warnings emitted by test functions. - See https://docs.pytest.org/en/latest/how-to/capture-warnings.html for information - on warning categories. + See :ref:`warnings` for information on warning categories. - tmp_path_factory [session scope] -- .../_pytest/tmpdir.py:242 - Return a :class:`pytest.TempPathFactory` instance for the test session. - - tmp_path -- .../_pytest/tmpdir.py:257 - Return a temporary directory path object which is unique to each test - function invocation, created as a sub directory of the base temporary - directory. + subtests -- .../_pytest/subtests.py:129 + Provides subtests functionality. - By default, a new base temporary directory is created each test session, - and old bases are removed after 3 sessions, to aid in debugging. - This behavior can be configured with :confval:`tmp_path_retention_count` and - :confval:`tmp_path_retention_policy`. - If ``--basetemp`` is used then it is cleared each session. See - :ref:`temporary directory location and retention`. + tmp_path_factory [session scope] -- .../_pytest/tmpdir.py:265 + Return a :class:`pytest.TempPathFactory` instance for the test session. - The returned object is a :class:`pathlib.Path` object. + tmp_path -- .../_pytest/tmpdir.py:280 + Return a temporary directory (as :class:`pathlib.Path` object) + which is unique to each test function invocation. + The temporary directory is created as a subdirectory + of the base temporary directory, with configurable retention, + as discussed in :ref:`temporary directory location and retention`. ========================== no tests ran in 0.12s =========================== diff --git a/doc/en/changelog.rst b/doc/en/changelog.rst index 8e3efd0479b..5bcd44c4226 100644 --- a/doc/en/changelog.rst +++ b/doc/en/changelog.rst @@ -31,6 +31,1336 @@ with advance notice in the **Deprecations** section of releases. .. towncrier release notes start +pytest 9.0.3 (2026-04-07) +========================= + +Bug fixes +--------- + +- `#12444 `_: Fixed :func:`pytest.approx` which now correctly takes into account :class:`~collections.abc.Mapping` keys order to compare them. + + +- `#13634 `_: Blocking a ``conftest.py`` file using the ``-p no:`` option is now explicitly disallowed. + + Previously this resulted in an internal assertion failure during plugin loading. + + Pytest now raises a clear ``UsageError`` explaining that conftest files are not plugins and cannot be disabled via ``-p``. + + +- `#13734 `_: Fixed crash when a test raises an exceptiongroup with ``__tracebackhide__ = True``. + + +- `#14195 `_: Fixed an issue where non-string messages passed to `unittest.TestCase.subTest()` were not printed. + + +- `#14343 `_: Fixed use of insecure temporary directory (CVE-2025-71176). + + + +Improved documentation +---------------------- + +- `#13388 `_: Clarified documentation for ``-p`` vs ``PYTEST_PLUGINS`` plugin loading and fixed an incorrect ``-p`` example. + + +- `#13731 `_: Clarified that capture fixtures (e.g. ``capsys`` and ``capfd``) take precedence over the ``-s`` / ``--capture=no`` command-line options in :ref:`Accessing captured output from a test function `. + + +- `#14088 `_: Clarified that the default :hook:`pytest_collection` hook sets ``session.items`` before it calls :hook:`pytest_collection_finish`, not after. + + +- `#14255 `_: TOML integer log levels must be quoted: Updating reference documentation. + + + +Contributor-facing changes +-------------------------- + +- `#12689 `_: The test reports are now published to Codecov from GitHub Actions. + The test statistics is visible `on the web interface + `__. + + -- by :user:`aleguy02` + + +pytest 9.0.2 (2025-12-06) +========================= + +Bug fixes +--------- + +- `#13896 `_: The terminal progress feature added in pytest 9.0.0 has been disabled by default, except on Windows, due to compatibility issues with some terminal emulators. + + You may enable it again by passing ``-p terminalprogress``. We may enable it by default again once compatibility improves in the future. + + Additionally, when the environment variable ``TERM`` is ``dumb``, the escape codes are no longer emitted, even if the plugin is enabled. + + +- `#13904 `_: Fixed the TOML type of the :confval:`tmp_path_retention_count` settings in the API reference from number to string. + + +- `#13946 `_: The private ``config.inicfg`` attribute was changed in a breaking manner in pytest 9.0.0. + Due to its usage in the ecosystem, it is now restored to working order using a compatibility shim. + It will be deprecated in pytest 9.1 and removed in pytest 10. + + +- `#13965 `_: Fixed quadratic-time behavior when handling ``unittest`` subtests in Python 3.10. + + + +Improved documentation +---------------------- + +- `#4492 `_: The API Reference now contains cross-reference-able documentation of :ref:`pytest's command-line flags `. + + +pytest 9.0.1 (2025-11-12) +========================= + +Bug fixes +--------- + +- `#13895 `_: Restore support for skipping tests via ``raise unittest.SkipTest``. + + +- `#13896 `_: The terminal progress plugin added in pytest 9.0 is now automatically disabled when iTerm2 is detected, it generated desktop notifications instead of the desired functionality. + + +- `#13904 `_: Fixed the TOML type of the verbosity settings in the API reference from number to string. + + +- `#13910 `_: Fixed `UserWarning: Do not expect file_or_dir` on some earlier Python 3.12 and 3.13 point versions. + + + +Packaging updates and notes for downstreams +------------------------------------------- + +- `#13933 `_: The tox configuration has been adjusted to make sure the desired + version string can be passed into its :ref:`package_env` through + the ``SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST`` environment + variable as a part of the release process -- by :user:`webknjaz`. + + + +Contributor-facing changes +-------------------------- + +- `#13891 `_, `#13942 `_: The CI/CD part of the release automation is now capable of + creating GitHub Releases without having a Git checkout on + disk -- by :user:`bluetech` and :user:`webknjaz`. + + +- `#13933 `_: The tox configuration has been adjusted to make sure the desired + version string can be passed into its :ref:`package_env` through + the ``SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST`` environment + variable as a part of the release process -- by :user:`webknjaz`. + + +pytest 9.0.0 (2025-11-05) +========================= + +New features +------------ + + +- `#1367 `_: **Support for subtests** has been added. + + :ref:`subtests ` are an alternative to parametrization, useful in situations where the parametrization values are not all known at collection time. + + Example: + + .. code-block:: python + + def contains_docstring(p: Path) -> bool: + """Return True if the given Python file contains a top-level docstring.""" + ... + + + def test_py_files_contain_docstring(subtests: pytest.Subtests) -> None: + for path in Path.cwd().glob("*.py"): + with subtests.test(path=str(path)): + assert contains_docstring(path) + + + Each assert failure or error is caught by the context manager and reported individually, giving a clear picture of all files that are missing a docstring. + + In addition, :meth:`unittest.TestCase.subTest` is now also supported. + + This feature was originally implemented as a separate plugin in `pytest-subtests `__, but since then has been merged into the core. + + .. note:: + + This feature is experimental and will likely evolve in future releases. By that we mean that we might change how subtests are reported on failure, but the functionality and how to use it are stable. + + +- `#13743 `_: Added support for **native TOML configuration files**. + + While pytest, since version 6, supports configuration in ``pyproject.toml`` files under ``[tool.pytest.ini_options]``, + it does so in an "INI compatibility mode", where all configuration values are treated as strings or list of strings. + Now, pytest supports the native TOML data model. + + In ``pyproject.toml``, the native TOML configuration is under the ``[tool.pytest]`` table. + + .. code-block:: toml + + # pyproject.toml + [tool.pytest] + minversion = "9.0" + addopts = ["-ra", "-q"] + testpaths = [ + "tests", + "integration", + ] + + The ``[tool.pytest.ini_options]`` table remains supported, but both tables cannot be used at the same time. + + If you prefer to use a separate configuration file, or don't use ``pyproject.toml``, you can use ``pytest.toml`` or ``.pytest.toml``: + + .. code-block:: toml + + # pytest.toml or .pytest.toml + [pytest] + minversion = "9.0" + addopts = ["-ra", "-q"] + testpaths = [ + "tests", + "integration", + ] + + The documentation now (sometimes) shows configuration snippets in both TOML and INI formats, in a tabbed interface. + + See :ref:`config file formats` for full details. + + +- `#13823 `_: Added a **"strict mode"** enabled by the :confval:`strict` configuration option. + + When set to ``true``, the :confval:`strict` option currently enables + + * :confval:`strict_config` + * :confval:`strict_markers` + * :confval:`strict_parametrization_ids` + * :confval:`strict_xfail` + + The individual strictness options can be explicitly set to override the global :confval:`strict` setting. + + The previously-deprecated ``--strict`` command-line flag now enables strict mode. + + If pytest adds new strictness options in the future, they will also be enabled in strict mode. + Therefore, you should only enable strict mode if you use a pinned/locked version of pytest, + or if you want to proactively adopt new strictness options as they are added. + + See :ref:`strict mode` for more details. + + +- `#13737 `_: Added the :confval:`strict_parametrization_ids` configuration option. + + When set, pytest emits an error if it detects non-unique parameter set IDs, + rather than automatically making the IDs unique by adding `0`, `1`, ... to them. + This can be particularly useful for catching unintended duplicates. + + +- `#13072 `_: Added support for displaying test session **progress in the terminal tab** using the `OSC 9;4; `_ ANSI sequence. + + **Note**: *This feature has been disabled by default in version 9.0.2, except on Windows, due to compatibility issues with some terminal emulators. + You may enable it again by passing* ``-p terminalprogress``. *We may enable it by default again once compatibility improves in the future.* + + When pytest runs in a supported terminal emulator like ConEmu, Gnome Terminal, Ptyxis, Windows Terminal, Kitty or Ghostty, + you'll see the progress in the terminal tab or window, + allowing you to monitor pytest's progress at a glance. + + This feature is automatically enabled when running in a TTY. It is implemented as an internal plugin. If needed, it can be disabled as follows: + - On a user level, using ``-p no:terminalprogress`` on the command line or via an environment variable ``PYTEST_ADDOPTS='-p no:terminalprogress'``. + - On a project configuration level, using ``addopts = "-p no:terminalprogress"``. + + +- `#478 `_: Support PEP420 (implicit namespace packages) as `--pyargs` target when :confval:`consider_namespace_packages` is `true` in the config. + + Previously, this option only impacted package imports, now it also impacts tests discovery. + + +- `#13678 `_: Added a new :confval:`faulthandler_exit_on_timeout` configuration option set to "false" by default to let `faulthandler` interrupt the `pytest` process after a timeout in case of deadlock. + + Previously, a `faulthandler` timeout would only dump the traceback of all threads to stderr, but would not interrupt the `pytest` process. + + -- by :user:`ogrisel`. + + +- `#13829 `_: Added support for configuration option aliases via the ``aliases`` parameter in :meth:`Parser.addini() `. + + Plugins can now register alternative names for configuration options, + allowing for more flexibility in configuration naming and supporting backward compatibility when renaming options. + The canonical name always takes precedence if both the canonical name and an alias are specified in the configuration file. + + + +Improvements in existing functionality +-------------------------------------- + +- `#13330 `_: Having pytest configuration spread over more than one file (for example having both a ``pytest.ini`` file and ``pyproject.toml`` with a ``[tool.pytest.ini_options]`` table) will now print a warning to make it clearer to the user that only one of them is actually used. + + -- by :user:`sgaist` + + +- `#13574 `_: The single argument ``--version`` no longer loads the entire plugin infrastructure, making it faster and more reliable when displaying only the pytest version. + + Passing ``--version`` twice (e.g., ``pytest --version --version``) retains the original behavior, showing both the pytest version and plugin information. + + .. note:: + + Since ``--version`` is now processed early, it only takes effect when passed directly via the command line. It will not work if set through other mechanisms, such as :envvar:`PYTEST_ADDOPTS` or :confval:`addopts`. + + +- `#13823 `_: Added :confval:`strict_xfail` as an alias to the ``xfail_strict`` option, + :confval:`strict_config` as an alias to the ``--strict-config`` flag, + and :confval:`strict_markers` as an alias to the ``--strict-markers`` flag. + This makes all strictness options consistently have configuration options with the prefix ``strict_``. + +- `#13700 `_: `--junitxml` no longer prints the `generated xml file` summary at the end of the pytest session when `--quiet` is given. + + +- `#13732 `_: Previously, when filtering warnings, pytest would fail if the filter referenced a class that could not be imported. Now, this only outputs a message indicating the problem. + + +- `#13859 `_: Clarify the error message for `pytest.raises()` when a regex `match` fails. + + +- `#13861 `_: Better sentence structure in a test's expected error message. Previously, the error message would be "expected exception must be , but got ". Now, it is "Expected , but got ". + + +Removals and backward incompatible breaking changes +--------------------------------------------------- + +- `#12083 `_: Fixed a bug where an invocation such as `pytest a/ a/b` would cause only tests from `a/b` to run, and not other tests under `a/`. + + The fix entails a few breaking changes to how such overlapping arguments and duplicates are handled: + + 1. `pytest a/b a/` or `pytest a/ a/b` are equivalent to `pytest a`; if an argument overlaps another arguments, only the prefix remains. + + 2. `pytest x.py x.py` is equivalent to `pytest x.py`; previously such an invocation was taken as an explicit request to run the tests from the file twice. + + If you rely on these behaviors, consider using :ref:`--keep-duplicates `, which retains its existing behavior (including the bug). + + +- `#13719 `_: Support for Python 3.9 is dropped following its end of life. + + +- `#13766 `_: Previously, pytest would assume it was running in a CI/CD environment if either of the environment variables `$CI` or `$BUILD_NUMBER` was defined; + now, CI mode is only activated if at least one of those variables is defined and set to a *non-empty* value. + + +- The non-public ``config.args`` attribute used to be able to contain ``pathlib.Path`` instances; now it can only contain strings. + + +- `#13779 `_: **PytestRemovedIn9Warning deprecation warnings are now errors by default.** + + Following our plan to remove deprecated features with as little disruption as + possible, all warnings of type ``PytestRemovedIn9Warning`` now generate errors + instead of warning messages by default. + + **The affected features will be effectively removed in pytest 9.1**, so please consult the + :ref:`deprecations` section in the docs for directions on how to update existing code. + + In the pytest ``9.0.X`` series, it is possible to change the errors back into warnings as a + stopgap measure by adding this to your ``pytest.ini`` file: + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore::pytest.PytestRemovedIn9Warning + + But this will stop working when pytest ``9.1`` is released. + + **If you have concerns** about the removal of a specific feature, please add a + comment to :issue:`13779`. + + + +Deprecations (removal in next major release) +-------------------------------------------- + +- `#13807 `_: :meth:`monkeypatch.syspath_prepend() ` now issues a deprecation warning when the prepended path contains legacy namespace packages (those using ``pkg_resources.declare_namespace()``). + Users should migrate to native namespace packages (:pep:`420`). + See :ref:`monkeypatch-fixup-namespace-packages` for details. + + +Bug fixes +--------- + +- `#13445 `_: Made the type annotations of :func:`pytest.skip` and friends more spec-complaint to have them work across more type checkers. + + +- `#13537 `_: Fixed a bug in which :class:`ExceptionGroup` with only ``Skipped`` exceptions in teardown was not handled correctly and showed as error. + + +- `#13598 `_: Fixed possible collection confusion on Windows when short paths and symlinks are involved. + + +- `#13716 `_: Fixed a bug where a nonsensical invocation like ``pytest x.py[a]`` (a file cannot be parametrized) was silently treated as ``pytest x.py``. This is now a usage error. + + +- `#13722 `_: Fixed a misleading assertion failure message when using :func:`pytest.approx` on mappings with differing lengths. + + +- `#13773 `_: Fixed the static fixture closure calculation to properly consider transitive dependencies requested by overridden fixtures. + + +- `#13816 `_: Fixed :func:`pytest.approx` which now returns a clearer error message when comparing mappings with different keys. + + +- `#13849 `_: Hidden ``.pytest.ini`` files are now picked up as the config file even if empty. + This was an inconsistency with non-hidden ``pytest.ini``. + + +- `#13865 `_: Fixed `--show-capture` with `--tb=line`. + + +- `#13522 `_: Fixed :fixture:`pytester` in subprocess mode ignored all :attr:`pytester.plugins ` except the first. + + Fixed :fixture:`pytester` in subprocess mode silently ignored non-str :attr:`pytester.plugins `. + Now it errors instead. + If you are affected by this, specify the plugin by name, or switch the affected tests to use :func:`pytester.runpytest_inprocess ` explicitly instead. + + + +Packaging updates and notes for downstreams +------------------------------------------- + +- `#13791 `_: Minimum requirements on ``iniconfig`` and ``packaging`` were bumped to ``1.0.1`` and ``22.0.0``, respectively. + + + +Contributor-facing changes +-------------------------- + +- `#12244 `_: Fixed self-test failures when `TERM=dumb`. + + +- `#12474 `_: Added scheduled GitHub Action Workflow to run Sphinx linkchecks in repo documentation. + + +- `#13621 `_: pytest's own testsuite now handles the ``lsof`` command hanging (e.g. due to unreachable network filesystems), with the affected selftests being skipped after 10 seconds. + + +- `#13638 `_: Fixed deprecated :command:`gh pr new` command in :file:`scripts/prepare-release-pr.py`. + The script now uses :command:`gh pr create` which is compatible with GitHub CLI v2.0+. + + +- `#13695 `_: Flush `stdout` and `stderr` in `Pytester.run` to avoid truncated outputs in `test_faulthandler.py::test_timeout` on CI -- by :user:`ogrisel`. + + +- `#13771 `_: Skip `test_do_not_collect_symlink_siblings` on Windows environments without symlink support to avoid false negatives. + + +- `#13841 `_: ``tox>=4`` is now required when contributing to pytest. + +- `#13625 `_: Added missing docstrings to ``pytest_addoption()``, ``pytest_configure()``, and ``cacheshow()`` functions in ``cacheprovider.py``. + + + +Miscellaneous internal changes +------------------------------ + +- `#13830 `_: Configuration overrides (``-o``/``--override-ini``) are now processed during startup rather than during :func:`config.getini() `. + + +pytest 8.4.2 (2025-09-03) +========================= + +Bug fixes +--------- + +- `#13478 `_: Fixed a crash when using :confval:`console_output_style` with ``times`` and a module is skipped. + + +- `#13530 `_: Fixed a crash when using :func:`pytest.approx` and :class:`decimal.Decimal` instances with the :class:`decimal.FloatOperation` trap set. + + +- `#13549 `_: No longer evaluate type annotations in Python ``3.14`` when inspecting function signatures. + + This prevents crashes during module collection when modules do not explicitly use ``from __future__ import annotations`` and import types for annotations within a ``if TYPE_CHECKING:`` block. + + +- `#13559 `_: Added missing `int` and `float` variants to the `Literal` type annotation of the `type` parameter in :meth:`pytest.Parser.addini`. + + +- `#13563 `_: :func:`pytest.approx` now only imports ``numpy`` if NumPy is already in ``sys.modules``. This fixes unconditional import behavior introduced in `8.4.0`. + + + +Improved documentation +---------------------- + +- `#13577 `_: Clarify that ``pytest_generate_tests`` is discovered in test modules/classes; other hooks must be in ``conftest.py`` or plugins. + + + +Contributor-facing changes +-------------------------- + +- `#13480 `_: Self-testing: fixed a few test failures when run with ``-Wdefault`` or a similar override. + + +- `#13547 `_: Self-testing: corrected expected message for ``test_doctest_unexpected_exception`` in Python ``3.14``. + + +- `#13684 `_: Make pytest's own testsuite insensitive to the presence of the ``CI`` environment variable -- by :user:`ogrisel`. + + +pytest 8.4.1 (2025-06-17) +========================= + +Bug fixes +--------- + +- `#13461 `_: Corrected ``_pytest.terminal.TerminalReporter.isatty`` to support + being called as a method. Before it was just a boolean which could + break correct code when using ``-o log_cli=true``). + + +- `#13477 `_: Reintroduced :class:`pytest.PytestReturnNotNoneWarning` which was removed by accident in pytest `8.4`. + + This warning is raised when a test functions returns a value other than ``None``, which is often a mistake made by beginners. + + See :ref:`return-not-none` for more information. + + +- `#13497 `_: Fixed compatibility with ``Twisted 25+``. + + + +Improved documentation +---------------------- + +- `#13492 `_: Fixed outdated warning about ``faulthandler`` not working on Windows. + + +pytest 8.4.0 (2025-06-02) +========================= + +Removals and backward incompatible breaking changes +--------------------------------------------------- + +- `#11372 `_: Async tests will now fail, instead of warning+skipping, if you don't have any suitable plugin installed. + + +- `#12346 `_: Tests will now fail, instead of raising a warning, if they return any value other than None. + + +- `#12874 `_: We dropped support for Python 3.8 following its end of life (2024-10-07). + + +- `#12960 `_: Test functions containing a yield now cause an explicit error. They have not been run since pytest 4.0, and were previously marked as an expected failure and deprecation warning. + + See :ref:`the docs ` for more information. + + + +Deprecations (removal in next major release) +-------------------------------------------- + +- `#10839 `_: Requesting an asynchronous fixture without a `pytest_fixture_setup` hook that resolves it will now give a DeprecationWarning. This most commonly happens if a sync test requests an async fixture. This should have no effect on a majority of users with async tests or fixtures using async pytest plugins, but may affect non-standard hook setups or ``autouse=True``. For guidance on how to work around this warning see :ref:`sync-test-async-fixture`. + + + +New features +------------ + +- `#11538 `_: Added :class:`pytest.RaisesGroup` as an equivalent to :func:`pytest.raises` for expecting :exc:`ExceptionGroup`. Also adds :class:`pytest.RaisesExc` which is now the logic behind :func:`pytest.raises` and used as parameter to :class:`pytest.RaisesGroup`. ``RaisesGroup`` includes the ability to specify multiple different expected exceptions, the structure of nested exception groups, and flags for emulating :ref:`except* `. See :ref:`assert-matching-exception-groups` and docstrings for more information. + + +- `#12081 `_: Added :fixture:`capteesys` to capture AND pass output to next handler set by ``--capture=``. + + +- `#12504 `_: :func:`pytest.mark.xfail` now accepts :class:`pytest.RaisesGroup` for the ``raises`` parameter when you expect an exception group. You can also pass a :class:`pytest.RaisesExc` if you e.g. want to make use of the ``check`` parameter. + + +- `#12713 `_: New `--force-short-summary` option to force condensed summary output regardless of verbosity level. + + This lets users still see condensed summary output of failures for quick reference in log files from job outputs, being especially useful if non-condensed output is very verbose. + + +- `#12749 `_: pytest traditionally collects classes/functions in the test module namespace even if they are imported from another file. + + For example: + + .. code-block:: python + + # contents of src/domain.py + class Testament: ... + + + # contents of tests/test_testament.py + from domain import Testament + + + def test_testament(): ... + + In this scenario with the default options, pytest will collect the class `Testament` from `tests/test_testament.py` because it starts with `Test`, even though in this case it is a production class being imported in the test module namespace. + + This behavior can now be prevented by setting the new :confval:`collect_imported_tests` configuration option to ``false``, which will make pytest collect classes/functions from test files **only** if they are defined in that file. + + -- by :user:`FreerGit` + + +- `#12765 `_: Thresholds to trigger snippet truncation can now be set with :confval:`truncation_limit_lines` and :confval:`truncation_limit_chars`. + + See :ref:`truncation-params` for more information. + + +- `#13125 `_: :confval:`console_output_style` now supports ``times`` to show execution time of each test. + + +- `#13192 `_: :func:`pytest.raises` will now raise a warning when passing an empty string to ``match``, as this will match against any value. Use ``match="^$"`` if you want to check that an exception has no message. + + +- `#13192 `_: :func:`pytest.raises` will now print a helpful string diff if matching fails and the match parameter has ``^`` and ``$`` and is otherwise escaped. + + +- `#13192 `_: You can now pass :func:`with pytest.raises(check=fn): `, where ``fn`` is a function which takes a raised exception and returns a boolean. The ``raises`` fails if no exception was raised (as usual), passes if an exception is raised and ``fn`` returns ``True`` (as well as ``match`` and the type matching, if specified, which are checked before), and propagates the exception if ``fn`` returns ``False`` (which likely also fails the test). + + +- `#13228 `_: :ref:`hidden-param` can now be used in ``id`` of :func:`pytest.param` or in + ``ids`` of :py:func:`Metafunc.parametrize `. + It hides the parameter set from the test name. + + +- `#13253 `_: New flag: :ref:`--disable-plugin-autoload ` which works as an alternative to :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD` when setting environment variables is inconvenient; and allows setting it in config files with :confval:`addopts`. + + + +Improvements in existing functionality +-------------------------------------- + +- `#10224 `_: pytest's ``short`` and ``long`` traceback styles (:ref:`how-to-modifying-python-tb-printing`) + now have partial :pep:`657` support and will show specific code segments in the + traceback. + + .. code-block:: pytest + + ================================= FAILURES ================================= + _______________________ test_gets_correct_tracebacks _______________________ + + test_tracebacks.py:12: in test_gets_correct_tracebacks + assert manhattan_distance(p1, p2) == 1 + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + test_tracebacks.py:6: in manhattan_distance + return abs(point_1.x - point_2.x) + abs(point_1.y - point_2.y) + ^^^^^^^^^ + E AttributeError: 'NoneType' object has no attribute 'x' + + -- by :user:`ammaraskar` + + +- `#11118 `_: Now :confval:`pythonpath` configures `$PYTHONPATH` earlier than before during the initialization process, which now also affects plugins loaded via the `-p` command-line option. + + -- by :user:`millerdev` + + +- `#11381 `_: The ``type`` parameter of the ``parser.addini`` method now accepts `"int"` and ``"float"`` parameters, facilitating the parsing of configuration values in the configuration file. + + Example: + + .. code-block:: python + + def pytest_addoption(parser): + parser.addini("int_value", type="int", default=2, help="my int value") + parser.addini("float_value", type="float", default=4.2, help="my float value") + + The `pytest.ini` file: + + .. code-block:: ini + + [pytest] + int_value = 3 + float_value = 5.4 + + +- `#11525 `_: Fixtures are now clearly represented in the output as a "fixture object", not as a normal function as before, making it easy for beginners to catch mistakes such as referencing a fixture declared in the same module but not requested in the test function. + + -- by :user:`the-compiler` and :user:`glyphack` + + +- `#12426 `_: A warning is now issued when :ref:`pytest.mark.usefixtures ref` is used without specifying any fixtures. Previously, empty usefixtures markers were silently ignored. + + +- `#12707 `_: Exception chains can be navigated when dropped into Pdb in Python 3.13+. + + +- `#12736 `_: Added a new attribute `name` with the fixed value `"pytest tests"` to the root tag `testsuites` of the junit-xml generated by pytest. + + This attribute is part of many junit-xml specifications and is even part of the `junit-10.xsd` specification that pytest's implementation is based on. + + +- `#12943 `_: If a test fails with an exceptiongroup with a single exception, the contained exception will now be displayed in the short test summary info. + + +- `#12958 `_: A number of :ref:`unraisable ` enhancements: + + * Set the unraisable hook as early as possible and unset it as late as possible, to collect the most possible number of unraisable exceptions. + * Call the garbage collector just before unsetting the unraisable hook, to collect any straggling exceptions. + * Collect multiple unraisable exceptions per test phase. + * Report the :mod:`tracemalloc` allocation traceback (if available). + * Avoid using a generator based hook to allow handling :class:`StopIteration` in test failures. + * Report the unraisable exception as the cause of the :class:`pytest.PytestUnraisableExceptionWarning` exception if raised. + * Compute the ``repr`` of the unraisable object in the unraisable hook so you get the latest information if available, and should help with resurrection of the object. + + +- `#13010 `_: :func:`pytest.approx` now can compare collections that contain numbers and non-numbers mixed. + + +- `#13016 `_: A number of :ref:`threadexception ` enhancements: + + * Set the excepthook as early as possible and unset it as late as possible, to collect the most possible number of unhandled exceptions from threads. + * Collect multiple thread exceptions per test phase. + * Report the :mod:`tracemalloc` allocation traceback (if available). + * Avoid using a generator based hook to allow handling :class:`StopIteration` in test failures. + * Report the thread exception as the cause of the :class:`pytest.PytestUnhandledThreadExceptionWarning` exception if raised. + * Extract the ``name`` of the thread object in the excepthook which should help with resurrection of the thread. + + +- `#13031 `_: An empty parameter set as in ``pytest.mark.parametrize([], ids=idfunc)`` will no longer trigger a call to ``idfunc`` with internal objects. + + +- `#13115 `_: Allows supplying ``ExceptionGroup[Exception]`` and ``BaseExceptionGroup[BaseException]`` to ``pytest.raises`` to keep full typing on :class:`ExceptionInfo `: + + .. code-block:: python + + with pytest.raises(ExceptionGroup[Exception]) as exc_info: + some_function() + + Parametrizing with other exception types remains an error - we do not check the types of child exceptions and thus do not permit code that might look like we do. + + +- `#13122 `_: The ``--stepwise`` mode received a number of improvements: + + * It no longer forgets the last failed test in case pytest is executed later without the flag. + + This enables the following workflow: + + 1. Execute pytest with ``--stepwise``, pytest then stops at the first failing test; + 2. Iteratively update the code and run the test in isolation, without the ``--stepwise`` flag (for example in an IDE), until it is fixed. + 3. Execute pytest with ``--stepwise`` again and pytest will continue from the previously failed test, and if it passes, continue on to the next tests. + + Previously, at step 3, pytest would start from the beginning, forgetting the previously failed test. + + This change however might cause issues if the ``--stepwise`` mode is used far apart in time, as the state might get stale, so the internal state will be reset automatically in case the test suite changes (for now only the number of tests are considered for this, we might change/improve this on the future). + + * New ``--stepwise-reset``/``--sw-reset`` flag, allowing the user to explicitly reset the stepwise state and restart the workflow from the beginning. + + +- `#13308 `_: Added official support for Python 3.14. + + +- `#13380 `_: Fix :class:`ExceptionGroup` traceback filtering to exclude pytest internals. + + +- `#13415 `_: The author metadata of the BibTex example is now correctly formatted with last names following first names. + An example of BibLaTex has been added. + BibTex and BibLaTex examples now clearly indicate that what is cited is software. + + -- by :user:`willynilly` + + +- `#13420 `_: Improved test collection performance by optimizing path resolution used in ``FSCollector``. + + +- `#13457 `_: The error message about duplicate parametrization no longer displays an internal stack trace. + + +- `#4112 `_: Using :ref:`pytest.mark.usefixtures ` on :func:`pytest.param` now produces an error instead of silently doing nothing. + + +- `#5473 `_: Replace `:` with `;` in the assertion rewrite warning message so it can be filtered using standard Python warning filters before calling :func:`pytest.main`. + + +- `#6985 `_: Improved :func:`pytest.approx` to enhance the readability of value ranges and tolerances between 0.001 and 1000. + * The `repr` method now provides clearer output for values within those ranges, making it easier to interpret the results. + * Previously, the output for those ranges of values and tolerances was displayed in scientific notation (e.g., `42 ± 1.0e+00`). The updated method now presents the tolerance as a decimal for better readability (e.g., `42 ± 1`). + + Example: + + **Previous Output:** + + .. code-block:: console + + >>> pytest.approx(42, abs=1) + 42 ± 1.0e+00 + + **Current Output:** + + .. code-block:: console + + >>> pytest.approx(42, abs=1) + 42 ± 1 + + -- by :user:`fazeelghafoor` + + +- `#7683 `_: The formerly optional ``pygments`` dependency is now required, causing output always to be source-highlighted (unless disabled via the ``--code-highlight=no`` CLI option). + + + +Bug fixes +--------- + +- `#10404 `_: Apply filterwarnings from config/cli as soon as possible, and revert them as late as possible + so that warnings as errors are collected throughout the pytest run and before the + unraisable and threadexcept hooks are removed. + + This allows very late warnings and unraisable/threadexcept exceptions to fail the test suite. + + This also changes the warning that the lsof plugin issues from PytestWarning to the new warning PytestFDWarning so it can be more easily filtered. + + +- `#11067 `_: The test report is now consistent regardless if the test xfailed via :ref:`pytest.mark.xfail ` or :func:`pytest.fail`. + + Previously, *xfailed* tests via the marker would have the string ``"reason: "`` prefixed to the message, while those *xfailed* via the function did not. The prefix has been removed. + + +- `#12008 `_: In :pr:`11220`, an unintended change in reordering was introduced by changing the way indices were assigned to direct params. More specifically, before that change, the indices of direct params to metafunc's callspecs were assigned after all parametrizations took place. Now, that change is reverted. + + +- `#12863 `_: Fix applying markers, including :ref:`pytest.mark.parametrize ` when placed above `@staticmethod` or `@classmethod`. + + +- `#12929 `_: Handle StopIteration from test cases, setup and teardown correctly. + + +- `#12938 `_: Fixed ``--durations-min`` argument not respected if ``-vv`` is used. + + +- `#12946 `_: Fixed missing help for :mod:`pdb` commands wrapped by pytest -- by :user:`adamchainz`. + + +- `#12981 `_: Prevent exceptions in :func:`pytest.Config.add_cleanup` callbacks preventing further cleanups. + + +- `#13047 `_: Restore :func:`pytest.approx` handling of equality checks between `bool` and `numpy.bool_` types. + + Comparing `bool` and `numpy.bool_` using :func:`pytest.approx` accidentally changed in version `8.3.4` and `8.3.5` to no longer match: + + .. code-block:: pycon + + >>> import numpy as np + >>> from pytest import approx + >>> [np.True_, np.True_] == pytest.approx([True, True]) + False + + This has now been fixed: + + .. code-block:: pycon + + >>> [np.True_, np.True_] == pytest.approx([True, True]) + True + + +- `#13119 `_: Improved handling of invalid regex patterns for filter warnings by providing a clear error message. + + +- `#13175 `_: The diff is now also highlighted correctly when comparing two strings. + + +- `#13248 `_: Fixed an issue where passing a ``scope`` in :py:func:`Metafunc.parametrize ` with ``indirect=True`` + could result in other fixtures being unable to depend on the parametrized fixture. + + +- `#13291 `_: Fixed ``repr`` of ``attrs`` objects in assertion failure messages when using ``attrs>=25.2``. + + +- `#13312 `_: Fixed a possible ``KeyError`` crash on PyPy during collection of tests involving higher-scoped parameters. + + +- `#13345 `_: Fix type hints for :attr:`pytest.TestReport.when` and :attr:`pytest.TestReport.location`. + + +- `#13377 `_: Fixed handling of test methods with positional-only parameter syntax. + + Now, methods are supported that formally define ``self`` as positional-only + and/or fixture parameters as keyword-only, e.g.: + + .. code-block:: python + + class TestClass: + + def test_method(self, /, *, fixture): ... + + Before, this caused an internal error in pytest. + + +- `#13384 `_: Fixed an issue where pytest could report negative durations. + + +- `#13420 `_: Added ``lru_cache`` to ``nodes._check_initialpaths_for_relpath``. + + +- `#9037 `_: Honor :confval:`disable_test_id_escaping_and_forfeit_all_rights_to_community_support` when escaping ids in parametrized tests. + + + +Improved documentation +---------------------- + +- `#12535 `_: `This + example` + showed ``print`` statements that do not exactly reflect what the + different branches actually do. The fix makes the example more precise. + + +- `#13218 `_: Pointed out in the :func:`pytest.approx` documentation that it considers booleans unequal to numeric zero or one. + + +- `#13221 `_: Improved grouping of CLI options in the ``--help`` output. + + +- `#6649 `_: Added :class:`~pytest.TerminalReporter` to the :ref:`api-reference` documentation page. + + +- `#8612 `_: Add a recipe for handling abstract test classes in the documentation. + + A new example has been added to the documentation to demonstrate how to use a mixin class to handle abstract + test classes without manually setting the ``__test__`` attribute for subclasses. + This ensures that subclasses of abstract test classes are automatically collected by pytest. + + + +Packaging updates and notes for downstreams +------------------------------------------- + +- `#13317 `_: Specified minimum allowed versions of ``colorama``, ``iniconfig``, + and ``packaging``; and bumped the minimum allowed version + of ``exceptiongroup`` for ``python_version<'3.11'`` from a release + candidate to a full release. + + + +Contributor-facing changes +-------------------------- + +- `#12017 `_: Mixed internal improvements: + + * Migrate formatting to f-strings in some tests. + * Use type-safe constructs in JUnitXML tests. + * Moved`` MockTiming`` into ``_pytest.timing``. + + -- by :user:`RonnyPfannschmidt` + + +- `#12647 `_: Fixed running the test suite with the ``hypothesis`` pytest plugin. + + + +Miscellaneous internal changes +------------------------------ + +- `#6649 `_: Added :class:`~pytest.TerminalReporter` to the public pytest API, as it is part of the signature of the :hook:`pytest_terminal_summary` hook. + + +pytest 8.3.5 (2025-03-02) +========================= + +Bug fixes +--------- + +- `#11777 `_: Fixed issue where sequences were still being shortened even with ``-vv`` verbosity. + + +- `#12888 `_: Fixed broken input when using Python 3.13+ and a ``libedit`` build of Python, such as on macOS or with uv-managed Python binaries from the ``python-build-standalone`` project. This could manifest e.g. by a broken prompt when using ``Pdb``, or seeing empty inputs with manual usage of ``input()`` and suspended capturing. + + +- `#13026 `_: Fixed :class:`AttributeError` crash when using ``--import-mode=importlib`` when top-level directory same name as another module of the standard library. + + +- `#13053 `_: Fixed a regression in pytest 8.3.4 where, when using ``--import-mode=importlib``, a directory containing py file with the same name would cause an ``ImportError`` + + +- `#13083 `_: Fixed issue where pytest could crash if one of the collected directories got removed during collection. + + + +Improved documentation +---------------------- + +- `#12842 `_: Added dedicated page about using types with pytest. + + See :ref:`types` for detailed usage. + + + +Contributor-facing changes +-------------------------- + +- `#13112 `_: Fixed selftest failures in ``test_terminal.py`` with Pygments >= 2.19.0 + + +- `#13256 `_: Support for Towncrier versions released in 2024 has been re-enabled + when building Sphinx docs -- by :user:`webknjaz`. + + +pytest 8.3.4 (2024-12-01) +========================= + +Bug fixes +--------- + +- `#12592 `_: Fixed :class:`KeyError` crash when using ``--import-mode=importlib`` in a directory layout where a directory contains a child directory with the same name. + + +- `#12818 `_: Assertion rewriting now preserves the source ranges of the original instructions, making it play well with tools that deal with the ``AST``, like `executing `__. + + +- `#12849 `_: ANSI escape codes for colored output now handled correctly in :func:`pytest.fail` with `pytrace=False`. + + +- `#9353 `_: :func:`pytest.approx` now uses strict equality when given booleans. + + + +Improved documentation +---------------------- + +- `#10558 `_: Fix ambiguous docstring of :func:`pytest.Config.getoption`. + + +- `#10829 `_: Improve documentation on the current handling of the ``--basetemp`` option and its lack of retention functionality (:ref:`temporary directory location and retention`). + + +- `#12866 `_: Improved cross-references concerning the :fixture:`recwarn` fixture. + + +- `#12966 `_: Clarify :ref:`filterwarnings` docs on filter precedence/order when using multiple :ref:`@pytest.mark.filterwarnings ` marks. + + + +Contributor-facing changes +-------------------------- + +- `#12497 `_: Fixed two failing pdb-related tests on Python 3.13. + + +pytest 8.3.3 (2024-09-09) +========================= + +Bug fixes +--------- + +- `#12446 `_: Avoid calling ``@property`` (and other instance descriptors) during fixture discovery -- by :user:`asottile` + + +- `#12659 `_: Fixed the issue of not displaying assertion failure differences when using the parameter ``--import-mode=importlib`` in pytest>=8.1. + + +- `#12667 `_: Fixed a regression where type change in `ExceptionInfo.errisinstance` caused `mypy` to fail. + + +- `#12744 `_: Fixed typing compatibility with Python 3.9 or less -- replaced `typing.Self` with `typing_extensions.Self` -- by :user:`Avasam` + + +- `#12745 `_: Fixed an issue with backslashes being incorrectly converted in nodeid paths on Windows, ensuring consistent path handling across environments. + + +- `#6682 `_: Fixed bug where the verbosity levels where not being respected when printing the "msg" part of failed assertion (as in ``assert condition, msg``). + + +- `#9422 `_: Fix bug where disabling the terminal plugin via ``-p no:terminal`` would cause crashes related to missing the ``verbose`` option. + + -- by :user:`GTowers1` + + + +Improved documentation +---------------------- + +- `#12663 `_: Clarify that the `pytest_deselected` hook should be called from `pytest_collection_modifyitems` hook implementations when items are deselected. + + +- `#12678 `_: Remove erroneous quotes from `tmp_path_retention_policy` example in docs. + + + +Miscellaneous internal changes +------------------------------ + +- `#12769 `_: Fix typos discovered by codespell and add codespell to pre-commit hooks. + + +pytest 8.3.2 (2024-07-24) +========================= + +Bug fixes +--------- + +- `#12652 `_: Resolve regression `conda` environments where no longer being automatically detected. + + -- by :user:`RonnyPfannschmidt` + + +pytest 8.3.1 (2024-07-20) +========================= + +The 8.3.0 release failed to include the change notes and docs for the release. This patch release remedies this. There are no other changes. + + +pytest 8.3.0 (2024-07-20) +========================= + +New features +------------ + +- `#12231 `_: Added `--xfail-tb` flag, which turns on traceback output for XFAIL results. + + * If the `--xfail-tb` flag is not given, tracebacks for XFAIL results are NOT shown. + * The style of traceback for XFAIL is set with `--tb`, and can be `auto|long|short|line|native|no`. + * Note: Even if you have `--xfail-tb` set, you won't see them if `--tb=no`. + + Some history: + + With pytest 8.0, `-rx` or `-ra` would not only turn on summary reports for xfail, but also report the tracebacks for xfail results. This caused issues with some projects that utilize xfail, but don't want to see all of the xfail tracebacks. + + This change detaches xfail tracebacks from `-rx`, and now we turn on xfail tracebacks with `--xfail-tb`. With this, the default `-rx`/ `-ra` behavior is identical to pre-8.0 with respect to xfail tracebacks. While this is a behavior change, it brings default behavior back to pre-8.0.0 behavior, which ultimately was considered the better course of action. + + -- by :user:`okken` + + +- `#12281 `_: Added support for keyword matching in marker expressions. + + Now tests can be selected by marker keyword arguments. + Supported values are :class:`int`, (unescaped) :class:`str`, :class:`bool` & :data:`None`. + + See :ref:`marker examples ` for more information. + + -- by :user:`lovetheguitar` + + +- `#12567 `_: Added ``--no-fold-skipped`` command line option. + + If this option is set, then skipped tests in short summary are no longer grouped + by reason but all tests are printed individually with their nodeid in the same + way as other statuses. + + -- by :user:`pbrezina` + + + +Improvements in existing functionality +-------------------------------------- + +- `#12469 `_: The console output now uses the "third-party plugins" terminology, + replacing the previously established but confusing and outdated + reference to :std:doc:`setuptools ` + -- by :user:`webknjaz`. + + +- `#12544 `_, `#12545 `_: Python virtual environment detection was improved by + checking for a :file:`pyvenv.cfg` file, ensuring reliable detection on + various platforms -- by :user:`zachsnickers`. + + +- `#2871 `_: Do not truncate arguments to functions in output when running with `-vvv`. + + +- `#389 `_: The readability of assertion introspection of bound methods has been enhanced + -- by :user:`farbodahm`, :user:`webknjaz`, :user:`obestwalter`, :user:`flub` + and :user:`glyphack`. + + Earlier, it was like: + + .. code-block:: console + + =================================== FAILURES =================================== + _____________________________________ test _____________________________________ + + def test(): + > assert Help().fun() == 2 + E assert 1 == 2 + E + where 1 = >() + E + where > = .fun + E + where = Help() + + example.py:7: AssertionError + =========================== 1 failed in 0.03 seconds =========================== + + + And now it's like: + + .. code-block:: console + + =================================== FAILURES =================================== + _____________________________________ test _____________________________________ + + def test(): + > assert Help().fun() == 2 + E assert 1 == 2 + E + where 1 = fun() + E + where fun = .fun + E + where = Help() + + test_local.py:13: AssertionError + =========================== 1 failed in 0.03 seconds =========================== + + +- `#7662 `_: Added timezone information to the testsuite timestamp in the JUnit XML report. + + + +Bug fixes +--------- + +- `#11706 `_: Fixed reporting of teardown errors in higher-scoped fixtures when using `--maxfail` or `--stepwise`. + + Originally added in pytest 8.0.0, but reverted in 8.0.2 due to a regression in pytest-xdist. + This regression was fixed in pytest-xdist 3.6.1. + + +- `#11797 `_: :func:`pytest.approx` now correctly handles :class:`Sequence `-like objects. + + +- `#12204 `_, `#12264 `_: Fixed a regression in pytest 8.0 where tracebacks get longer and longer when multiple + tests fail due to a shared higher-scope fixture which raised -- by :user:`bluetech`. + + Also fixed a similar regression in pytest 5.4 for collectors which raise during setup. + + The fix necessitated internal changes which may affect some plugins: + + * ``FixtureDef.cached_result[2]`` is now a tuple ``(exc, tb)`` + instead of ``exc``. + * ``SetupState.stack`` failures are now a tuple ``(exc, tb)`` + instead of ``exc``. + + +- `#12275 `_: Fixed collection error upon encountering an :mod:`abstract ` class, including abstract `unittest.TestCase` subclasses. + + +- `#12328 `_: Fixed a regression in pytest 8.0.0 where package-scoped parameterized items were not correctly reordered to minimize setups/teardowns in some cases. + + +- `#12424 `_: Fixed crash with `assert testcase is not None` assertion failure when re-running unittest tests using plugins like pytest-rerunfailures. Regressed in 8.2.2. + + +- `#12472 `_: Fixed a crash when returning category ``"error"`` or ``"failed"`` with a custom test status from :hook:`pytest_report_teststatus` hook -- :user:`pbrezina`. + + +- `#12505 `_: Improved handling of invalid regex patterns in :func:`pytest.raises(match=r'...') ` by providing a clear error message. + + +- `#12580 `_: Fixed a crash when using the cache class on Windows and the cache directory was created concurrently. + + +- `#6962 `_: Parametrization parameters are now compared using `==` instead of `is` (`is` is still used as a fallback if the parameter does not support `==`). + This fixes use of parameters such as lists, which have a different `id` but compare equal, causing fixtures to be re-computed instead of being cached. + + +- `#7166 `_: Fixed progress percentages (the ``[ 87%]`` at the edge of the screen) sometimes not aligning correctly when running with pytest-xdist ``-n``. + + + +Improved documentation +---------------------- + +- `#12153 `_: Documented using :envvar:`PYTEST_VERSION` to detect if code is running from within a pytest run. + + +- `#12469 `_: The external plugin mentions in the documentation now avoid mentioning + :std:doc:`setuptools entry-points ` as the concept is + much more generic nowadays. Instead, the terminology of "external", + "installed", or "third-party" plugins (or packages) replaces that. + + -- by :user:`webknjaz` + + +- `#12577 `_: `CI` and `BUILD_NUMBER` environment variables role is described in + the reference doc. They now also appear when doing `pytest -h` + -- by :user:`MarcBresson`. + + + +Contributor-facing changes +-------------------------- + +- `#12467 `_: Migrated all internal type-annotations to the python3.10+ style by using the `annotations` future import. + + -- by :user:`RonnyPfannschmidt` + + +- `#11771 `_, `#12557 `_: The PyPy runtime version has been updated to 3.9 from 3.8 that introduced + a flaky bug at the garbage collector which was not expected to fix there + as the 3.8 is EoL. + + -- by :user:`x612skm` + + +- `#12493 `_: The change log draft preview integration has been refactored to use a + third party extension ``sphinxcontib-towncrier``. The previous in-repo + script was putting the change log preview file at + :file:`doc/en/_changelog_towncrier_draft.rst`. Said file is no longer + ignored in Git and might show up among untracked files in the + development environments of the contributors. To address that, the + contributors can run the following command that will clean it up: + + .. code-block:: console + + $ git clean -x -i -- doc/en/_changelog_towncrier_draft.rst + + -- by :user:`webknjaz` + + +- `#12498 `_: All the undocumented ``tox`` environments now have descriptions. + They can be listed in one's development environment by invoking + ``tox -av`` in a terminal. + + -- by :user:`webknjaz` + + +- `#12501 `_: The changelog configuration has been updated to introduce more accurate + audience-tailored categories. Previously, there was a ``trivial`` + change log fragment type with an unclear and broad meaning. It was + removed and we now have ``contrib``, ``misc`` and ``packaging`` in + place of it. + + The new change note types target the readers who are downstream + packagers and project contributors. Additionally, the miscellaneous + section is kept for unspecified updates that do not fit anywhere else. + + -- by :user:`webknjaz` + + +- `#12502 `_: The UX of the GitHub automation making pull requests to update the + plugin list has been updated. Previously, the maintainers had to close + the automatically created pull requests and re-open them to trigger the + CI runs. From now on, they only need to click the `Ready for review` + button instead. + + -- by :user:`webknjaz` + + +- `#12522 `_: The ``:pull:`` RST role has been replaced with a shorter + ``:pr:`` due to starting to use the implementation from + the third-party :pypi:`sphinx-issues` Sphinx extension + -- by :user:`webknjaz`. + + +- `#12531 `_: The coverage reporting configuration has been updated to exclude + pytest's own tests marked as expected to fail from the coverage + report. This has an effect of reducing the influence of flaky + tests on the resulting number. + + -- by :user:`webknjaz` + + +- `#12533 `_: The ``extlinks`` Sphinx extension is no longer enabled. The ``:bpo:`` + role it used to declare has been removed with that. BPO itself has + migrated to GitHub some years ago and it is possible to link the + respective issues by using their GitHub issue numbers and the + ``:issue:`` role that the ``sphinx-issues`` extension implements. + + -- by :user:`webknjaz` + + +- `#12562 `_: Possible typos in using the ``:user:`` RST role is now being linted + through the pre-commit tool integration -- by :user:`webknjaz`. + + pytest 8.2.2 (2024-06-04) ========================= @@ -43,7 +1373,7 @@ Bug Fixes - `#12367 `_: Fix a regression in pytest 8.2.0 where unittest class instances (a fresh one is created for each test) were not released promptly on test teardown but only on session teardown. -- `#12381 `_: Fix possible "Directory not empty" crashes arising from concurent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0. +- `#12381 `_: Fix possible "Directory not empty" crashes arising from concurrent cache dir (``.pytest_cache``) creation. Regressed in pytest 8.2.0. @@ -54,7 +1384,7 @@ Improved Documentation - `#12356 `_: Added a subsection to the documentation for debugging flaky tests to mention - lack of thread safety in pytest as a possible source of flakyness. + lack of thread safety in pytest as a possible source of flakiness. - `#12363 `_: The documentation webpages now links to a canonical version to reduce outdated documentation in search engine results. @@ -400,7 +1730,7 @@ Bug Fixes This bug was introduced in pytest 8.0.0rc1. -- `#9765 `_, `#11816 `_: Fixed a frustrating bug that afflicted some users with the only error being ``assert mod not in mods``. The issue was caused by the fact that ``str(Path(mod))`` and ``mod.__file__`` don't necessarily produce the same string, and was being erroneously used interchangably in some places in the code. +- `#9765 `_, `#11816 `_: Fixed a frustrating bug that afflicted some users with the only error being ``assert mod not in mods``. The issue was caused by the fact that ``str(Path(mod))`` and ``mod.__file__`` don't necessarily produce the same string, and was being erroneously used interchangeably in some places in the code. This fix also broke the internal API of ``PytestPluginManager.consider_conftest`` by introducing a new parameter -- we mention this in case it is being used by external code, even if marked as *private*. @@ -1140,7 +2470,7 @@ pytest 7.2.0 (2022-10-23) Deprecations ------------ -- `#10012 `_: Update :class:`pytest.PytestUnhandledCoroutineWarning` to a deprecation; it will raise an error in pytest 8. +- `#10012 `_: Update ``pytest.PytestUnhandledCoroutineWarning`` to a deprecation; it will raise an error in pytest 8. - `#10396 `_: pytest no longer depends on the ``py`` library. ``pytest`` provides a vendored copy of ``py.error`` and ``py.path`` modules but will use the ``py`` library if it is installed. If you need other ``py.*`` modules, continue to install the deprecated ``py`` library separately, otherwise it can usually be removed as a dependency. @@ -1527,6 +2857,7 @@ Breaking Changes - `#8246 `_: ``--version`` now writes version information to ``stdout`` rather than ``stderr``. +- `#8592 `_: The ``pytest_cmdline_preparse`` hook has been removed following its deprecation. See :ref:`the deprecation note ` for more details. - `#8733 `_: Drop a workaround for `pyreadline `__ that made it work with ``--pdb``. @@ -8095,10 +9426,10 @@ time or change existing behaviors in order to make them less surprising/more use non-ascii characters. Thanks Bruno Oliveira for the PR. - fix #1204: another error when collecting with a nasty __getattr__(). - Thanks Florian Bruhin for the PR. + Thanks Freya Bruhin for the PR. - fix the summary printed when no tests did run. - Thanks Florian Bruhin for the PR. + Thanks Freya Bruhin for the PR. - fix #1185 - ensure MANIFEST.in exactly matches what should go to a sdist - a number of documentation modernizations wrt good practices. @@ -8220,7 +9551,7 @@ time or change existing behaviors in order to make them less surprising/more use - fix issue934: when string comparison fails and a diff is too large to display without passing -vv, still show a few lines of the diff. - Thanks Florian Bruhin for the report and Bruno Oliveira for the PR. + Thanks Freya Bruhin for the report and Bruno Oliveira for the PR. - fix issue736: Fix a bug where fixture params would be discarded when combined with parametrization markers. @@ -8233,7 +9564,7 @@ time or change existing behaviors in order to make them less surprising/more use - parametrize now also generates meaningful test IDs for enum, regex and class objects (as opposed to class instances). - Thanks to Florian Bruhin for the PR. + Thanks to Freya Bruhin for the PR. - Add 'warns' to assert that warnings are thrown (like 'raises'). Thanks to Eric Hunsberger for the PR. @@ -8360,7 +9691,7 @@ time or change existing behaviors in order to make them less surprising/more use one will also have a "reprec" attribute with the recorded events/reports. - fix monkeypatch.setattr("x.y", raising=False) to actually not raise - if "y" is not a preexisting attribute. Thanks Florian Bruhin. + if "y" is not a preexisting attribute. Thanks Freya Bruhin. - fix issue741: make running output from testdir.run copy/pasteable Thanks Bruno Oliveira. @@ -8420,7 +9751,7 @@ time or change existing behaviors in order to make them less surprising/more use - fix issue833: --fixtures now shows all fixtures of collected test files, instead of just the fixtures declared on the first one. - Thanks Florian Bruhin for reporting and Bruno Oliveira for the PR. + Thanks Freya Bruhin for reporting and Bruno Oliveira for the PR. - fix issue863: skipped tests now report the correct reason when a skip/xfail condition is met when using multiple markers. diff --git a/doc/en/conf.py b/doc/en/conf.py index 9558a75f927..81156493131 100644 --- a/doc/en/conf.py +++ b/doc/en/conf.py @@ -34,6 +34,7 @@ "sphinx.ext.todo", "sphinx.ext.viewcode", "sphinx_removed_in", + "sphinx_inline_tabs", "sphinxcontrib_trio", "sphinxcontrib.towncrier.ext", # provides `towncrier-draft-entries` directive "sphinx_issues", # implements `:issue:`, `:pr:` and other GH-related roles @@ -75,6 +76,7 @@ ("py:class", "_pytest._code.code.TerminalRepr"), ("py:class", "TerminalRepr"), ("py:class", "_pytest.fixtures.FixtureFunctionMarker"), + ("py:class", "_pytest.fixtures.FixtureFunctionDefinition"), ("py:class", "_pytest.logging.LogCaptureHandler"), ("py:class", "_pytest.mark.structures.ParameterSet"), # Intentionally undocumented/private @@ -105,6 +107,8 @@ ("py:obj", "_pytest.fixtures.FixtureValue"), ("py:obj", "_pytest.stash.T"), ("py:class", "_ScopeName"), + ("py:class", "BaseExcT_1"), + ("py:class", "ExcT_1"), ] add_module_names = False diff --git a/doc/en/contact.rst b/doc/en/contact.rst index ef9d1e8edca..311224eeef0 100644 --- a/doc/en/contact.rst +++ b/doc/en/contact.rst @@ -24,19 +24,26 @@ Chat `_) - ``#pytest`` `on Matrix `_. +Microblogging +------------- + +- Bluesky: `@pytest.org `_ +- Mastodon: `@pytest@fosstodon.org `_ +- Twitter/X: `@pytestdotorg `_ + Mail ---- - `Testing In Python`_: a mailing list for Python testing tools and discussion. -- `pytest-dev at python.org`_ a mailing list for pytest specific announcements and discussions. - Mail to `core@pytest.org `_ for topics that cannot be discussed in public. Mails sent there will be distributed among the members in the pytest core team, who can also be contacted individually: - * Ronny Pfannschmidt (:user:`RonnyPfannschmidt`, `ronny@pytest.org `_) - * Florian Bruhin (:user:`The-Compiler`, `florian@pytest.org `_) * Bruno Oliveira (:user:`nicoddemus`, `bruno@pytest.org `_) + * Freya Bruhin (:user:`The-Compiler`, `freya@pytest.org `_) + * Pierre Sassoulas (:user:`Pierre-Sassoulas`, `pierre@pytest.org `_) * Ran Benita (:user:`bluetech`, `ran@pytest.org `_) + * Ronny Pfannschmidt (:user:`RonnyPfannschmidt`, `ronny@pytest.org `_) * Zac Hatfield-Dodds (:user:`Zac-HD`, `zac@pytest.org `_) Other @@ -44,10 +51,9 @@ Other - The :doc:`contribution guide ` for help on submitting pull requests to GitHub. -- Florian Bruhin (:user:`The-Compiler`) offers pytest professional teaching and +- Freya Bruhin (:user:`The-Compiler`) offers pytest professional teaching and consulting via `Bruhin Software `_. .. _`pytest issue tracker`: https://github.com/pytest-dev/pytest/issues .. _`pytest discussions`: https://github.com/pytest-dev/pytest/discussions .. _`Testing in Python`: http://lists.idyll.org/listinfo/testing-in-python -.. _`pytest-dev at python.org`: http://mail.python.org/mailman/listinfo/pytest-dev diff --git a/doc/en/deprecations.rst b/doc/en/deprecations.rst index bf6268a4980..57c583fd852 100644 --- a/doc/en/deprecations.rst +++ b/doc/en/deprecations.rst @@ -15,6 +15,111 @@ Below is a complete list of all pytest features which are considered deprecated. :class:`~pytest.PytestWarning` or subclasses, which can be filtered using :ref:`standard warning filters `. +.. _monkeypatch-fixup-namespace-packages: + +``monkeypatch.syspath_prepend`` with legacy namespace packages +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 9.0 + +When using :meth:`monkeypatch.syspath_prepend() `, +pytest automatically calls ``pkg_resources.fixup_namespace_packages()`` if ``pkg_resources`` is imported. +This is only needed for legacy namespace packages that use ``pkg_resources.declare_namespace()``. + +Legacy namespace packages are deprecated in favor of native namespace packages (:pep:`420`). +If you are using ``pkg_resources.declare_namespace()`` in your ``__init__.py`` files, +you should migrate to native namespace packages by removing the ``__init__.py`` files from your namespace packages. + +This deprecation warning will only be issued when: + +1. ``pkg_resources`` is imported, and +2. The specific path being prepended contains a declared namespace package (via ``pkg_resources.declare_namespace()``) + +To fix this warning, convert your legacy namespace packages to native namespace packages: + +**Legacy namespace package** (deprecated): + +.. code-block:: python + + # mypkg/__init__.py + __import__("pkg_resources").declare_namespace(__name__) + +**Native namespace package** (recommended): + +Simply remove the ``__init__.py`` file entirely. +Python 3.3+ natively supports namespace packages without ``__init__.py``. + + +.. _sync-test-async-fixture: + +sync test depending on async fixture +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. deprecated:: 8.4 + +Pytest has for a long time given an error when encountering an asynchronous test function, prompting the user to install +a plugin that can handle it. It has not given any errors if you have an asynchronous fixture that's depended on by a +synchronous test. If the fixture was an async function you did get an "unawaited coroutine" warning, but for async yield fixtures you didn't even get that. +This is a problem even if you do have a plugin installed for handling async tests, as they may require +special decorators for async fixtures to be handled, and some may not robustly handle if a user accidentally requests an +async fixture from their sync tests. Fixture values being cached can make this even more unintuitive, where everything will +"work" if the fixture is first requested by an async test, and then requested by a synchronous test. + +Unfortunately there is no 100% reliable method of identifying when a user has made a mistake, versus when they expect an +unawaited object from their fixture that they will handle on their own. To suppress this warning +when you in fact did intend to handle this you can wrap your async fixture in a synchronous fixture: + +.. code-block:: python + + import asyncio + import pytest + + + @pytest.fixture + async def unawaited_fixture(): + return 1 + + + def test_foo(unawaited_fixture): + assert 1 == asyncio.run(unawaited_fixture) + +should be changed to + + +.. code-block:: python + + import asyncio + import pytest + + + @pytest.fixture + def unawaited_fixture(): + async def inner_fixture(): + return 1 + + return inner_fixture() + + + def test_foo(unawaited_fixture): + assert 1 == asyncio.run(unawaited_fixture) + + +You can also make use of `pytest_fixture_setup` to handle the coroutine/asyncgen before pytest sees it - this is the way current async pytest plugins handle it. + +If a user has an async fixture with ``autouse=True`` in their ``conftest.py``, or in a file +containing both synchronous tests and the fixture, they will receive this warning. +Unless you're using a plugin that specifically handles async fixtures +with synchronous tests, we strongly recommend against this practice. +It can lead to unpredictable behavior (with larger scopes, it may appear to "work" if an async +test is the first to request the fixture, due to value caching) and will generate +unawaited-coroutine runtime warnings (but only for non-yield fixtures). +Additionally, it creates ambiguity for other developers about whether the fixture is intended to perform +setup for synchronous tests. + +The `anyio pytest plugin `_ supports +synchronous tests with async fixtures, though certain limitations apply. + + .. _import-or-skip-import-error: ``pytest.importorskip`` default behavior regarding :class:`ImportError` @@ -34,7 +139,7 @@ In ``8.2`` the ``exc_type`` parameter has been added, giving users the ability o to skip tests only if the module cannot really be found, and not because of some other error. Catching only :class:`ModuleNotFoundError` by default (and letting other errors propagate) would be the best solution, -however for backward compatibility, pytest will keep the existing behavior but raise an warning if: +however for backward compatibility, pytest will keep the existing behavior but raise a warning if: 1. The captured exception is of type :class:`ImportError`, and: 2. The user does not pass ``exc_type`` explicitly. @@ -43,7 +148,7 @@ If the import attempt raises :class:`ModuleNotFoundError` (the usual case), then warning is emitted. This way, the usual cases will keep working the same way, while unexpected errors will now issue a warning, with -users being able to supress the warning by passing ``exc_type=ImportError`` explicitly. +users being able to suppress the warning by passing ``exc_type=ImportError`` explicitly. In ``9.0``, the warning will turn into an error, and in ``9.1`` :func:`pytest.importorskip` will only capture :class:`ModuleNotFoundError` by default and no warnings will be issued anymore -- but users can still capture @@ -246,63 +351,59 @@ Users expected in this case that the ``usefixtures`` mark would have its intende Now pytest will issue a warning when it encounters this problem, and will raise an error in the future versions. -Returning non-None value in test functions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``yield_fixture`` function/decorator +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -.. deprecated:: 7.2 +.. deprecated:: 6.2 -A :class:`pytest.PytestReturnNotNoneWarning` is now emitted if a test function returns something other than `None`. +``pytest.yield_fixture`` is a deprecated alias for :func:`pytest.fixture`. -This prevents a common mistake among beginners that expect that returning a `bool` would cause a test to pass or fail, for example: +It has been so for a very long time, so it can be searched/replaced safely. -.. code-block:: python - @pytest.mark.parametrize( - ["a", "b", "result"], - [ - [1, 2, 5], - [2, 3, 8], - [5, 3, 18], - ], - ) - def test_foo(a, b, result): - return foo(a, b) == result +Removed Features and Breaking Changes +------------------------------------- -Given that pytest ignores the return value, this might be surprising that it will never fail. +As stated in our :ref:`backwards-compatibility` policy, deprecated features are removed only in major releases after +an appropriate period of deprecation has passed. -The proper fix is to change the `return` to an `assert`: +Some breaking changes which could not be deprecated are also listed. -.. code-block:: python +.. _yield tests deprecated: - @pytest.mark.parametrize( - ["a", "b", "result"], - [ - [1, 2, 5], - [2, 3, 8], - [5, 3, 18], - ], - ) - def test_foo(a, b, result): - assert foo(a, b) == result +``yield`` tests +~~~~~~~~~~~~~~~ +.. versionremoved:: 4.0 -The ``yield_fixture`` function/decorator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ``yield`` tests ``xfail``. -.. deprecated:: 6.2 +.. versionremoved:: 8.4 -``pytest.yield_fixture`` is a deprecated alias for :func:`pytest.fixture`. + ``yield`` tests raise a collection error. -It has been so for a very long time, so can be search/replaced safely. +pytest no longer supports ``yield``-style tests, where a test function actually ``yield`` functions and values +that are then turned into proper test methods. Example: +.. code-block:: python -Removed Features and Breaking Changes -------------------------------------- + def check(x, y): + assert x**x == y -As stated in our :ref:`backwards-compatibility` policy, deprecated features are removed only in major releases after -an appropriate period of deprecation has passed. -Some breaking changes which could not be deprecated are also listed. + def test_squared(): + yield check, 2, 4 + yield check, 3, 9 + +This would result in two actual test functions being generated. + +This form of test function doesn't support fixtures properly, and users should switch to ``pytest.mark.parametrize``: + +.. code-block:: python + + @pytest.mark.parametrize("x, y", [(2, 4), (3, 9)]) + def test_squared(x, y): + assert x**x == y .. _nose-deprecation: @@ -488,18 +589,20 @@ removed in pytest 8 (deprecated since pytest 2.4.0): - ``parser.addoption(..., type="int/string/float/complex")`` - use ``type=int`` etc. instead. -The ``--strict`` command-line option -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +The ``--strict`` command-line option (reintroduced) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ .. deprecated:: 6.2 -.. versionremoved:: 8.0 +.. versionchanged:: 9.0 -The ``--strict`` command-line option has been deprecated in favor of ``--strict-markers``, which +The ``--strict`` command-line option had been deprecated in favor of ``--strict-markers``, which better conveys what the option does. -We have plans to maybe in the future to reintroduce ``--strict`` and make it an encompassing -flag for all strictness related options (``--strict-markers`` and ``--strict-config`` -at the moment, more might be introduced in the future). +In version 8.1, we accidentally un-deprecated ``--strict``. + +In version 9.0, we changed ``--strict`` to make it set the new :confval:`strict` +configuration option. It now enables all strictness related options (including +:confval:`strict_markers`). .. _cmdline-preparse-deprecated: @@ -671,7 +774,7 @@ The ``pytest._fillfuncargs`` function This function was kept for backward compatibility with an older plugin. -It's functionality is not meant to be used directly, but if you must replace +Its functionality is not meant to be used directly, but if you must replace it, use `function._request._fillfixtures()` instead, though note this is not a public API and may break in the future. @@ -702,7 +805,7 @@ The ``--result-log`` option produces a stream of test reports which can be analysed at runtime, but it uses a custom format which requires users to implement their own parser. -The `pytest-reportlog `__ plugin provides a ``--report-log`` option, a more standard and extensible alternative, producing +The :pypi:`pytest-reportlog` plugin provides a ``--report-log`` option, a more standard and extensible alternative, producing one JSON object per-line, and should cover the same use cases. Please try it out and provide feedback. The ``pytest-reportlog`` plugin might even be merged into the core @@ -742,20 +845,38 @@ that manipulate this type of file (for example, Jenkins, Azure Pipelines, etc.). Users are recommended to try the new ``xunit2`` format and see if their tooling that consumes the JUnit XML file supports it. -To use the new format, update your ``pytest.ini``: +To use the new format, update your configuration file: + +.. tab:: toml + + .. code-block:: toml -.. code-block:: ini + [pytest] + junit_family = "xunit2" - [pytest] - junit_family=xunit2 +.. tab:: ini + + .. code-block:: ini + + [pytest] + junit_family = xunit2 If you discover that your tooling does not support the new format, and want to keep using the legacy version, set the option to ``legacy`` instead: -.. code-block:: ini +.. tab:: toml + + .. code-block:: toml + + [pytest] + junit_family = "legacy" + +.. tab:: ini - [pytest] - junit_family=legacy + .. code-block:: ini + + [pytest] + junit_family = legacy By using ``legacy`` you will keep using the legacy/xunit1 format when upgrading to pytest 6.0, where the default format will be ``xunit2``. @@ -1200,36 +1321,6 @@ with the ``name`` parameter: return cell() -.. _yield tests deprecated: - -``yield`` tests -~~~~~~~~~~~~~~~ - -.. versionremoved:: 4.0 - -pytest supported ``yield``-style tests, where a test function actually ``yield`` functions and values -that are then turned into proper test methods. Example: - -.. code-block:: python - - def check(x, y): - assert x**x == y - - - def test_squared(): - yield check, 2, 4 - yield check, 3, 9 - -This would result into two actual test functions being generated. - -This form of test function doesn't support fixtures properly, and users should switch to ``pytest.mark.parametrize``: - -.. code-block:: python - - @pytest.mark.parametrize("x, y", [(2, 4), (3, 9)]) - def test_squared(x, y): - assert x**x == y - .. _internal classes accessed through node deprecated: Internal classes accessed through ``Node`` diff --git a/doc/en/example/.ruff.toml b/doc/en/example/.ruff.toml new file mode 100644 index 00000000000..feddc5c0654 --- /dev/null +++ b/doc/en/example/.ruff.toml @@ -0,0 +1 @@ +lint.ignore = ["RUF059"] diff --git a/doc/en/example/assertion/failure_demo.py b/doc/en/example/assertion/failure_demo.py index dd1485b0b21..16a578fda12 100644 --- a/doc/en/example/assertion/failure_demo.py +++ b/doc/en/example/assertion/failure_demo.py @@ -267,9 +267,9 @@ class A: a = 1 b = 2 - assert ( - A.a == b - ), "A.a appears not to be b\nor does not appear to be b\none of those" + assert A.a == b, ( + "A.a appears not to be b\nor does not appear to be b\none of those" + ) def test_custom_repr(self): class JSON: diff --git a/doc/en/example/attic.rst b/doc/en/example/attic.rst index 2b1f2766dce..3a2e228337e 100644 --- a/doc/en/example/attic.rst +++ b/doc/en/example/attic.rst @@ -75,7 +75,7 @@ decorate its result. This mechanism allows us to stay ignorant of how/where the function argument is provided - in our example from a `conftest plugin`_. -sidenote: the temporary directory used here are instances of +Side note: the temporary directories used here are instances of the `py.path.local`_ class which provides many of the os.path methods in a convenient way. diff --git a/doc/en/example/customdirectory.rst b/doc/en/example/customdirectory.rst index 1e4d7e370de..705a3373654 100644 --- a/doc/en/example/customdirectory.rst +++ b/doc/en/example/customdirectory.rst @@ -36,13 +36,13 @@ You can create a ``manifest.json`` file and some test files: .. include:: customdirectory/tests/test_third.py :literal: -An you can now execute the test specification: +And you can now execute the test specification: .. code-block:: pytest customdirectory $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project/customdirectory configfile: pytest.ini collected 2 items @@ -62,7 +62,7 @@ You can verify that your custom collector appears in the collection tree: customdirectory $ pytest --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project/customdirectory configfile: pytest.ini collected 2 items diff --git a/doc/en/example/markers.rst b/doc/en/example/markers.rst index babcd9e2f3a..c8e4172a696 100644 --- a/doc/en/example/markers.rst +++ b/doc/en/example/markers.rst @@ -47,7 +47,7 @@ You can then restrict a test run to only run tests marked with ``webtest``: $ pytest -v -m webtest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 3 deselected / 1 selected @@ -62,7 +62,7 @@ Or the inverse, running all tests except the webtest ones: $ pytest -v -m "not webtest" =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 1 deselected / 3 selected @@ -82,7 +82,7 @@ keyword arguments, e.g. to run only tests marked with ``device`` and the specifi $ pytest -v -m "device(serial='123')" =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 3 deselected / 1 selected @@ -106,7 +106,7 @@ tests based on their module, class, method, or function name: $ pytest -v test_server.py::TestClass::test_method =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 1 item @@ -121,7 +121,7 @@ You can also select on the class: $ pytest -v test_server.py::TestClass =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 1 item @@ -136,7 +136,7 @@ Or select multiple nodes: $ pytest -v test_server.py::TestClass test_server.py::test_send_http =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 2 items @@ -167,9 +167,9 @@ Using ``-k expr`` to select tests based on their name .. versionadded:: 2.0/2.3.4 -You can use the ``-k`` command line option to specify an expression +You can use the :option:`-k` command line option to specify an expression which implements a substring match on the test names instead of the -exact match on markers that ``-m`` provides. This makes it easy to +exact match on markers that :option:`-m` provides. This makes it easy to select tests based on their names: .. versionchanged:: 5.4 @@ -180,7 +180,7 @@ The expression matching is now case-insensitive. $ pytest -v -k http # running with the above defined example module =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 3 deselected / 1 selected @@ -195,7 +195,7 @@ And you can also run all tests except the ones that match the keyword: $ pytest -k "not send_http" -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 1 deselected / 3 selected @@ -212,7 +212,7 @@ Or to select "http" and "quick" tests: $ pytest -k "http or quick" -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 4 items / 2 deselected / 2 selected @@ -225,7 +225,7 @@ Or to select "http" and "quick" tests: You can use ``and``, ``or``, ``not`` and parentheses. -In addition to the test's name, ``-k`` also matches the names of the test's parents (usually, the name of the file and class it's in), +In addition to the test's name, :option:`-k` also matches the names of the test's parents (usually, the name of the file and class it's in), attributes set on the test function, markers applied to it or its parents and any :attr:`extra keywords <_pytest.nodes.Node.extra_keyword_matches>` explicitly added to it or its parents. @@ -239,13 +239,11 @@ Registering markers Registering markers for your test suite is simple: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - markers = - webtest: mark a test as a webtest. - slow: mark test as slow. + markers = ["webtest: mark a test as a webtest.", "slow: mark test as slow."] Multiple custom markers can be registered, by defining each one in its own line, as shown in above example. @@ -264,7 +262,7 @@ You can ask which markers exist for your test suite - the list includes our just @pytest.mark.skipif(condition, ..., *, reason=...): skip the given test function if any of the conditions evaluate to True. Example: skipif(sys.platform == 'win32') skips the test if we are on the win32 platform. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-skipif - @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail + @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=strict_xfail): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail @pytest.mark.parametrize(argnames, argvalues): call a test function multiple times passing in different arguments in turn. argvalues generally needs to be a list of values if argnames specifies only one name or a list of tuples of values if argnames specifies multiple names. Example: @parametrize('arg1', [1,2]) would lead to two calls of the decorated test function, one with arg1=1 and another with arg1=2.see https://docs.pytest.org/en/stable/how-to/parametrize.html for more info and examples. @@ -286,8 +284,7 @@ For an example on how to add and work with markers from a plugin, see * Asking for existing markers via ``pytest --markers`` gives good output - * Typos in function markers are treated as an error if you use - the ``--strict-markers`` option. + * Typos in function markers are treated as an error if you use the :confval:`strict_markers` configuration option. .. _`scoped-marking`: @@ -414,14 +411,14 @@ A test file using this local plugin: def test_basic_db_operation(): pass -and an example invocations specifying a different environment than what +and an example invocation specifying a different environment than what the test needs: .. code-block:: pytest $ pytest -E stage2 =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -435,7 +432,7 @@ and here is one that specifies exactly the environment needed: $ pytest -E stage1 =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -443,7 +440,7 @@ and here is one that specifies exactly the environment needed: ============================ 1 passed in 0.12s ============================= -The ``--markers`` option always gives you a list of available markers: +The :option:`--markers` option always gives you a list of available markers: .. code-block:: pytest @@ -456,7 +453,7 @@ The ``--markers`` option always gives you a list of available markers: @pytest.mark.skipif(condition, ..., *, reason=...): skip the given test function if any of the conditions evaluate to True. Example: skipif(sys.platform == 'win32') skips the test if we are on the win32 platform. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-skipif - @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail + @pytest.mark.xfail(condition, ..., *, reason=..., run=True, raises=None, strict=strict_xfail): mark the test function as an expected failure if any of the conditions evaluate to True. Optionally specify a reason for better reporting and run=False if you don't even want to execute the test function. If only specific exception(s) are expected, you can list them in raises, and if the test fails in other ways, it will be reported as a true failure. See https://docs.pytest.org/en/stable/reference/reference.html#pytest-mark-xfail @pytest.mark.parametrize(argnames, argvalues): call a test function multiple times passing in different arguments in turn. argvalues generally needs to be a list of values if argnames specifies only one name or a list of tuples of values if argnames specifies multiple names. Example: @parametrize('arg1', [1,2]) would lead to two calls of the decorated test function, one with arg1=1 and another with arg1=2.see https://docs.pytest.org/en/stable/how-to/parametrize.html for more info and examples. @@ -628,7 +625,7 @@ then you will see two tests skipped and two executed tests as expected: $ pytest -rs # this option reports skip reasons =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items @@ -644,7 +641,7 @@ Note that if you specify a platform via the marker-command line option like this $ pytest -m linux =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items / 3 deselected / 1 selected @@ -661,7 +658,7 @@ Automatically adding markers based on test names If you have a test suite where test function names indicate a certain type of test, you can implement a hook that automatically defines -markers so that you can use the ``-m`` option with it. Let's look +markers so that you can use the :option:`-m` option with it. Let's look at this test module: .. code-block:: python @@ -707,7 +704,7 @@ We can now use the ``-m option`` to select one set: $ pytest -m interface --tb=short =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items / 2 deselected / 2 selected @@ -733,7 +730,7 @@ or to select both "event" and "interface" tests: $ pytest -m "interface or event" --tb=short =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items / 1 deselected / 3 selected diff --git a/doc/en/example/multipython.py b/doc/en/example/multipython.py index f54524213bc..c04a2868812 100644 --- a/doc/en/example/multipython.py +++ b/doc/en/example/multipython.py @@ -10,7 +10,7 @@ import pytest -pythonlist = ["python3.9", "python3.10", "python3.11"] +pythonlist = ["python3.11", "python3.12", "python3.13"] @pytest.fixture(params=pythonlist) diff --git a/doc/en/example/nonpython.rst b/doc/en/example/nonpython.rst index aa463e2416b..54391d72fd4 100644 --- a/doc/en/example/nonpython.rst +++ b/doc/en/example/nonpython.rst @@ -28,7 +28,7 @@ now execute the test specification: nonpython $ pytest test_simple.yaml =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project/nonpython collected 2 items @@ -40,7 +40,7 @@ now execute the test specification: spec failed: 'some': 'other' no further details known at this point. ========================= short test summary info ========================== - FAILED test_simple.yaml::hello + FAILED test_simple.yaml::hello - usecase execution failed ======================= 1 failed, 1 passed in 0.12s ======================== .. regendoc:wipe @@ -58,13 +58,18 @@ your own domain specific testing language this way. will be reported as a (red) string. ``reportinfo()`` is used for representing the test location and is also -consulted when reporting in ``verbose`` mode: +consulted when reporting in ``verbose`` mode. It should return a tuple +``(path, lineno, description)``, where: + +* ``path`` is the path shown in reports (usually ``self.path`` or ``self.fspath``). +* ``lineno`` is a zero-based line number, or ``0`` when no specific line applies. +* ``description`` is a short label shown for the collected item: .. code-block:: pytest nonpython $ pytest -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project/nonpython collecting ... collected 2 items @@ -78,7 +83,7 @@ consulted when reporting in ``verbose`` mode: spec failed: 'some': 'other' no further details known at this point. ========================= short test summary info ========================== - FAILED test_simple.yaml::hello + FAILED test_simple.yaml::hello - usecase execution failed ======================= 1 failed, 1 passed in 0.12s ======================== .. regendoc:wipe @@ -90,7 +95,7 @@ interesting to just look at the collection tree: nonpython $ pytest --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project/nonpython collected 2 items diff --git a/doc/en/example/parametrize.rst b/doc/en/example/parametrize.rst index d540bf08337..ae64a7c62d5 100644 --- a/doc/en/example/parametrize.rst +++ b/doc/en/example/parametrize.rst @@ -4,7 +4,7 @@ Parametrizing tests ================================================= -``pytest`` allows to easily parametrize test functions. +``pytest`` allows you to easily parametrize test functions. For basic docs, see :ref:`parametrize-basics`. In the following we provide some examples using @@ -83,9 +83,9 @@ Different options for test IDs ------------------------------------ pytest will build a string that is the test ID for each set of values in a -parametrized test. These IDs can be used with ``-k`` to select specific cases +parametrized test. These IDs can be used with :option:`-k` to select specific cases to run, and they will also identify the specific case when one is failing. -Running pytest with ``--collect-only`` will show the generated IDs. +Running pytest with :option:`--collect-only` will show the generated IDs. Numbers, strings, booleans and None will have their usual string representation used in the test ID. For other objects, pytest will make a string based on @@ -158,11 +158,11 @@ objects, they are still using the default pytest representation: $ pytest test_time.py --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 8 items - + @@ -221,7 +221,7 @@ this is a fully self-contained example which you can run with: $ pytest test_scenarios.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items @@ -235,11 +235,11 @@ If you just collect tests you'll also nicely see 'advanced' and 'basic' as varia $ pytest --collect-only test_scenarios.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items - + @@ -314,11 +314,11 @@ Let's first see how it looks like at collection time: $ pytest test_backends.py --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items - + @@ -352,7 +352,7 @@ The first invocation with ``db == "DB1"`` passed while the second with ``db == " Indirect parametrization --------------------------------------------------- -Using the ``indirect=True`` parameter when parametrizing a test allows to +Using the ``indirect=True`` parameter when parametrizing a test allows one to parametrize a test with a fixture receiving the values before passing them to a test: @@ -413,7 +413,7 @@ The result of this test will be successful: $ pytest -v test_indirect_list.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 1 item @@ -503,11 +503,10 @@ Running it results in some skips if we don't have all the python interpreters in .. code-block:: pytest . $ pytest -rs -q multipython.py - ssssssssssss...ssssssssssss [100%] + ssssssssssss......sss...... [100%] ========================= short test summary info ========================== - SKIPPED [12] multipython.py:65: 'python3.9' not found - SKIPPED [12] multipython.py:65: 'python3.11' not found - 3 passed, 24 skipped in 0.12s + SKIPPED [15] multipython.py:67: 'python3.11' not found + 12 passed, 15 skipped in 0.12s Parametrization of optional implementations/imports --------------------------------------------------- @@ -567,7 +566,7 @@ If you run this with reporting for skips enabled: $ pytest -rs test_module.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -628,7 +627,7 @@ Then run ``pytest`` with verbose mode and with only the ``basic`` marker: $ pytest -v -m basic =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 24 items / 21 deselected / 3 selected @@ -643,7 +642,7 @@ As the result: - Four tests were collected - One test was deselected because it doesn't have the ``basic`` mark. -- Three tests with the ``basic`` mark was selected. +- Three tests with the ``basic`` mark were selected. - The test ``test_eval[1+7-8]`` passed, but the name is autogenerated and confusing. - The test ``test_eval[basic_2+4]`` passed. - The test ``test_eval[basic_6*9]`` was expected to fail and did fail. @@ -686,5 +685,5 @@ For example: assert (6 / example_input) == e In the example above, the first three test cases should run without any -exceptions, while the fourth should raise a``ZeroDivisionError`` exception, +exceptions, while the fourth should raise a ``ZeroDivisionError`` exception, which is expected by pytest. diff --git a/doc/en/example/pythoncollection.rst b/doc/en/example/pythoncollection.rst index 39b799ed934..48ee2c8533f 100644 --- a/doc/en/example/pythoncollection.rst +++ b/doc/en/example/pythoncollection.rst @@ -5,7 +5,7 @@ Ignore paths during test collection ----------------------------------- You can easily ignore certain test directories and modules during collection -by passing the ``--ignore=path`` option on the cli. ``pytest`` allows multiple +by passing the :option:`--ignore=path` option on the cli. ``pytest`` allows multiple ``--ignore`` options. Example: .. code-block:: text @@ -43,18 +43,20 @@ you will see that ``pytest`` only collects test-modules, which do not match the ========================= 5 passed in 0.02 seconds ========================= -The ``--ignore-glob`` option allows to ignore test file paths based on Unix shell-style wildcards. -If you want to exclude test-modules that end with ``_01.py``, execute ``pytest`` with ``--ignore-glob='*_01.py'``. +The :option:`--ignore-glob` option allows to ignore test file paths based on Unix shell-style wildcards. +If you want to exclude test-modules that end with ``_01.py``, execute ``pytest`` with :option:`--ignore-glob='*_01.py'`. Deselect tests during test collection ------------------------------------- -Tests can individually be deselected during collection by passing the ``--deselect=item`` option. +Tests can individually be deselected during collection by passing the :option:`--deselect=item` option. For example, say ``tests/foobar/test_foobar_01.py`` contains ``test_a`` and ``test_b``. You can run all of the tests within ``tests/`` *except* for ``tests/foobar/test_foobar_01.py::test_a`` -by invoking ``pytest`` with ``--deselect tests/foobar/test_foobar_01.py::test_a``. +by invoking ``pytest`` with ``--deselect=tests/foobar/test_foobar_01.py::test_a``. ``pytest`` allows multiple ``--deselect`` options. +.. _duplicate-paths: + Keeping duplicate paths specified from command line ---------------------------------------------------- @@ -71,7 +73,7 @@ Example: Just collect tests once. -To collect duplicate tests, use the ``--keep-duplicates`` option on the cli. +To collect duplicate tests, use the :option:`--keep-duplicates` option on the cli. Example: .. code-block:: pytest @@ -82,29 +84,17 @@ Example: collected 2 items ... -As the collector just works on directories, if you specify twice a single test file, ``pytest`` will -still collect it twice, no matter if the ``--keep-duplicates`` is not specified. -Example: - -.. code-block:: pytest - - pytest test_a.py test_a.py - - ... - collected 2 items - ... - Changing directory recursion ----------------------------------------------------- -You can set the :confval:`norecursedirs` option in an ini-file, for example your ``pytest.ini`` in the project root directory: +You can set the :confval:`norecursedirs` option in a configuration file: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - norecursedirs = .svn _build tmp* + norecursedirs = [".svn", "_build", "tmp*"] This would tell ``pytest`` to not recurse into typical subversion or sphinx-build directories or into any ``tmp`` prefixed directory. @@ -118,14 +108,14 @@ the :confval:`python_files`, :confval:`python_classes` and :confval:`python_functions` in your :ref:`configuration file `. Here is an example: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml # Example 1: have pytest look for "check" instead of "test" [pytest] - python_files = check_*.py - python_classes = Check - python_functions = *_check + python_files = ["check_*.py"] + python_classes = ["Check"] + python_functions = ["*_check"] This would make ``pytest`` look for tests in files that match the ``check_* .py`` glob-pattern, ``Check`` prefixes in classes, and functions and methods @@ -147,12 +137,12 @@ The test collection would look like this: $ pytest --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project - configfile: pytest.ini + configfile: pytest.toml collected 2 items - + @@ -162,23 +152,23 @@ The test collection would look like this: You can check for multiple glob patterns by adding a space between the patterns: -.. code-block:: ini +.. code-block:: toml + # content of pytest.toml # Example 2: have pytest look for files with "test" and "example" - # content of pytest.ini [pytest] - python_files = test_*.py example_*.py + python_files = ["test_*.py", "example_*.py"] .. note:: - the ``python_functions`` and ``python_classes`` options has no effect + the ``python_functions`` and ``python_classes`` options have no effect for ``unittest.TestCase`` test discovery because pytest delegates discovery of test case methods to unittest code. Interpreting cmdline arguments as Python packages ----------------------------------------------------- -You can use the ``--pyargs`` option to make ``pytest`` try +You can use the :option:`--pyargs` option to make ``pytest`` try interpreting arguments as python package names, deriving their file system path and then running the test. For example if you have unittest2 installed you can type: @@ -188,14 +178,14 @@ example if you have unittest2 installed you can type: pytest --pyargs unittest2.test.test_skipping -q which would run the respective test module. Like with -other options, through an ini-file and the :confval:`addopts` option you +other options, through a configuration file and the :confval:`addopts` option you can make this change more permanently: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - addopts = --pyargs + addopts = ["--pyargs"] Now a simple invocation of ``pytest NAME`` will check if NAME exists as an importable package/module and otherwise @@ -210,12 +200,12 @@ You can always peek at the collection tree without running tests like this: . $ pytest --collect-only pythoncollection.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project - configfile: pytest.ini + configfile: pytest.toml collected 3 items - + @@ -234,14 +224,14 @@ Customizing test collection You can easily instruct ``pytest`` to discover tests from every Python file: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - python_files = *.py + python_files = ["*.py"] However, many projects will have a ``setup.py`` which they don't want to be -imported. Moreover, there may files only importable by a specific python +imported. Moreover, there may be files only importable by a specific python version. For such cases you can dynamically define files to be ignored by listing them in a ``conftest.py`` file: @@ -294,9 +284,9 @@ file will be left out: $ pytest --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project - configfile: pytest.ini + configfile: pytest.toml collected 0 items ======================= no tests collected in 0.12s ======================== @@ -325,3 +315,30 @@ with ``Test`` by setting a boolean ``__test__`` attribute to ``False``. # Will not be discovered as a test class TestClass: __test__ = False + +.. note:: + + If you are working with abstract test classes and want to avoid manually setting + the ``__test__`` attribute for subclasses, you can use a mixin class to handle + this automatically. For example: + + .. code-block:: python + + # Mixin to handle abstract test classes + class NotATest: + def __init_subclass__(cls): + cls.__test__ = NotATest not in cls.__bases__ + + + # Abstract test class + class AbstractTest(NotATest): + pass + + + # Subclass that will be collected as a test + class RealTest(AbstractTest): + def test_example(self): + assert 1 + 1 == 2 + + This approach ensures that subclasses of abstract test classes are automatically + collected without needing to explicitly set the ``__test__`` attribute. diff --git a/doc/en/example/reportingdemo.rst b/doc/en/example/reportingdemo.rst index 2c34cc2b00d..29ba190b7e7 100644 --- a/doc/en/example/reportingdemo.rst +++ b/doc/en/example/reportingdemo.rst @@ -9,7 +9,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: assertion $ pytest failure_demo.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project/assertion collected 44 items @@ -25,7 +25,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert param1 * 2 < param2 E assert (3 * 2) < 6 - failure_demo.py:19: AssertionError + failure_demo.py:21: AssertionError _________________________ TestFailing.test_simple __________________________ self = @@ -42,7 +42,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + where 42 = .f at 0xdeadbeef0002>() E + and 43 = .g at 0xdeadbeef0003>() - failure_demo.py:30: AssertionError + failure_demo.py:32: AssertionError ____________________ TestFailing.test_simple_multiline _____________________ self = @@ -50,7 +50,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_simple_multiline(self): > otherfunc_multi(42, 6 * 9) - failure_demo.py:33: + failure_demo.py:35: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ a = 42, b = 54 @@ -59,7 +59,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert a == b E assert 42 == 54 - failure_demo.py:14: AssertionError + failure_demo.py:16: AssertionError ___________________________ TestFailing.test_not ___________________________ self = @@ -72,7 +72,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert not 42 E + where 42 = .f at 0xdeadbeef0006>() - failure_demo.py:39: AssertionError + failure_demo.py:41: AssertionError _________________ TestSpecialisedExplanations.test_eq_text _________________ self = @@ -84,7 +84,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E - eggs E + spam - failure_demo.py:44: AssertionError + failure_demo.py:46: AssertionError _____________ TestSpecialisedExplanations.test_eq_similar_text _____________ self = @@ -98,7 +98,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + foo 1 bar E ? ^ - failure_demo.py:47: AssertionError + failure_demo.py:49: AssertionError ____________ TestSpecialisedExplanations.test_eq_multiline_text ____________ self = @@ -112,7 +112,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + spam E bar - failure_demo.py:50: AssertionError + failure_demo.py:52: AssertionError ______________ TestSpecialisedExplanations.test_eq_long_text _______________ self = @@ -130,7 +130,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + 1111111111a222222222 E ? ^ - failure_demo.py:55: AssertionError + failure_demo.py:57: AssertionError _________ TestSpecialisedExplanations.test_eq_long_text_multiline __________ self = @@ -150,7 +150,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E E ...Full output truncated (7 lines hidden), use '-vv' to show - failure_demo.py:60: AssertionError + failure_demo.py:62: AssertionError _________________ TestSpecialisedExplanations.test_eq_list _________________ self = @@ -162,7 +162,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E At index 2 diff: 2 != 3 E Use -v to get more diff - failure_demo.py:63: AssertionError + failure_demo.py:65: AssertionError ______________ TestSpecialisedExplanations.test_eq_list_long _______________ self = @@ -176,7 +176,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E At index 100 diff: 1 != 2 E Use -v to get more diff - failure_demo.py:68: AssertionError + failure_demo.py:70: AssertionError _________________ TestSpecialisedExplanations.test_eq_dict _________________ self = @@ -194,7 +194,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E {'d': 0} E Use -v to get more diff - failure_demo.py:71: AssertionError + failure_demo.py:73: AssertionError _________________ TestSpecialisedExplanations.test_eq_set __________________ self = @@ -212,7 +212,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E 21 E Use -v to get more diff - failure_demo.py:74: AssertionError + failure_demo.py:76: AssertionError _____________ TestSpecialisedExplanations.test_eq_longer_list ______________ self = @@ -224,7 +224,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E Right contains one more item: 3 E Use -v to get more diff - failure_demo.py:77: AssertionError + failure_demo.py:79: AssertionError _________________ TestSpecialisedExplanations.test_in_list _________________ self = @@ -233,7 +233,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert 1 in [0, 2, 3, 4, 5] E assert 1 in [0, 2, 3, 4, 5] - failure_demo.py:80: AssertionError + failure_demo.py:82: AssertionError __________ TestSpecialisedExplanations.test_not_in_text_multiline __________ self = @@ -252,7 +252,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E and a E tail - failure_demo.py:84: AssertionError + failure_demo.py:86: AssertionError ___________ TestSpecialisedExplanations.test_not_in_text_single ____________ self = @@ -266,7 +266,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E single foo line E ? +++ - failure_demo.py:88: AssertionError + failure_demo.py:90: AssertionError _________ TestSpecialisedExplanations.test_not_in_text_single_long _________ self = @@ -280,7 +280,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E head head foo tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail E ? +++ - failure_demo.py:92: AssertionError + failure_demo.py:94: AssertionError ______ TestSpecialisedExplanations.test_not_in_text_single_long_term _______ self = @@ -294,7 +294,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E head head fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffftail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail tail E ? ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - failure_demo.py:96: AssertionError + failure_demo.py:98: AssertionError ______________ TestSpecialisedExplanations.test_eq_dataclass _______________ self = @@ -321,7 +321,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E - c E + b - failure_demo.py:108: AssertionError + failure_demo.py:110: AssertionError ________________ TestSpecialisedExplanations.test_eq_attrs _________________ self = @@ -348,7 +348,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E - c E + b - failure_demo.py:120: AssertionError + failure_demo.py:122: AssertionError ______________________________ test_attribute ______________________________ def test_attribute(): @@ -360,7 +360,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert 1 == 2 E + where 1 = .Foo object at 0xdeadbeef0018>.b - failure_demo.py:128: AssertionError + failure_demo.py:130: AssertionError _________________________ test_attribute_instance __________________________ def test_attribute_instance(): @@ -372,7 +372,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + where 1 = .Foo object at 0xdeadbeef0019>.b E + where .Foo object at 0xdeadbeef0019> = .Foo'>() - failure_demo.py:135: AssertionError + failure_demo.py:137: AssertionError __________________________ test_attribute_failure __________________________ def test_attribute_failure(): @@ -384,8 +384,9 @@ Here is a nice run of several failures and how ``pytest`` presents things: i = Foo() > assert i.b == 2 + ^^^ - failure_demo.py:146: + failure_demo.py:148: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = .Foo object at 0xdeadbeef001a> @@ -394,7 +395,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > raise Exception("Failed to get attrib") E Exception: Failed to get attrib - failure_demo.py:141: Exception + failure_demo.py:143: Exception _________________________ test_attribute_multiple __________________________ def test_attribute_multiple(): @@ -411,7 +412,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + and 2 = .Bar object at 0xdeadbeef001c>.b E + where .Bar object at 0xdeadbeef001c> = .Bar'>() - failure_demo.py:156: AssertionError + failure_demo.py:158: AssertionError __________________________ TestRaises.test_raises __________________________ self = @@ -421,7 +422,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > raises(TypeError, int, s) E ValueError: invalid literal for int() with base 10: 'qwe' - failure_demo.py:166: ValueError + failure_demo.py:168: ValueError ______________________ TestRaises.test_raises_doesnt _______________________ self = @@ -430,7 +431,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > raises(OSError, int, "3") E Failed: DID NOT RAISE - failure_demo.py:169: Failed + failure_demo.py:171: Failed __________________________ TestRaises.test_raise ___________________________ self = @@ -439,16 +440,17 @@ Here is a nice run of several failures and how ``pytest`` presents things: > raise ValueError("demo error") E ValueError: demo error - failure_demo.py:172: ValueError + failure_demo.py:174: ValueError ________________________ TestRaises.test_tupleerror ________________________ self = def test_tupleerror(self): > a, b = [1] # noqa: F841 + ^^^^ E ValueError: not enough values to unpack (expected 2, got 1) - failure_demo.py:175: ValueError + failure_demo.py:177: ValueError ______ TestRaises.test_reinterpret_fails_with_print_for_the_fun_of_it ______ self = @@ -457,9 +459,10 @@ Here is a nice run of several failures and how ``pytest`` presents things: items = [1, 2, 3] print(f"items is {items!r}") > a, b = items.pop() + ^^^^ E TypeError: cannot unpack non-iterable int object - failure_demo.py:180: TypeError + failure_demo.py:182: TypeError --------------------------- Captured stdout call --------------------------- items is [1, 2, 3] ________________________ TestRaises.test_some_error ________________________ @@ -468,9 +471,10 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_some_error(self): > if namenotexi: # noqa: F821 + ^^^^^^^^^^ E NameError: name 'namenotexi' is not defined - failure_demo.py:183: NameError + failure_demo.py:185: NameError ____________________ test_dynamic_compile_shows_nicely _____________________ def test_dynamic_compile_shows_nicely(): @@ -486,7 +490,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: sys.modules[name] = module > module.foo() - failure_demo.py:202: + failure_demo.py:204: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ > ??? @@ -506,9 +510,9 @@ Here is a nice run of several failures and how ``pytest`` presents things: > somefunc(f(), g()) - failure_demo.py:213: + failure_demo.py:215: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ - failure_demo.py:10: in somefunc + failure_demo.py:12: in somefunc otherfunc(x, y) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ @@ -518,7 +522,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert a == b E assert 44 == 43 - failure_demo.py:6: AssertionError + failure_demo.py:8: AssertionError ___________________ TestMoreErrors.test_z1_unpack_error ____________________ self = @@ -526,9 +530,10 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_z1_unpack_error(self): items = [] > a, b = items + ^^^^ E ValueError: not enough values to unpack (expected 2, got 0) - failure_demo.py:217: ValueError + failure_demo.py:219: ValueError ____________________ TestMoreErrors.test_z2_type_error _____________________ self = @@ -536,9 +541,10 @@ Here is a nice run of several failures and how ``pytest`` presents things: def test_z2_type_error(self): items = 3 > a, b = items + ^^^^ E TypeError: cannot unpack non-iterable int object - failure_demo.py:221: TypeError + failure_demo.py:223: TypeError ______________________ TestMoreErrors.test_startswith ______________________ self = @@ -551,7 +557,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + where False = ('456') E + where = '123'.startswith - failure_demo.py:226: AssertionError + failure_demo.py:228: AssertionError __________________ TestMoreErrors.test_startswith_nested ___________________ self = @@ -570,7 +576,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + where '123' = .f at 0xdeadbeef0029>() E + and '456' = .g at 0xdeadbeef002a>() - failure_demo.py:235: AssertionError + failure_demo.py:237: AssertionError _____________________ TestMoreErrors.test_global_func ______________________ self = @@ -581,7 +587,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E + where False = isinstance(43, float) E + where 43 = globf(42) - failure_demo.py:238: AssertionError + failure_demo.py:240: AssertionError _______________________ TestMoreErrors.test_instance _______________________ self = @@ -592,7 +598,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert 42 != 42 E + where 42 = .x - failure_demo.py:242: AssertionError + failure_demo.py:244: AssertionError _______________________ TestMoreErrors.test_compare ________________________ self = @@ -602,7 +608,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert 11 < 5 E + where 11 = globf(10) - failure_demo.py:245: AssertionError + failure_demo.py:247: AssertionError _____________________ TestMoreErrors.test_try_finally ______________________ self = @@ -613,7 +619,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: > assert x == 0 E assert 1 == 0 - failure_demo.py:250: AssertionError + failure_demo.py:252: AssertionError ___________________ TestCustomAssertMsg.test_single_line ___________________ self = @@ -628,7 +634,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert 1 == 2 E + where 1 = .A'>.a - failure_demo.py:261: AssertionError + failure_demo.py:263: AssertionError ____________________ TestCustomAssertMsg.test_multiline ____________________ self = @@ -638,16 +644,16 @@ Here is a nice run of several failures and how ``pytest`` presents things: a = 1 b = 2 - > assert ( - A.a == b - ), "A.a appears not to be b\nor does not appear to be b\none of those" + > assert A.a == b, ( + "A.a appears not to be b\nor does not appear to be b\none of those" + ) E AssertionError: A.a appears not to be b E or does not appear to be b E one of those E assert 1 == 2 E + where 1 = .A'>.a - failure_demo.py:268: AssertionError + failure_demo.py:270: AssertionError ___________________ TestCustomAssertMsg.test_custom_repr ___________________ self = @@ -669,7 +675,7 @@ Here is a nice run of several failures and how ``pytest`` presents things: E assert 1 == 2 E + where 1 = This is JSON\n{\n 'foo': 'bar'\n}.a - failure_demo.py:281: AssertionError + failure_demo.py:283: AssertionError ========================= short test summary info ========================== FAILED failure_demo.py::test_generative[3-6] - assert (3 * 2) < 6 FAILED failure_demo.py::TestFailing::test_simple - assert 42 == 43 diff --git a/doc/en/example/simple.rst b/doc/en/example/simple.rst index d4ace3f0413..a07927280ae 100644 --- a/doc/en/example/simple.rst +++ b/doc/en/example/simple.rst @@ -11,12 +11,11 @@ every time you use ``pytest``. For example, if you always want to see detailed info on skipped and xfailed tests, as well as have terser "dot" progress output, you can write it into a configuration file: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - addopts = -ra -q - + addopts = ["-ra", "-q"] Alternatively, you can set a ``PYTEST_ADDOPTS`` environment variable to add command line options while the environment is in use: @@ -29,7 +28,7 @@ Here's how the command-line is built in the presence of ``addopts`` or the envir .. code-block:: text - $PYTEST_ADDOPTS + $PYTEST_ADDOPTS So if the user executes in the command-line: @@ -44,7 +43,7 @@ The actual command line executed is: pytest -ra -q -v -m slow Note that as usual for other command-line applications, in case of conflicting options the last one wins, so the example -above will show verbose output because ``-v`` overwrites ``-q``. +above will show verbose output because :option:`-v` overwrites :option:`-q`. .. _request example: @@ -104,6 +103,7 @@ Let's run this without supplying our new option: elif cmdopt == "type2": print("second") > assert 0 # to see what was printed + ^^^^^^^^ E assert 0 test_sample.py:6: AssertionError @@ -130,6 +130,7 @@ And now with supplying a command line option: elif cmdopt == "type2": print("second") > assert 0 # to see what was printed + ^^^^^^^^ E assert 0 test_sample.py:6: AssertionError @@ -164,7 +165,9 @@ Now we'll get feedback on a bad argument: $ pytest -q --cmdopt=type3 ERROR: usage: pytest [options] [file_or_dir] [file_or_dir] [...] - pytest: error: argument --cmdopt: invalid choice: 'type3' (choose from 'type1', 'type2') + pytest: error: argument --cmdopt: invalid choice: 'type3' (choose from type1, type2) + inifile: None + rootdir: /home/sweet/project If you need to provide more detailed error messages, you can use the @@ -232,7 +235,7 @@ directory with the above conftest.py: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 0 items @@ -296,7 +299,7 @@ and when running it will see a skipped "slow" test: $ pytest -rs # "-rs" means report details on the little 's' =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -312,7 +315,7 @@ Or run it including the ``slow`` marked test: $ pytest --runslow =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -350,7 +353,7 @@ Example: The ``__tracebackhide__`` setting influences ``pytest`` showing of tracebacks: the ``checkconfig`` function will not be shown -unless the ``--full-trace`` command line option is specified. +unless the :option:`--full-trace` command line option is specified. Let's run our little function: .. code-block:: pytest @@ -413,10 +416,10 @@ running from a test you can do this: if os.environ.get("PYTEST_VERSION") is not None: - # Things you want to to do if your code is called by pytest. + # Things you want to do if your code is called by pytest. ... else: - # Things you want to to do if your code is not called by pytest. + # Things you want to do if your code is not called by pytest. ... @@ -441,7 +444,7 @@ which will add the string to the test header accordingly: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y project deps: mylib-1.1 rootdir: /home/sweet/project collected 0 items @@ -460,7 +463,7 @@ display more information if applicable: def pytest_report_header(config): - if config.getoption("verbose") > 0: + if config.get_verbosity() > 0: return ["info1: did you know that ...", "did you?"] which will add info only when run with "--v": @@ -469,7 +472,7 @@ which will add info only when run with "--v": $ pytest -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache info1: did you know that ... did you? @@ -484,7 +487,7 @@ and nothing when run plainly: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 0 items @@ -523,7 +526,7 @@ Now we can profile which test functions execute the slowest: $ pytest --durations=3 =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 3 items @@ -550,12 +553,10 @@ an ``incremental`` marker which is to be used on classes: # content of conftest.py - from typing import Dict, Tuple - import pytest # store history of failures per test class name and per index in parametrize (if parametrize used) - _test_failed_incremental: Dict[str, Dict[Tuple[int, ...], str]] = {} + _test_failed_incremental: dict[str, dict[tuple[int, ...], str]] = {} def pytest_runtest_makereport(item, call): @@ -629,7 +630,7 @@ If we run this: $ pytest -rx =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 4 items @@ -645,33 +646,8 @@ If we run this: E assert 0 test_step.py:11: AssertionError - ================================ XFAILURES ================================= - ______________________ TestUserHandling.test_deletion ______________________ - - item = - - def pytest_runtest_setup(item): - if "incremental" in item.keywords: - # retrieve the class name of the test - cls_name = str(item.cls) - # check if a previous test has failed for this class - if cls_name in _test_failed_incremental: - # retrieve the index of the test (if parametrize is used in combination with incremental) - parametrize_index = ( - tuple(item.callspec.indices.values()) - if hasattr(item, "callspec") - else () - ) - # retrieve the name of the first test function to fail for this class name and index - test_name = _test_failed_incremental[cls_name].get(parametrize_index, None) - # if name found, test has failed for the combination of class name & test name - if test_name is not None: - > pytest.xfail(f"previous test failed ({test_name})") - E _pytest.outcomes.XFailed: previous test failed (test_modification) - - conftest.py:47: XFailed ========================= short test summary info ========================== - XFAIL test_step.py::TestUserHandling::test_deletion - reason: previous test failed (test_modification) + XFAIL test_step.py::TestUserHandling::test_deletion - previous test failed (test_modification) ================== 1 failed, 2 passed, 1 xfailed in 0.12s ================== We'll see that ``test_deletion`` was not executed because ``test_modification`` @@ -736,7 +712,7 @@ We can run this: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 7 items @@ -750,7 +726,7 @@ We can run this: file /home/sweet/project/b/test_error.py, line 1 def test_root(db): # no db here, will error out E fixture 'db' not found - > available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory + > available fixtures: cache, capfd, capfdbinary, caplog, capsys, capsysbinary, capteesys, doctest_namespace, monkeypatch, pytestconfig, record_property, record_testsuite_property, record_xml_attribute, recwarn, subtests, tmp_path, tmp_path_factory, tmpdir, tmpdir_factory > use 'pytest --fixtures [testpath]' for help on them. /home/sweet/project/b/test_error.py:1 @@ -761,6 +737,7 @@ We can run this: def test_a1(db): > assert 0, db # to show value + ^^^^^^^^^^^^ E AssertionError: E assert 0 @@ -771,6 +748,7 @@ We can run this: def test_a2(db): > assert 0, db # to show value + ^^^^^^^^^^^^ E AssertionError: E assert 0 @@ -795,7 +773,7 @@ The two test modules in the ``a`` directory see the same ``db`` fixture instance while the one test in the sister-directory ``b`` doesn't see it. We could of course also define a ``db`` fixture in that sister directory's ``conftest.py`` file. Note that each fixture is only instantiated if there is a test actually needing -it (unless you use "autouse" fixture which are always executed ahead of the first test +it (unless you use "autouse" fixtures which are always executed ahead of the first test executing). @@ -856,7 +834,7 @@ and run them: $ pytest test_module.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -903,11 +881,10 @@ here is a little example implemented via a local plugin: .. code-block:: python # content of conftest.py - from typing import Dict import pytest from pytest import StashKey, CollectReport - phase_report_key = StashKey[Dict[str, CollectReport]]() + phase_report_key = StashKey[dict[str, CollectReport]]() @pytest.hookimpl(wrapper=True, tryfirst=True) @@ -929,7 +906,9 @@ here is a little example implemented via a local plugin: # "function" scope report = request.node.stash[phase_report_key] if report["setup"].failed: - print("setting up a test failed or skipped", request.node.nodeid) + print("setting up a test failed", request.node.nodeid) + elif report["setup"].skipped: + print("setting up a test skipped", request.node.nodeid) elif ("call" not in report) or report["call"].failed: print("executing test failed or skipped", request.node.nodeid) @@ -965,11 +944,11 @@ and run it: $ pytest -s test_module.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 3 items - test_module.py Esetting up a test failed or skipped test_module.py::test_setup_fails + test_module.py Esetting up a test failed test_module.py::test_setup_fails Fexecuting test failed or skipped test_module.py::test_call_fails F @@ -1016,7 +995,7 @@ information. Sometimes a test session might get stuck and there might be no easy way to figure out -which test got stuck, for example if pytest was run in quiet mode (``-q``) or you don't have access to the console +which test got stuck, for example if pytest was run in quiet mode (:option:`-q`) or you don't have access to the console output. This is particularly a problem if the problem happens only sporadically, the famous "flaky" kind of tests. ``pytest`` sets the :envvar:`PYTEST_CURRENT_TEST` environment variable when running tests, which can be inspected diff --git a/doc/en/explanation/ci.rst b/doc/en/explanation/ci.rst index 45fe658d14f..1c03f840b43 100644 --- a/doc/en/explanation/ci.rst +++ b/doc/en/explanation/ci.rst @@ -8,7 +8,7 @@ Rationale The goal of testing in a CI pipeline is different from testing locally. Indeed, you can quickly edit some code and run your tests again on your computer, but -it is not possible with CI pipeline. They run on a separate server and are +it is not possible with CI pipelines. They run on a separate server and are triggered by specific actions. From that observation, pytest can detect when it is in a CI environment and @@ -17,11 +17,10 @@ adapt some of its behaviours. How CI is detected ------------------ -Pytest knows it is in a CI environment when either one of these environment variables are set, -regardless of their value: +Pytest knows it is in a CI environment when either one of these environment variables is set to a non-empty value: -* `CI`: used by many CI systems. -* `BUILD_NUMBER`: used by Jenkins. +* :envvar:`CI`: used by many CI systems. +* :envvar:`BUILD_NUMBER`: used by Jenkins. Effects on CI ------------- @@ -51,7 +50,7 @@ Running this locally, without any extra options, will output: $ pytest test_ci.py ... ========================= short test summary info ========================== - FAILED test_backends.py::test_db_initialized[d2] - Failed: deliberately f... + FAILED test_ci.py::test_db_initialized - Failed: deliberately f... *(Note the truncated text)* @@ -64,7 +63,7 @@ While running this on CI will output: $ pytest test_ci.py ... ========================= short test summary info ========================== - FAILED test_backends.py::test_db_initialized[d2] - Failed: deliberately failing + FAILED test_ci.py::test_db_initialized - Failed: deliberately failing for demo purpose, Lorem ipsum dolor sit amet, consectetur adipiscing elit. Cras facilisis, massa in suscipit dignissim, mauris lacus molestie nisi, quis varius metus nulla ut ipsum. diff --git a/doc/en/explanation/fixtures.rst b/doc/en/explanation/fixtures.rst index 0bb3bf49fb0..53d4796c825 100644 --- a/doc/en/explanation/fixtures.rst +++ b/doc/en/explanation/fixtures.rst @@ -75,7 +75,7 @@ style of setup/teardown functions: * fixture management scales from simple unit to complex functional testing, allowing to parametrize fixtures and tests according - to configuration and component options, or to re-use fixtures + to configuration and component options, or to reuse fixtures across function, class, module or whole test session scopes. * teardown logic can be easily, and safely managed, no matter how many fixtures diff --git a/doc/en/explanation/flaky.rst b/doc/en/explanation/flaky.rst index cb6c3983424..918d6f10b36 100644 --- a/doc/en/explanation/flaky.rst +++ b/doc/en/explanation/flaky.rst @@ -42,7 +42,7 @@ It is of course possible (and common) for tests and fixtures to spawn threads th * Make sure to eventually wait on any spawned threads -- for example at the end of a test, or during the teardown of a fixture. * Avoid using primitives provided by pytest (:func:`pytest.warns`, :func:`pytest.raises`, etc) from multiple threads, as they are not thread-safe. -If your test suite uses threads and your are seeing flaky test results, do not discount the possibility that the test is implicitly using global state in pytest itself. +If your test suite uses threads and you are seeing flaky test results, do not discount the possibility that the test is implicitly using global state in pytest itself. Related features ^^^^^^^^^^^^^^^^ @@ -117,8 +117,11 @@ This is a limited list, please submit an issue or pull request to expand it! * Gao, Zebao, Yalan Liang, Myra B. Cohen, Atif M. Memon, and Zhen Wang. "Making system user interactive tests repeatable: When and what should we control?." In *Software Engineering (ICSE), 2015 IEEE/ACM 37th IEEE International Conference on*, vol. 1, pp. 55-65. IEEE, 2015. `PDF `__ * Palomba, Fabio, and Andy Zaidman. "Does refactoring of test smells induce fixing flaky tests?." In *Software Maintenance and Evolution (ICSME), 2017 IEEE International Conference on*, pp. 1-12. IEEE, 2017. `PDF in Google Drive `__ -* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `__ -* Dutta, Saikat and Shi, August and Choudhary, Rutvik and Zhang, Zhekun and Jain, Aryaman and Misailovic, Sasa. "Detecting flaky tests in probabilistic and machine learning applications." In *Proceedings of the 29th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA)*, pp. 211-224. ACM, 2020. `PDF `__ +* Bell, Jonathan, Owolabi Legunsen, Michael Hilton, Lamyaa Eloussi, Tifany Yung, and Darko Marinov. "DeFlaker: Automatically detecting flaky tests." In *Proceedings of the 2018 International Conference on Software Engineering*. 2018. `PDF `__ +* Dutta, Saikat and Shi, August and Choudhary, Rutvik and Zhang, Zhekun and Jain, Aryaman and Misailovic, Sasa. "Detecting flaky tests in probabilistic and machine learning applications." In *Proceedings of the 29th ACM SIGSOFT International Symposium on Software Testing and Analysis (ISSTA)*, pp. 211-224. ACM, 2020. `PDF `__ +* Habchi, Sarra and Haben, Guillaume and Sohn, Jeongju and Franci, Adriano and Papadakis, Mike and Cordy, Maxime and Le Traon, Yves. "What Made This Test Flake? Pinpointing Classes Responsible for Test Flakiness." In Proceedings of the 38th IEEE International Conference on Software Maintenance and Evolution (ICSME), IEEE, 2022. `PDF `__ +* Lamprou, Sokrates. "Non-deterministic tests and where to find them: Empirically investigating the relationship between flaky tests and test smells by examining test order dependency." Bachelor thesis, Department of Computer and Information Science, Linköping University, 2022. LIU-IDA/LITH-EX-G–19/056–SE. `PDF `__ +* Leinen, Fabian and Elsner, Daniel and Pretschner, Alexander and Stahlbauer, Andreas and Sailer, Michael and Jürgens, Elmar. "Cost of Flaky Tests in Continuous Integration: An Industrial Case Study." Technical University of Munich and CQSE GmbH, Munich, Germany, 2023. `PDF `__ Resources ^^^^^^^^^ @@ -137,5 +140,12 @@ Resources * `Flaky Tests at Google and How We Mitigate Them `_ by John Micco, 2016 * `Where do Google's flaky tests come from? `_ by Jeff Listfield, 2017 +* Dropbox: + * `Athena: Our automated build health management system `_ by Utsav Shah, 2019 + * `How To Manage Flaky Tests in your CI Workflows `_ by Li Haoyi, 2025 + +* Uber: + * `Handling Flaky Unit Tests in Java `_ by Uber Engineering, 2021 + * `Flaky Tests Overhaul at Uber `_ by Uber Engineering, 2024 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist diff --git a/doc/en/explanation/goodpractices.rst b/doc/en/explanation/goodpractices.rst index 1390ba4e8fe..52474d148c6 100644 --- a/doc/en/explanation/goodpractices.rst +++ b/doc/en/explanation/goodpractices.rst @@ -94,14 +94,13 @@ This has the following benefits: For new projects, we recommend to use ``importlib`` :ref:`import mode ` (see which-import-mode_ for a detailed explanation). -To this end, add the following to your ``pyproject.toml``: +To this end, add the following to your configuration file: .. code-block:: toml - [tool.pytest.ini_options] - addopts = [ - "--import-mode=importlib", - ] + # content of pytest.toml + [pytest] + addopts = ["--import-mode=importlib"] .. _src-layout: @@ -126,22 +125,36 @@ which are better explained in this excellent `blog post`_ by Ionel Cristian Măr PYTHONPATH=src pytest or in a permanent manner by using the :confval:`pythonpath` configuration variable and adding the - following to your ``pyproject.toml``: + following to your configuration file: - .. code-block:: toml + .. tab:: toml + + .. code-block:: toml + + [pytest] + pythonpath = ["src"] + + .. tab:: ini + + .. code-block:: ini - [tool.pytest.ini_options] - pythonpath = "src" + [pytest] + pythonpath = src .. note:: - If you do not use an editable install and not use the ``src`` layout (``mypkg`` directly in the root + If you do not use an editable install and do not use the ``src`` layout (``mypkg`` directly in the root directory) you can rely on the fact that Python by default puts the current directory in ``sys.path`` to import your package and run ``python -m pytest`` to execute the tests against the local copy directly. See :ref:`pytest vs python -m pytest` for more information about the difference between calling ``pytest`` and ``python -m pytest``. +.. seealso:: + + :doc:`packaging:discussions/src-layout-vs-flat-layout` + The Python Packaging User Guide discusses the trade-offs between the ``src`` layout and ``flat`` layout. + Tests as part of application code ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -162,7 +175,7 @@ want to distribute them along with your application: test_view.py ... -In this scheme, it is easy to run your tests using the ``--pyargs`` option: +In this scheme, it is easy to run your tests using the :option:`--pyargs` option: .. code-block:: bash @@ -209,9 +222,9 @@ Note that this layout also works in conjunction with the ``src`` layout mentione from each other and thus deriving a canonical import name helps to avoid surprises such as a test module getting imported twice. - With ``--import-mode=importlib`` things are less convoluted because - pytest doesn't need to change ``sys.path`` or ``sys.modules``, making things - much less surprising. + With :option:`--import-mode=importlib` things are less convoluted because + pytest doesn't need to change ``sys.path``, making things much less + surprising. .. _which-import-mode: @@ -313,3 +326,75 @@ A list of the lints detected by flake8-pytest-style can be found on its `PyPI pa .. note:: flake8-pytest-style is not an official pytest project. Some of the rules enforce certain style choices, such as using `@pytest.fixture()` over `@pytest.fixture`, but you can configure the plugin to fit your preferred style. + +.. _`strict mode`: + +Using pytest's strict mode +-------------------------- + +.. versionadded:: 9.0 + +Pytest contains a set of configuration options that make it more strict. +The options are off by default for compatibility or other reasons, +but you should enable them if you can. + +You can enable all of the strictness options at once by setting the :confval:`strict` configuration option: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + strict = true + +.. tab:: ini + + .. code-block:: ini + + [pytest] + strict = true + +See the :confval:`strict` documentation for the options it enables and their effect. + +If pytest adds new strictness options in the future, they will also be enabled in strict mode. +Therefore, you should only enable strict mode if you use a pinned/locked version of pytest, +or if you want to proactively adopt new strictness options as they are added. +If you don't want to automatically pick up new options, you can enable options individually: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + strict_config = true + strict_markers = true + strict_parametrization_ids = true + strict_xfail = true + +.. tab:: ini + + .. code-block:: ini + + [pytest] + strict_config = true + strict_markers = true + strict_parametrization_ids = true + strict_xfail = true + +If you want to use strict mode but are having trouble with a specific option, you can turn it off individually: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + strict = true + strict_parametrization_ids = false + +.. tab:: ini + + .. code-block:: ini + + [pytest] + strict = true + strict_parametrization_ids = false diff --git a/doc/en/explanation/index.rst b/doc/en/explanation/index.rst index 2edf60a5d8b..2606d7d4b34 100644 --- a/doc/en/explanation/index.rst +++ b/doc/en/explanation/index.rst @@ -12,5 +12,6 @@ Explanation fixtures goodpractices pythonpath + types ci flaky diff --git a/doc/en/explanation/pythonpath.rst b/doc/en/explanation/pythonpath.rst index d0314a6dbcd..cb3ae67216a 100644 --- a/doc/en/explanation/pythonpath.rst +++ b/doc/en/explanation/pythonpath.rst @@ -8,10 +8,10 @@ pytest import mechanisms and ``sys.path``/``PYTHONPATH`` Import modes ------------ -pytest as a testing framework that needs to import test modules and ``conftest.py`` files for execution. +pytest as a testing framework needs to import test modules and ``conftest.py`` files for execution. Importing files in Python is a non-trivial process, so aspects of the -import process can be controlled through the ``--import-mode`` command-line flag, which can assume +import process can be controlled through the :option:`--import-mode` command-line flag, which can assume these values: .. _`import-mode-prepend`: @@ -44,12 +44,12 @@ these values: pkg_under_test/ the tests will run against the installed version - of ``pkg_under_test`` when ``--import-mode=append`` is used whereas + of ``pkg_under_test`` when :option:`--import-mode=append` is used whereas with ``prepend``, they would pick up the local version. This kind of confusion is why we advocate for using :ref:`src-layouts `. Same as ``prepend``, requires test module names to be unique when the test directory tree is - not arranged in packages, because the modules will put in :py:data:`sys.modules` after importing. + not arranged in packages, because the modules will be put in :py:data:`sys.modules` after importing. .. _`import-mode-importlib`: @@ -64,7 +64,7 @@ these values: * Test modules can't import each other. * Testing utility modules in the tests directories (for example a ``tests.helpers`` module containing test-related functions/classes) - are not importable. The recommendation in this case it to place testing utility modules together with the application/library + are not importable. The recommendation in this case is to place testing utility modules together with the application/library code, for example ``app.testing.helpers``. Important: by "test utility modules", we mean functions/classes which are imported by @@ -78,7 +78,7 @@ these values: For non-test modules, this will work if they are accessible via :py:data:`sys.path`. So for example, ``.env/lib/site-packages/app/core.py`` will be importable as ``app.core``. - This is happens when plugins import non-test modules (for example doctesting). + This happens when plugins import non-test modules (for example doctesting). If this step succeeds, the module is returned. @@ -152,7 +152,7 @@ this case ``foo/``). To load the module, it will insert ``root/`` to the front The same logic applies to the ``conftest.py`` file: it will be imported as ``foo.conftest`` module. Preserving the full package name is important when tests live in a package to avoid problems -and allow test modules to have duplicated names. This is also discussed in details in +and allow test modules to have duplicated names. This is also discussed in detail in :ref:`test discovery`. Standalone test modules / ``conftest.py`` files @@ -182,7 +182,7 @@ with the ``conftest.py`` file by adding ``root/foo`` to :py:data:`sys.path` to i For this reason this layout cannot have test modules with the same name, as they all will be imported in the global import namespace. -This is also discussed in details in :ref:`test discovery`. +This is also discussed in detail in :ref:`test discovery`. .. _`pytest vs python -m pytest`: diff --git a/doc/en/explanation/types.rst b/doc/en/explanation/types.rst new file mode 100644 index 00000000000..827a2bf02b6 --- /dev/null +++ b/doc/en/explanation/types.rst @@ -0,0 +1,89 @@ +.. _types: + +Typing in pytest +================ + +.. note:: + This page assumes the reader is familiar with Python's typing system and its advantages. + + For more information, refer to `Python's Typing Documentation `_. + +Why type tests? +--------------- + +Typing tests provides significant advantages: + +- **Readability:** Clearly defines expected inputs and outputs, improving readability, especially in complex or parameterized tests. + +- **Refactoring:** This is the main benefit in typing tests, as it will greatly help with refactoring, letting the type checker point out the necessary changes in both production and tests, without needing to run the full test suite. + +For production code, typing also helps catching some bugs that might not be caught by tests at all (regardless of coverage), for example: + +.. code-block:: python + + def get_caption(target: int, items: list[tuple[int, str]]) -> str: + for value, caption in items: + if value == target: + return caption + + +The type checker will correctly error out that the function might return `None`, however even a full coverage test suite might miss that case: + +.. code-block:: python + + def test_get_caption() -> None: + assert get_caption(10, [(1, "foo"), (10, "bar")]) == "bar" + + +Note the code above has 100% coverage, but the bug is not caught (of course the example is "obvious", but serves to illustrate the point). + + + +Using typing in test suites +--------------------------- + +To type fixtures in pytest, just add normal types to the fixture functions -- there is nothing special that needs to be done just because of the `fixture` decorator. + +.. code-block:: python + + import pytest + + + @pytest.fixture + def sample_fixture() -> int: + return 38 + +In the same manner, the fixtures passed to test functions need be annotated with the fixture's return type: + +.. code-block:: python + + def test_sample_fixture(sample_fixture: int) -> None: + assert sample_fixture == 38 + +From the POV of the type checker, it does not matter that `sample_fixture` is actually a fixture managed by pytest, all it matters to it is that `sample_fixture` is a parameter of type `int`. + + +The same logic applies to :ref:`@pytest.mark.parametrize <@pytest.mark.parametrize>`: + +.. code-block:: python + + + @pytest.mark.parametrize("input_value, expected_output", [(1, 2), (5, 6), (10, 11)]) + def test_increment(input_value: int, expected_output: int) -> None: + assert input_value + 1 == expected_output + + +The same logic applies when typing fixture functions which receive other fixtures: + +.. code-block:: python + + @pytest.fixture + def mock_env_user(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("USER", "TestingUser") + + +Conclusion +---------- + +Incorporating typing into pytest tests enhances **clarity**, improves **debugging** and **maintenance**, and ensures **type safety**. +These practices lead to a **robust**, **readable**, and **easily maintainable** test suite that is better equipped to handle future changes with minimal risk of errors. diff --git a/doc/en/funcarg_compare.rst b/doc/en/funcarg_compare.rst index 8b900d30f20..7cd4c0f1676 100644 --- a/doc/en/funcarg_compare.rst +++ b/doc/en/funcarg_compare.rst @@ -16,9 +16,9 @@ Shortcomings of the previous ``pytest_funcarg__`` mechanism The pre pytest-2.3 funcarg mechanism calls a factory each time a funcarg for a test function is required. If a factory wants to -re-use a resource across different scopes, it often used +reuse a resource across different scopes, it often used the ``request.cached_setup()`` helper to manage caching of -resources. Here is a basic example how we could implement +resources. Here is a basic example of how we could implement a per-session Database object: .. code-block:: python @@ -39,10 +39,10 @@ a per-session Database object: There are several limitations and difficulties with this approach: -1. Scoping funcarg resource creation is not straight forward, instead one must +1. Scoping funcarg resource creation is not straightforward, instead one must understand the intricate cached_setup() method mechanics. -2. parametrizing the "db" resource is not straight forward: +2. parametrizing the "db" resource is not straightforward: you need to apply a "parametrize" decorator or implement a :hook:`pytest_generate_tests` hook calling :py:func:`~pytest.Metafunc.parametrize` which @@ -55,7 +55,7 @@ There are several limitations and difficulties with this approach: at the same time, making it hard for them to affect global state of the application under test. -4. there is no way how you can make use of funcarg factories +4. there is no way you can make use of funcarg factories in xUnit setup methods. 5. A non-parametrized fixture function cannot use a parametrized @@ -107,7 +107,7 @@ the tests requiring "db" will run twice as well. The "mysql" and "pg" values will also be used for reporting the test-invocation variants. This new way of parametrizing funcarg factories should in many cases -allow to re-use already written factories because effectively +allow to reuse already written factories because effectively ``request.param`` was already used when test functions/classes were parametrized via :py:func:`metafunc.parametrize(indirect=True) ` calls. @@ -164,7 +164,7 @@ hook which are often used to setup global resources. This suffers from several problems: 1. in distributed testing the managing process would setup test resources - that are never needed because it only co-ordinates the test run + that are never needed because it only coordinates the test run activities of the worker processes. 2. if you only perform a collection (with "--collect-only") diff --git a/doc/en/getting-started.rst b/doc/en/getting-started.rst index 85bee729ba1..76a4428c163 100644 --- a/doc/en/getting-started.rst +++ b/doc/en/getting-started.rst @@ -9,8 +9,6 @@ Get Started Install ``pytest`` ---------------------------------------- -``pytest`` requires: Python 3.8+ or PyPy3. - 1. Run the following command in your command line: .. code-block:: bash @@ -22,7 +20,7 @@ Install ``pytest`` .. code-block:: bash $ pytest --version - pytest 8.2.2 + pytest 9.0.3 .. _`simpletest`: @@ -47,7 +45,7 @@ The test $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -75,7 +73,7 @@ The ``[100%]`` refers to the overall progress of running all test cases. After i Run multiple tests ---------------------------------------------------------- -``pytest`` will run all files of the form test_*.py or \*_test.py in the current directory and its subdirectories. More generally, it follows :ref:`standard test discovery rules `. +``pytest`` will run all files of the form ``test_*.py`` or ``*_test.py`` in the current directory and its subdirectories. More generally, it follows :ref:`standard test discovery rules `. Assert that a certain exception is raised @@ -97,30 +95,6 @@ Use the :ref:`raises ` helper to assert that some code raises an e with pytest.raises(SystemExit): f() -You can also use the context provided by :ref:`raises ` to -assert that an expected exception is part of a raised :class:`ExceptionGroup`: - -.. code-block:: python - - # content of test_exceptiongroup.py - import pytest - - - def f(): - raise ExceptionGroup( - "Group message", - [ - RuntimeError(), - ], - ) - - - def test_exception_in_group(): - with pytest.raises(ExceptionGroup) as excinfo: - f() - assert excinfo.group_contains(RuntimeError) - assert not excinfo.group_contains(TypeError) - Execute the test function with “quiet” reporting mode: .. code-block:: pytest @@ -133,6 +107,8 @@ Execute the test function with “quiet” reporting mode: The ``-q/--quiet`` flag keeps the output brief in this and following examples. +See :ref:`assertraises` for specifying more details about the expected exception. + Group multiple tests in a class -------------------------------------------------------------- @@ -223,6 +199,26 @@ This is outlined below: Note that attributes added at class level are *class attributes*, so they will be shared between tests. +Compare floating-point values with pytest.approx +-------------------------------------------------------------- + +``pytest`` also provides a number of utilities to make writing tests easier. +For example, you can use :func:`pytest.approx` to compare floating-point +values that may have small rounding errors: + +.. code-block:: python + + # content of test_approx.py + import pytest + + + def test_sum(): + assert (0.1 + 0.2) == pytest.approx(0.3) + +This avoids the need for manual tolerance checks or using +``math.isclose`` and works with scalars, lists, and NumPy arrays. + + Request a unique temporary directory for functional tests -------------------------------------------------------------- @@ -266,7 +262,7 @@ Find out what kind of builtin :ref:`pytest fixtures ` exist with the c pytest --fixtures # shows builtin and custom fixtures -Note that this command omits fixtures with leading ``_`` unless the ``-v`` option is added. +Note that this command omits fixtures with leading ``_`` unless the :option:`-v` option is added. Continue reading ------------------------------------- diff --git a/doc/en/historical-notes.rst b/doc/en/historical-notes.rst index be67036d6ca..d93c7b94793 100644 --- a/doc/en/historical-notes.rst +++ b/doc/en/historical-notes.rst @@ -263,20 +263,24 @@ configuration value which you might have added: @pytest.mark.skipif("not config.getvalue('db')") def test_function(): ... -The equivalent with "boolean conditions" is: +The equivalent with "boolean conditions" using ``request.config`` is: .. code-block:: python - @pytest.mark.skipif(not pytest.config.getvalue("db"), reason="--db was not specified") + @pytest.fixture(autouse=True) + def skip_if_no_db(request): + if not request.config.getoption("--db", default=False): + pytest.skip("--db was not specified") + + def test_function(): pass .. note:: - You cannot use ``pytest.config.getvalue()`` in code - imported before pytest's argument parsing takes place. For example, - ``conftest.py`` files are imported before command line parsing and thus - ``config.getvalue()`` will not execute correctly. + ``pytest.config`` was removed in pytest 5.0. Use ``request.config`` + (via the ``request`` fixture) or the ``pytestconfig`` fixture instead. + See :ref:`pytest.config global deprecated` for details. ``pytest.set_trace()`` ---------------------- @@ -304,7 +308,7 @@ For more details see :ref:`breakpoints`. -Access of ``Module``, ``Function``, ``Class``, ``Instance``, ``File`` and ``Item`` through ``Node`` instances have long +Access of ``Module``, ``Function``, ``Class``, ``Instance``, ``File`` and ``Item`` through ``Node`` instances has long been documented as deprecated, but started to emit warnings from pytest ``3.9`` and onward. Users should just ``import pytest`` and access those objects using the ``pytest`` module. diff --git a/doc/en/how-to/assert.rst b/doc/en/how-to/assert.rst index 7b027744695..006cf475b02 100644 --- a/doc/en/how-to/assert.rst +++ b/doc/en/how-to/assert.rst @@ -29,7 +29,7 @@ you will see the return value of the function call: $ pytest test_assert1.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -66,6 +66,33 @@ See :ref:`assert-details` for more information on assertion introspection. .. _`assertraises`: +Assertions about approximate equality +------------------------------------- + +When comparing floating point values (or arrays of floats), small rounding +errors are common. Instead of using ``assert abs(a - b) < tol`` or +``numpy.isclose``, you can use :func:`pytest.approx`: + +.. code-block:: python + + import pytest + import numpy as np + + + def test_floats(): + assert (0.1 + 0.2) == pytest.approx(0.3) + + + def test_arrays(): + a = np.array([1.0, 2.0, 3.0]) + b = np.array([0.9999, 2.0001, 3.0]) + assert a == pytest.approx(b) + +``pytest.approx`` works with scalars, lists, dictionaries, and NumPy arrays. +It also supports comparisons involving NaNs. + +See :func:`pytest.approx` for details. + Assertions about expected exceptions ------------------------------------------ @@ -145,8 +172,93 @@ Notes: .. _`assert-matching-exception-groups`: -Matching exception groups -~~~~~~~~~~~~~~~~~~~~~~~~~ +Assertions about expected exception groups +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +When expecting a :exc:`BaseExceptionGroup` or :exc:`ExceptionGroup` you can use :class:`pytest.RaisesGroup`: + +.. code-block:: python + + def test_exception_in_group(): + with pytest.RaisesGroup(ValueError): + raise ExceptionGroup("group msg", [ValueError("value msg")]) + with pytest.RaisesGroup(ValueError, TypeError): + raise ExceptionGroup("msg", [ValueError("foo"), TypeError("bar")]) + + +It accepts a ``match`` parameter, that checks against the group message, and a ``check`` parameter that takes an arbitrary callable which it passes the group to, and only succeeds if the callable returns ``True``. + +.. code-block:: python + + def test_raisesgroup_match_and_check(): + with pytest.RaisesGroup(BaseException, match="my group msg"): + raise BaseExceptionGroup("my group msg", [KeyboardInterrupt()]) + with pytest.RaisesGroup( + Exception, check=lambda eg: isinstance(eg.__cause__, ValueError) + ): + raise ExceptionGroup("", [TypeError()]) from ValueError() + +It is strict about structure and unwrapped exceptions, unlike :ref:`except* `, so you might want to set the ``flatten_subgroups`` and/or ``allow_unwrapped`` parameters. + +.. code-block:: python + + def test_structure(): + with pytest.RaisesGroup(pytest.RaisesGroup(ValueError)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + with pytest.RaisesGroup(ValueError, flatten_subgroups=True): + raise ExceptionGroup("1st group", [ExceptionGroup("2nd group", [ValueError()])]) + with pytest.RaisesGroup(ValueError, allow_unwrapped=True): + raise ValueError + +To specify more details about the contained exception you can use :class:`pytest.RaisesExc` + +.. code-block:: python + + def test_raises_exc(): + with pytest.RaisesGroup(pytest.RaisesExc(ValueError, match="foo")): + raise ExceptionGroup("", (ValueError("foo"))) + +They both supply a method :meth:`pytest.RaisesGroup.matches` :meth:`pytest.RaisesExc.matches` if you want to do matching outside of using it as a :external+python:std:ref:`context manager `. This can be helpful when checking ``.__context__`` or ``.__cause__``. + +.. code-block:: python + + def test_matches(): + exc = ValueError() + exc_group = ExceptionGroup("", [exc]) + if RaisesGroup(ValueError).matches(exc_group): + ... + # helpful error is available in `.fail_reason` if it fails to match + r = RaisesExc(ValueError) + assert r.matches(e), r.fail_reason + +Check the documentation on :class:`pytest.RaisesGroup` and :class:`pytest.RaisesExc` for more details and examples. + +``ExceptionInfo.group_contains()`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. warning:: + + This helper makes it easy to check for the presence of specific exceptions, but it is very bad for checking that the group does *not* contain *any other exceptions*. So this will pass: + + .. code-block:: python + + class EXTREMELYBADERROR(BaseException): + """This is a very bad error to miss""" + + + def test_for_value_error(): + with pytest.raises(ExceptionGroup) as excinfo: + excs = [ValueError()] + if very_unlucky(): + excs.append(EXTREMELYBADERROR()) + raise ExceptionGroup("", excs) + # This passes regardless of if there's other exceptions. + assert excinfo.group_contains(ValueError) + # You can't simply list all exceptions you *don't* want to get here. + + + There is no good way of using :func:`excinfo.group_contains() ` to ensure you're not getting *any* other exceptions than the one you expected. + You should instead use :class:`pytest.RaisesGroup`, see :ref:`assert-matching-exception-groups`. You can also use the :func:`excinfo.group_contains() ` method to test for exceptions returned as part of an :class:`ExceptionGroup`: @@ -194,12 +306,12 @@ exception at a specific level; exceptions contained directly in the top assert not excinfo.group_contains(RuntimeError, depth=2) assert not excinfo.group_contains(TypeError, depth=1) -Alternate form (legacy) -~~~~~~~~~~~~~~~~~~~~~~~ +Alternate `pytest.raises` form (legacy) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -There is an alternate form where you pass -a function that will be executed, along ``*args`` and ``**kwargs``, and :func:`pytest.raises` -will execute the function with the arguments and assert that the given exception is raised: +There is an alternate form of :func:`pytest.raises` where you pass +a function that will be executed, along with ``*args`` and ``**kwargs``. :func:`pytest.raises` +will then execute the function with those arguments and assert that the given exception is raised: .. code-block:: python @@ -244,6 +356,18 @@ This will only "xfail" if the test fails by raising ``IndexError`` or subclasses * Using :func:`pytest.raises` is likely to be better for cases where you are testing exceptions your own code is deliberately raising, which is the majority of cases. +You can also use :class:`pytest.RaisesGroup`: + +.. code-block:: python + + def f(): + raise ExceptionGroup("", [IndexError()]) + + + @pytest.mark.xfail(raises=RaisesGroup(IndexError)) + def test_f(): + f() + .. _`assertwarns`: @@ -280,7 +404,7 @@ if you run this module: $ pytest test_assert2.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -312,6 +436,10 @@ Special comparisons are done for a number of cases: * comparing long sequences: first failing indices * comparing dicts: different entries +In string context diffs, lines prefixed with ``-`` come from the left-hand side +of ``assert left == right``, while lines prefixed with ``+`` come from the +right-hand side. + See the :ref:`reporting demo ` for many more examples. Defining your own explanation for failed assertions @@ -379,6 +507,50 @@ the conftest file: FAILED test_foocompare.py::test_compare - assert Comparing Foo instances: 1 failed in 0.12s +.. _`return-not-none`: + +Returning non-None value in test functions +------------------------------------------ + +A :class:`pytest.PytestReturnNotNoneWarning` is emitted when a test function returns a value other than ``None``. + +This helps prevent a common mistake made by beginners who assume that returning a ``bool`` (e.g., ``True`` or ``False``) will determine whether a test passes or fails. + +Example: + +.. code-block:: python + + @pytest.mark.parametrize( + ["a", "b", "result"], + [ + [1, 2, 5], + [2, 3, 8], + [5, 3, 18], + ], + ) + def test_foo(a, b, result): + return foo(a, b) == result # Incorrect usage, do not do this. + +Since pytest ignores return values, it might be surprising that the test will never fail based on the returned value. + +The correct fix is to replace the ``return`` statement with an ``assert``: + +.. code-block:: python + + @pytest.mark.parametrize( + ["a", "b", "result"], + [ + [1, 2, 5], + [2, 3, 8], + [5, 3, 18], + ], + ) + def test_foo(a, b, result): + assert foo(a, b) == result + + + + .. _assert-details: .. _`assert introspection`: @@ -415,7 +587,7 @@ Note that you still get the benefits of assertion introspection, the only change the ``.pyc`` files won't be cached on disk. Additionally, rewriting will silently skip caching if it cannot write new ``.pyc`` files, -i.e. in a read-only filesystem or a zipfile. +e.g. in a read-only filesystem or a zipfile. Disabling assert rewriting @@ -431,4 +603,4 @@ If this is the case you have two options: * Disable rewriting for a specific module by adding the string ``PYTEST_DONT_REWRITE`` to its docstring. -* Disable rewriting for all modules by using ``--assert=plain``. +* Disable rewriting for all modules by using :option:`--assert=plain`. diff --git a/doc/en/how-to/cache.rst b/doc/en/how-to/cache.rst index 40cd3f00dd6..ca345916fc5 100644 --- a/doc/en/how-to/cache.rst +++ b/doc/en/how-to/cache.rst @@ -13,11 +13,11 @@ Usage The plugin provides two command line options to rerun failures from the last ``pytest`` invocation: -* ``--lf``, ``--last-failed`` - to only re-run the failures. -* ``--ff``, ``--failed-first`` - to run the failures first and then the rest of +* :option:`--lf, --last-failed <--lf>` - to only re-run the failures. +* :option:`--ff, --failed-first <--ff>` - to run the failures first and then the rest of the tests. -For cleanup (usually not needed), a ``--cache-clear`` option allows to remove +For cleanup (usually not needed), a :option:`--cache-clear` option allows to remove all cross-session cache contents ahead of a test run. Other plugins may access the `config.cache`_ object to set/get @@ -33,7 +33,7 @@ Other plugins may access the `config.cache`_ object to set/get Rerunning only failures or failures first ----------------------------------------------- -First, let's create 50 test invocation of which only 2 fail: +First, let's create 50 test invocations of which only 2 fail: .. code-block:: python @@ -80,13 +80,13 @@ If you run this for the first time you will see two failures: FAILED test_50.py::test_num[25] - Failed: bad luck 2 failed, 48 passed in 0.12s -If you then run it with ``--lf``: +If you then run it with :option:`--lf`: .. code-block:: pytest $ pytest --lf =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items run-last-failure: rerun previous 2 failures @@ -124,7 +124,7 @@ If you then run it with ``--lf``: You have run only the two failing tests from the last run, while the 48 passing tests have not been run ("deselected"). -Now, if you run with the ``--ff`` option, all tests will be run but the first +Now, if you run with the :option:`--ff` option, all tests will be run but the first previous failures will be executed first (as can be seen from the series of ``FF`` and dots): @@ -132,7 +132,7 @@ of ``FF`` and dots): $ pytest --ff =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 50 items run-last-failure: rerun previous 2 failures first @@ -169,14 +169,14 @@ of ``FF`` and dots): .. _`config.cache`: -New ``--nf``, ``--new-first`` options: run new tests first followed by the rest +New :option:`--nf, --new-first <--nf>` option: run new tests first followed by the rest of the tests, in both cases tests are also sorted by the file modified time, with more recent files coming first. Behavior when no tests failed in the last run --------------------------------------------- -The ``--lfnf/--last-failed-no-failures`` option governs the behavior of ``--last-failed``. +The :option:`--lfnf, --last-failed-no-failures <--lfnf>` option governs the behavior of :option:`--last-failed`. Determines whether to execute tests when there are no previously (known) failures or when no cached ``lastfailed`` data was found. @@ -199,7 +199,7 @@ The new config.cache object Plugins or conftest.py support code can get a cached value using the pytest ``config`` object. Here is a basic example plugin which -implements a :ref:`fixture ` which re-uses previously created state +implements a :ref:`fixture ` which reuses previously created state across pytest invocations: .. code-block:: python @@ -275,13 +275,13 @@ Inspecting Cache content ------------------------ You can always peek at the content of the cache using the -``--cache-show`` command line option: +:option:`--cache-show` command line option: .. code-block:: pytest $ pytest --cache-show =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project cachedir: /home/sweet/project/.pytest_cache --------------------------- cache values for '*' --------------------------- @@ -289,21 +289,19 @@ You can always peek at the content of the cache using the {'test_caching.py::test_function': True} cache/nodeids contains: ['test_caching.py::test_function'] - cache/stepwise contains: - [] example/value contains: 42 ========================== no tests ran in 0.12s =========================== -``--cache-show`` takes an optional argument to specify a glob pattern for +:option:`--cache-show` takes an optional argument to specify a glob pattern for filtering: .. code-block:: pytest $ pytest --cache-show example/* =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project cachedir: /home/sweet/project/.pytest_cache ----------------------- cache values for 'example/*' ----------------------- @@ -316,7 +314,7 @@ Clearing Cache content ---------------------- You can instruct pytest to clear all cache files and values -by adding the ``--cache-clear`` option like this: +by adding the :option:`--cache-clear` option like this: .. code-block:: bash @@ -332,4 +330,4 @@ than speed. Stepwise -------- -As an alternative to ``--lf -x``, especially for cases where you expect a large part of the test suite will fail, ``--sw``, ``--stepwise`` allows you to fix them one at a time. The test suite will run until the first failure and then stop. At the next invocation, tests will continue from the last failing test and then run until the next failing test. You may use the ``--stepwise-skip`` option to ignore one failing test and stop the test execution on the second failing test instead. This is useful if you get stuck on a failing test and just want to ignore it until later. Providing ``--stepwise-skip`` will also enable ``--stepwise`` implicitly. +As an alternative to :option:`--lf` :option:`-x`, especially for cases where you expect a large part of the test suite will fail, :option:`--sw, --stepwise <--sw>` allows you to fix them one at a time. The test suite will run until the first failure and then stop. At the next invocation, tests will continue from the last failing test and then run until the next failing test. You may use the :option:`--stepwise-skip` option to ignore one failing test and stop the test execution on the second failing test instead. This is useful if you get stuck on a failing test and just want to ignore it until later. Providing ``--stepwise-skip`` will also enable ``--stepwise`` implicitly. diff --git a/doc/en/how-to/capture-stdout-stderr.rst b/doc/en/how-to/capture-stdout-stderr.rst index 5e23f0c024e..5de89bc0e3f 100644 --- a/doc/en/how-to/capture-stdout-stderr.rst +++ b/doc/en/how-to/capture-stdout-stderr.rst @@ -4,20 +4,26 @@ How to capture stdout/stderr output ========================================================= +Pytest intercepts stdout and stderr as configured by the :option:`--capture=` +command-line argument or by using fixtures. The ``--capture=`` flag configures +reporting, whereas the fixtures offer more granular control and allow +inspection of output during testing. The reports can be customized with the +:option:`-r` flag. + Default stdout/stderr/stdin capturing behaviour --------------------------------------------------------- During test execution any output sent to ``stdout`` and ``stderr`` is captured. If a test or a setup method fails its according captured -output will usually be shown along with the failure traceback. (this -behavior can be configured by the ``--show-capture`` command-line option). +output will usually be shown along with the failure traceback. (This +behavior can be configured by the :option:`--show-capture` command-line option). In addition, ``stdin`` is set to a "null" object which will fail on attempts to read from it because it is rarely desired to wait for interactive input when running automated tests. By default capturing is done by intercepting writes to low level -file descriptors. This allows to capture output from simple +file descriptors. This allows capturing output from simple print statements as well as output from a subprocess started by a test. @@ -83,7 +89,7 @@ of the failing function and hide the other one: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -103,12 +109,15 @@ of the failing function and hide the other one: FAILED test_module.py::test_func2 - assert False ======================= 1 failed, 1 passed in 0.12s ======================== +.. _accessing-captured-output: + Accessing captured output from a test function --------------------------------------------------- -The ``capsys``, ``capsysbinary``, ``capfd``, and ``capfdbinary`` fixtures -allow access to stdout/stderr output created during test execution. Here is -an example test function that performs some output related checks: +The :fixture:`capsys`, :fixture:`capteesys`, :fixture:`capsysbinary`, :fixture:`capfd`, and :fixture:`capfdbinary` +fixtures allow access to ``stdout``/``stderr`` output created during test execution. + +Here is an example test function that performs some output related checks: .. code-block:: python @@ -125,40 +134,27 @@ an example test function that performs some output related checks: The ``readouterr()`` call snapshots the output so far - and capturing will be continued. After the test function finishes the original streams will -be restored. Using ``capsys`` this way frees your +be restored. Using :fixture:`capsys` this way frees your test from having to care about setting/resetting output streams and also interacts well with pytest's own per-test capturing. -If you want to capture on filedescriptor level you can use -the ``capfd`` fixture which offers the exact -same interface but allows to also capture output from -libraries or subprocesses that directly write to operating -system level output streams (FD1 and FD2). - - - -The return value from ``readouterr`` changed to a ``namedtuple`` with two attributes, ``out`` and ``err``. - - +The return value of ``readouterr()`` is a ``namedtuple`` with two attributes, ``out`` and ``err``. -If the code under test writes non-textual data, you can capture this using -the ``capsysbinary`` fixture which instead returns ``bytes`` from +If the code under test writes non-textual data (``bytes``), you can capture this using +the :fixture:`capsysbinary` fixture which instead returns ``bytes`` from the ``readouterr`` method. +If you want to capture at the file descriptor level you can use +the :fixture:`capfd` fixture which offers the exact +same interface but allows to also capture output from +libraries or subprocesses that directly write to operating +system level output streams (FD1 and FD2). Similarly to :fixture:`capsysbinary`, :fixture:`capfdbinary` can be +used to capture ``bytes`` at the file descriptor level. - -If the code under test writes non-textual data, you can capture this using -the ``capfdbinary`` fixture which instead returns ``bytes`` from -the ``readouterr`` method. The ``capfdbinary`` fixture operates on the -filedescriptor level. - - - - -To temporarily disable capture within a test, both ``capsys`` -and ``capfd`` have a ``disabled()`` method that can be used +To temporarily disable capture within a test, the capture fixtures +have a ``disabled()`` method that can be used as a context manager, disabling capture inside the ``with`` block: .. code-block:: python @@ -168,3 +164,13 @@ as a context manager, disabling capture inside the ``with`` block: with capsys.disabled(): print("output not captured, going directly to sys.stdout") print("this output is also captured") + +.. note:: + + When a capture fixture such as :fixture:`capsys` or :fixture:`capfd` is used, + it takes precedence over the global capturing configuration set via + command-line options such as ``-s`` or ``--capture=no``. + + This means that output produced within a test using a capture fixture will + still be captured and available via ``readouterr()``, even if global capturing + is disabled. diff --git a/doc/en/how-to/capture-warnings.rst b/doc/en/how-to/capture-warnings.rst index afabad5da14..b0ff6a74892 100644 --- a/doc/en/how-to/capture-warnings.rst +++ b/doc/en/how-to/capture-warnings.rst @@ -28,7 +28,7 @@ Running pytest now produces this output: $ pytest test_show_warnings.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -66,6 +66,7 @@ as an error: def test_one(): > assert api_v1() == 1 + ^^^^^^^^ test_show_warnings.py:10: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ @@ -79,30 +80,32 @@ as an error: FAILED test_show_warnings.py::test_one - UserWarning: api v1, should use ... 1 failed in 0.12s -The same option can be set in the ``pytest.ini`` or ``pyproject.toml`` file using the -``filterwarnings`` ini option. For example, the configuration below will ignore all +The same option can be set in the configuration file using the +:confval:`filterwarnings` configuration option. For example, the configuration below will ignore all user warnings and specific deprecation warnings matching a regex, but will transform all other warnings into errors. -.. code-block:: ini +.. tab:: toml - # pytest.ini - [pytest] - filterwarnings = - error - ignore::UserWarning - ignore:function ham\(\) is deprecated:DeprecationWarning + .. code-block:: toml -.. code-block:: toml + [pytest] + filterwarnings = [ + "error", + "ignore::UserWarning", + # note the use of single quote below to denote "raw" strings in TOML + 'ignore:function ham\(\) is deprecated:DeprecationWarning', + ] + +.. tab:: ini - # pyproject.toml - [tool.pytest.ini_options] - filterwarnings = [ - "error", - "ignore::UserWarning", - # note the use of single quote below to denote "raw" strings in TOML - 'ignore:function ham\(\) is deprecated:DeprecationWarning', - ] + .. code-block:: ini + + [pytest] + filterwarnings = + error + ignore::UserWarning + ignore:function ham\(\) is deprecated:DeprecationWarning When a warning matches more than one option in the list, the action for the last matching option @@ -111,7 +114,7 @@ is performed. .. note:: - The ``-W`` flag and the ``filterwarnings`` ini option use warning filters that are + The ``-W`` flag and the :confval:`filterwarnings` configuration option use warning filters that are similar in structure, but each configuration option interprets its filter differently. For example, *message* in ``filterwarnings`` is a string containing a regular expression that the start of the warning message must match, @@ -128,7 +131,7 @@ is performed. -You can use the ``@pytest.mark.filterwarnings`` to add warning filters to specific test items, +You can use the :ref:`@pytest.mark.filterwarnings ` mark to add warning filters to specific test items, allowing you to have finer control of which warnings should be captured at test, class or even module level: @@ -147,10 +150,39 @@ even module level: assert api_v1() == 1 +You can specify multiple filters with separate decorators: + +.. code-block:: python + + # Ignore "api v1" warnings, but fail on all other warnings + @pytest.mark.filterwarnings("ignore:api v1") + @pytest.mark.filterwarnings("error") + def test_one(): + assert api_v1() == 1 + +You can also pass multiple filters to a single mark by providing multiple arguments: + +.. code-block:: python + + # Later arguments take precedence, matching warnings.filterwarnings behavior. + @pytest.mark.filterwarnings("error", "ignore:api v1") + def test_one(): + assert api_v1() == 1 + +.. important:: + + Regarding decorator order and filter precedence: + it's important to remember that decorators are evaluated in reverse order, + so you have to list the warning filters in the reverse order + compared to traditional :py:func:`warnings.filterwarnings` and :option:`-W option ` usage. + This means in practice that filters from earlier :ref:`@pytest.mark.filterwarnings ` decorators + take precedence over filters from later decorators, as illustrated in the example above. + + Filters applied using a mark take precedence over filters passed on the command line or configured -by the ``filterwarnings`` ini option. +by the :confval:`filterwarnings` configuration option. -You may apply a filter to all tests of a class by using the ``filterwarnings`` mark as a class +You may apply a filter to all tests of a class by using the :ref:`filterwarnings ` mark as a class decorator or to all tests in a module by setting the :globalvar:`pytestmark` variable: .. code-block:: python @@ -159,6 +191,13 @@ decorator or to all tests in a module by setting the :globalvar:`pytestmark` var pytestmark = pytest.mark.filterwarnings("error") +.. note:: + + If you want to apply multiple filters + (by assigning a list of :ref:`filterwarnings ` mark to :globalvar:`pytestmark`), + you must use the traditional :py:func:`warnings.filterwarnings` ordering approach (later filters take precedence), + which is the reverse of the decorator approach mentioned above. + *Credits go to Florian Schulze for the reference implementation in the* `pytest-warnings`_ *plugin.* @@ -168,20 +207,29 @@ decorator or to all tests in a module by setting the :globalvar:`pytestmark` var Disabling warnings summary -------------------------- -Although not recommended, you can use the ``--disable-warnings`` command-line option to suppress the +Although not recommended, you can use the :option:`--disable-warnings` command-line option to suppress the warning summary entirely from the test run output. Disabling warning capture entirely ---------------------------------- -This plugin is enabled by default but can be disabled entirely in your ``pytest.ini`` file with: +This plugin is enabled by default but can be disabled entirely in your configuration file with: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + addopts = ["-p", "no:warnings"] + +.. tab:: ini .. code-block:: ini [pytest] addopts = -p no:warnings -Or passing ``-p no:warnings`` in the command-line. This might be useful if your test suites handles warnings +Or passing ``-p no:warnings`` in the command-line. This might be useful if your test suite handles warnings using an external system. @@ -195,20 +243,31 @@ user code and third-party libraries, as recommended by :pep:`565`. This helps users keep their code modern and avoid breakages when deprecated warnings are effectively removed. However, in the specific case where users capture any type of warnings in their test, either with -:func:`pytest.warns`, :func:`pytest.deprecated_call` or using the :ref:`recwarn ` fixture, +:func:`pytest.warns`, :func:`pytest.deprecated_call` or using the :fixture:`recwarn` fixture, no warning will be displayed at all. Sometimes it is useful to hide some specific deprecation warnings that happen in code that you have no control over -(such as third-party libraries), in which case you might use the warning filters options (ini or marks) to ignore +(such as third-party libraries), in which case you might use the warning filters options (configuration or marks) to ignore those warnings. For example: -.. code-block:: ini +.. tab:: toml + + .. code-block:: toml + + [pytest] + filterwarnings = [ + 'ignore:.*U.*mode is deprecated:DeprecationWarning', + ] - [pytest] - filterwarnings = - ignore:.*U.*mode is deprecated:DeprecationWarning +.. tab:: ini + + .. code-block:: ini + + [pytest] + filterwarnings = + ignore:.*U.*mode is deprecated:DeprecationWarning This will ignore all warnings of type ``DeprecationWarning`` where the start of the message matches @@ -223,7 +282,7 @@ See :ref:`@pytest.mark.filterwarnings ` and the :envvar:`python:PYTHONWARNINGS` environment variable or the ``-W`` command-line option, pytest will not configure any filters by default. - Also pytest doesn't follow :pep:`506` suggestion of resetting all warning filters because + Also pytest doesn't follow :pep:`565` suggestion of resetting all warning filters because it might break test suites that configure warning filters themselves by calling :func:`warnings.simplefilter` (see :issue:`2430` for an example of that). @@ -236,8 +295,8 @@ Ensuring code triggers a deprecation warning -------------------------------------------- You can also use :func:`pytest.deprecated_call` for checking -that a certain function call triggers a ``DeprecationWarning`` or -``PendingDeprecationWarning``: +that a certain function call triggers a ``DeprecationWarning``, ``PendingDeprecationWarning`` or +``FutureWarning``: .. code-block:: python @@ -332,10 +391,10 @@ additional information: assert record[0].message.args[0] == "another warning" Alternatively, you can examine raised warnings in detail using the -:ref:`recwarn ` fixture (see below). +:fixture:`recwarn` fixture (see :ref:`below `). -The :ref:`recwarn ` fixture automatically ensures to reset the warnings +The :fixture:`recwarn` fixture automatically ensures to reset the warnings filter at the end of the test, so no global state is leaked. .. _`recording warnings`: @@ -345,8 +404,8 @@ filter at the end of the test, so no global state is leaked. Recording warnings ------------------ -You can record raised warnings either using :func:`pytest.warns` or with -the ``recwarn`` fixture. +You can record raised warnings either using the :func:`pytest.warns` context manager or with +the :fixture:`recwarn` fixture. To record with :func:`pytest.warns` without asserting anything about the warnings, pass no arguments as the expected warning type and it will default to a generic Warning: @@ -361,7 +420,7 @@ pass no arguments as the expected warning type and it will default to a generic assert str(record[0].message) == "user" assert str(record[1].message) == "runtime" -The ``recwarn`` fixture will record warnings for the whole function: +The :fixture:`recwarn` fixture will record warnings for the whole function: .. code-block:: python @@ -377,12 +436,11 @@ The ``recwarn`` fixture will record warnings for the whole function: assert w.filename assert w.lineno -Both ``recwarn`` and :func:`pytest.warns` return the same interface for recorded -warnings: a WarningsRecorder instance. To view the recorded warnings, you can +Both the :fixture:`recwarn` fixture and the :func:`pytest.warns` context manager return the same interface for recorded +warnings: a :class:`~_pytest.recwarn.WarningsRecorder` instance. To view the recorded warnings, you can iterate over this instance, call ``len`` on it to get the number of recorded warnings, or index into it to get a particular recorded warning. -Full API: :class:`~_pytest.recwarn.WarningsRecorder`. .. _`warns use cases`: diff --git a/doc/en/how-to/doctest.rst b/doc/en/how-to/doctest.rst index c2a6cc8e958..59d1033ed4f 100644 --- a/doc/en/how-to/doctest.rst +++ b/doc/en/how-to/doctest.rst @@ -11,7 +11,7 @@ can change the pattern by issuing: pytest --doctest-glob="*.rst" -on the command line. ``--doctest-glob`` can be given multiple times in the command-line. +on the command line. :option:`--doctest-glob` can be given multiple times in the command-line. If you then have a text file like this: @@ -30,7 +30,7 @@ then you can just invoke ``pytest`` directly: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -39,7 +39,7 @@ then you can just invoke ``pytest`` directly: ============================ 1 passed in 0.12s ============================= By default, pytest will collect ``test*.txt`` files looking for doctest directives, but you -can pass additional globs using the ``--doctest-glob`` option (multi-allowed). +can pass additional globs using the :option:`--doctest-glob` option (multi-allowed). In addition to text files, you can also execute doctests directly from docstrings of your classes and functions, including from test modules: @@ -58,7 +58,7 @@ and functions, including from test modules: $ pytest --doctest-modules =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -68,27 +68,34 @@ and functions, including from test modules: ============================ 2 passed in 0.12s ============================= You can make these changes permanent in your project by -putting them into a pytest.ini file like this: +putting them into a configuration file like this: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - addopts = --doctest-modules - + addopts = ["--doctest-modules"] Encoding -------- The default encoding is **UTF-8**, but you can specify the encoding that will be used for those doctest files using the -``doctest_encoding`` ini option: +:confval:`doctest_encoding` configuration option: -.. code-block:: ini +.. tab:: toml - # content of pytest.ini - [pytest] - doctest_encoding = latin1 + .. code-block:: toml + + [pytest] + doctest_encoding = "latin1" + +.. tab:: ini + + .. code-block:: ini + + [pytest] + doctest_encoding = latin1 .. _using doctest options: @@ -102,10 +109,19 @@ configuration file. For example, to make pytest ignore trailing whitespaces and ignore lengthy exception stack traces you can just write: -.. code-block:: ini +.. tab:: toml - [pytest] - doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL + .. code-block:: toml + + [pytest] + doctest_optionflags = ["NORMALIZE_WHITESPACE", "IGNORE_EXCEPTION_DETAIL"] + +.. tab:: ini + + .. code-block:: ini + + [pytest] + doctest_optionflags = NORMALIZE_WHITESPACE IGNORE_EXCEPTION_DETAIL Alternatively, options can be enabled by an inline comment in the doc test itself: @@ -169,7 +185,7 @@ Output format ------------- You can change the diff output format on failure for your doctests -by using one of standard doctest modules format in options +by using one of the standard doctest module's format options (see :data:`python:doctest.REPORT_UDIFF`, :data:`python:doctest.REPORT_CDIFF`, :data:`python:doctest.REPORT_NDIFF`, :data:`python:doctest.REPORT_ONLY_FIRST_FAILURE`): @@ -307,7 +323,7 @@ While the built-in pytest support provides a good set of functionalities for usi doctests, if you use them extensively you might be interested in those external packages which add many more features, and include pytest integration: -* `pytest-doctestplus `__: provides +* `pytest-doctestplus `__: provides advanced doctest support and enables the testing of reStructuredText (".rst") files. * `Sybil `__: provides a way to test examples in diff --git a/doc/en/how-to/failures.rst b/doc/en/how-to/failures.rst index b3d0c155b48..878c869d525 100644 --- a/doc/en/how-to/failures.rst +++ b/doc/en/how-to/failures.rst @@ -93,8 +93,8 @@ Pytest supports the use of ``breakpoint()`` with the following behaviours: - When ``breakpoint()`` is called and ``PYTHONBREAKPOINT`` is set to the default value, pytest will use the custom internal PDB trace UI instead of the system default ``Pdb``. - When tests are complete, the system will default back to the system ``Pdb`` trace UI. - - With ``--pdb`` passed to pytest, the custom internal Pdb trace UI is used with both ``breakpoint()`` and failed tests/unhandled exceptions. - - ``--pdbcls`` can be used to specify a custom debugger class. + - With :option:`--pdb` passed to pytest, the custom internal Pdb trace UI is used with both ``breakpoint()`` and failed tests/unhandled exceptions. + - :option:`--pdbcls` can be used to specify a custom debugger class. .. _faulthandler: @@ -112,7 +112,7 @@ on the command-line. Also the :confval:`faulthandler_timeout=X` configuration option can be used to dump the traceback of all threads if a test takes longer than ``X`` -seconds to finish (not available on Windows). +seconds to finish. .. note:: diff --git a/doc/en/how-to/fixtures.rst b/doc/en/how-to/fixtures.rst index ecd297867c5..2f554fb8c60 100644 --- a/doc/en/how-to/fixtures.rst +++ b/doc/en/how-to/fixtures.rst @@ -433,7 +433,7 @@ marked ``smtp_connection`` fixture function. Running the test looks like this: $ pytest test_module.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -449,6 +449,7 @@ marked ``smtp_connection`` fixture function. Running the test looks like this: assert response == 250 assert b"smtp.gmail.com" in msg > assert 0 # for demo purposes + ^^^^^^^^ E assert 0 test_module.py:7: AssertionError @@ -460,6 +461,7 @@ marked ``smtp_connection`` fixture function. Running the test looks like this: response, msg = smtp_connection.noop() assert response == 250 > assert 0 # for demo purposes + ^^^^^^^^ E assert 0 test_module.py:13: AssertionError @@ -469,7 +471,7 @@ marked ``smtp_connection`` fixture function. Running the test looks like this: ============================ 2 failed in 0.12s ============================= You see the two ``assert 0`` failing and more importantly you can also see -that the **exactly same** ``smtp_connection`` object was passed into the +that the **exact same** ``smtp_connection`` object was passed into the two test functions because pytest shows the incoming argument values in the traceback. As a result, the two test functions using ``smtp_connection`` run as quick as a single one because they reuse the same instance. @@ -771,7 +773,7 @@ For yield fixtures, the first teardown code to run is from the right-most fixtur $ pytest -s test_finalizers.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -805,7 +807,7 @@ For finalizers, the first fixture to run is last call to `request.addfinalizer`. $ pytest -s test_finalizers.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -1308,6 +1310,7 @@ So let's just do another run: assert response == 250 assert b"smtp.gmail.com" in msg > assert 0 # for demo purposes + ^^^^^^^^ E assert 0 test_module.py:7: AssertionError @@ -1319,6 +1322,7 @@ So let's just do another run: response, msg = smtp_connection.noop() assert response == 250 > assert 0 # for demo purposes + ^^^^^^^^ E assert 0 test_module.py:13: AssertionError @@ -1343,6 +1347,7 @@ So let's just do another run: response, msg = smtp_connection.noop() assert response == 250 > assert 0 # for demo purposes + ^^^^^^^^ E assert 0 test_module.py:13: AssertionError @@ -1363,9 +1368,9 @@ different server string is expected than what arrived. pytest will build a string that is the test ID for each fixture value in a parametrized fixture, e.g. ``test_ehlo[smtp.gmail.com]`` and ``test_ehlo[mail.python.org]`` in the above examples. These IDs can -be used with ``-k`` to select specific cases to run, and they will +be used with :option:`-k` to select specific cases to run, and they will also identify the specific case when one is failing. Running pytest -with ``--collect-only`` will show the generated IDs. +with :option:`--collect-only` will show the generated IDs. Numbers, strings, booleans and ``None`` will have their usual string representation used in the test ID. For other objects, pytest will @@ -1414,11 +1419,11 @@ Running the above tests results in the following test IDs being used: $ pytest --collect-only =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 12 items - + @@ -1469,7 +1474,7 @@ Running this test will *skip* the invocation of ``data_set`` with value ``2``: $ pytest test_fixture_marks.py -v =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 3 items @@ -1487,7 +1492,7 @@ Modularity: using fixtures from a fixture function In addition to using fixtures in test functions, fixture functions can use other fixtures themselves. This contributes to a modular design -of your fixtures and allows re-use of framework-specific fixtures across +of your fixtures and allows reuse of framework-specific fixtures across many projects. As a simple example, we can extend the previous example and instantiate an object ``app`` where we stick the already defined ``smtp_connection`` resource into it: @@ -1519,7 +1524,7 @@ Here we declare an ``app`` fixture which receives the previously defined $ pytest -v test_appsetup.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 2 items @@ -1599,7 +1604,7 @@ Let's run the tests in verbose mode and with looking at the print-output: $ pytest -v -s test_module.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y -- $PYTHON_PREFIX/bin/python cachedir: .pytest_cache rootdir: /home/sweet/project collecting ... collected 8 items @@ -1640,7 +1645,7 @@ Let's run the tests in verbose mode and with looking at the print-output: ============================ 8 passed in 0.12s ============================= You can see that the parametrized module-scoped ``modarg`` resource caused an -ordering of test execution that lead to the fewest possible "active" resources. +ordering of test execution that led to the fewest possible "active" resources. The finalizer for the ``mod1`` parametrized resource was executed before the ``mod2`` resource was setup. @@ -1649,7 +1654,7 @@ Then test_1 is executed with ``mod1``, then test_2 with ``mod1``, then test_1 with ``mod2`` and finally test_2 with ``mod2``. The ``otherarg`` parametrized resource (having function scope) was set up before -and teared down after every test that used it. +and torn down after every test that used it. .. _`usefixtures`: @@ -1731,14 +1736,13 @@ and you may specify fixture usage at the test module level using :globalvar:`pyt It is also possible to put fixtures required by all tests in your project -into an ini-file: +into a configuration file: -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - usefixtures = cleandir - + usefixtures = ["cleandir"] .. warning:: @@ -1758,8 +1762,8 @@ into an ini-file: Overriding fixtures on various levels ------------------------------------- -In relatively large test suite, you most likely need to ``override`` a ``global`` or ``root`` fixture with a ``locally`` -defined one, keeping the test code readable and maintainable. +In a relatively large test suite, you may want to *override* a fixture, to augment +or change its behavior inside of certain test modules or directories. Override a fixture on a folder (conftest) level ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -1796,7 +1800,7 @@ Given the tests file structure is: def test_username(username): assert username == 'overridden-username' -As you can see, a fixture with the same name can be overridden for certain test folder level. +As you can see, a fixture with the same name can be overridden for a certain test directory level. Note that the ``base`` or ``super`` fixture can be accessed from the ``overriding`` fixture easily - used in the example above. @@ -1838,7 +1842,7 @@ Given the tests file structure is: def test_username(username): assert username == 'overridden-else-username' -In the example above, a fixture with the same name can be overridden for certain test module. +In the example above, a fixture with the same name can be overridden for a certain test module. Override a fixture with direct test parametrization diff --git a/doc/en/how-to/index.rst b/doc/en/how-to/index.rst index 225f289651e..9796f1f8090 100644 --- a/doc/en/how-to/index.rst +++ b/doc/en/how-to/index.rst @@ -16,6 +16,7 @@ Core pytest functionality fixtures mark parametrize + subtests tmp_path monkeypatch doctest diff --git a/doc/en/how-to/logging.rst b/doc/en/how-to/logging.rst index 300e9f6e6c2..25b4e9017e2 100644 --- a/doc/en/how-to/logging.rst +++ b/doc/en/how-to/logging.rst @@ -47,15 +47,25 @@ Shows failed tests like so: text going to stderr ==================== 2 failed in 0.02 seconds ===================== -These options can also be customized through ``pytest.ini`` file: +These options can also be customized through a configuration file: -.. code-block:: ini +.. tab:: toml - [pytest] - log_format = %(asctime)s %(levelname)s %(message)s - log_date_format = %Y-%m-%d %H:%M:%S + .. code-block:: toml -Specific loggers can be disabled via ``--log-disable={logger_name}``. + [pytest] + log_format = "%(asctime)s %(levelname)s %(message)s" + log_date_format = "%Y-%m-%d %H:%M:%S" + +.. tab:: ini + + .. code-block:: ini + + [pytest] + log_format = %(asctime)s %(levelname)s %(message)s + log_date_format = %Y-%m-%d %H:%M:%S + +Specific loggers can be disabled via :option:`--log-disable={logger_name}`. This argument can be passed multiple times: .. code-block:: bash @@ -189,48 +199,48 @@ By setting the :confval:`log_cli` configuration option to ``true``, pytest will logging records as they are emitted directly into the console. You can specify the logging level for which log records with equal or higher -level are printed to the console by passing ``--log-cli-level``. This setting +level are printed to the console by passing :option:`--log-cli-level`. This setting accepts the logging level names or numeric values as seen in :ref:`logging's documentation `. -Additionally, you can also specify ``--log-cli-format`` and -``--log-cli-date-format`` which mirror and default to ``--log-format`` and -``--log-date-format`` if not provided, but are applied only to the console +Additionally, you can also specify :option:`--log-cli-format` and +:option:`--log-cli-date-format` which mirror and default to :option:`--log-format` and +:option:`--log-date-format` if not provided, but are applied only to the console logging handler. -All of the CLI log options can also be set in the configuration INI file. The +All of the CLI log options can also be set in the configuration file. The option names are: -* ``log_cli_level`` -* ``log_cli_format`` -* ``log_cli_date_format`` +* :confval:`log_cli_level` +* :confval:`log_cli_format` +* :confval:`log_cli_date_format` If you need to record the whole test suite logging calls to a file, you can pass -``--log-file=/path/to/log/file``. This log file is opened in write mode by default which -means that it will be overwritten at each run tests session. -If you'd like the file opened in append mode instead, then you can pass ``--log-file-mode=a``. +:option:`--log-file=/path/to/log/file`. This log file is opened in write mode by default, which +means that it will be overwritten at each test session. +If you'd like the file opened in append mode instead, then you can pass :option:`--log-file-mode=a`. Note that relative paths for the log-file location, whether passed on the CLI or declared in a config file, are always resolved relative to the current working directory. You can also specify the logging level for the log file by passing -``--log-file-level``. This setting accepts the logging level names or numeric +:option:`--log-file-level`. This setting accepts the logging level names or numeric values as seen in :ref:`logging's documentation `. -Additionally, you can also specify ``--log-file-format`` and -``--log-file-date-format`` which are equal to ``--log-format`` and -``--log-date-format`` but are applied to the log file logging handler. +Additionally, you can also specify :option:`--log-file-format` and +:option:`--log-file-date-format` which are equal to ``--log-format`` and +:option:`--log-date-format` but are applied to the log file logging handler. -All of the log file options can also be set in the configuration INI file. The +All of the log file options can also be set in the configuration file. The option names are: -* ``log_file`` -* ``log_file_mode`` -* ``log_file_level`` -* ``log_file_format`` -* ``log_file_date_format`` +* :confval:`log_file` +* :confval:`log_file_mode` +* :confval:`log_file_level` +* :confval:`log_file_format` +* :confval:`log_file_date_format` You can call ``set_log_path()`` to customize the log_file path dynamically. This functionality -is considered **experimental**. Note that ``set_log_path()`` respects the ``log_file_mode`` option. +is considered **experimental**. Note that ``set_log_path()`` respects the :confval:`log_file_mode` option. .. _log_colors: @@ -266,12 +276,21 @@ This feature was introduced as a drop-in replacement for the with each other. The backward compatibility API with ``pytest-capturelog`` has been dropped when this feature was introduced, so if for that reason you still need ``pytest-catchlog`` you can disable the internal feature by -adding to your ``pytest.ini``: +adding to your configuration file: + +.. tab:: toml + + .. code-block:: toml -.. code-block:: ini + [pytest] + addopts = ["-p", "no:logging"] - [pytest] - addopts=-p no:logging +.. tab:: ini + + .. code-block:: ini + + [pytest] + addopts = -p no:logging .. _log_changes_3_4: @@ -283,23 +302,33 @@ This feature was introduced in ``3.3`` and some **incompatible changes** have be made in ``3.4`` after community feedback: * Log levels are no longer changed unless explicitly requested by the :confval:`log_level` configuration - or ``--log-level`` command-line options. This allows users to configure logger objects themselves. + or :option:`--log-level` command-line options. This allows users to configure logger objects themselves. Setting :confval:`log_level` will set the level that is captured globally so if a specific test requires a lower level than this, use the ``caplog.set_level()`` functionality otherwise that test will be prone to failure. * :ref:`Live Logs ` is now disabled by default and can be enabled setting the :confval:`log_cli` configuration option to ``true``. When enabled, the verbosity is increased so logging for each test is visible. -* :ref:`Live Logs ` are now sent to ``sys.stdout`` and no longer require the ``-s`` command-line option +* :ref:`Live Logs ` are now sent to ``sys.stdout`` and no longer require the :option:`-s` command-line option to work. -If you want to partially restore the logging behavior of version ``3.3``, you can add this options to your ``ini`` +If you want to partially restore the logging behavior of version ``3.3``, you can add these options to your configuration file: -.. code-block:: ini +.. tab:: toml + + .. code-block:: toml + + [pytest] + log_cli = true + log_level = "NOTSET" + +.. tab:: ini + + .. code-block:: ini - [pytest] - log_cli=true - log_level=NOTSET + [pytest] + log_cli = true + log_level = NOTSET -More details about the discussion that lead to this changes can be read in :issue:`3013`. +More details about the discussion that led to these changes can be read in :issue:`3013`. diff --git a/doc/en/how-to/mark.rst b/doc/en/how-to/mark.rst index 33f9d18bfe3..e22219414a0 100644 --- a/doc/en/how-to/mark.rst +++ b/doc/en/how-to/mark.rst @@ -21,7 +21,7 @@ Here are some of the builtin markers: It's easy to create custom markers or to apply markers to whole test classes or modules. Those markers can be used by plugins, and also -are commonly used to :ref:`select tests ` on the command-line with the ``-m`` option. +are commonly used to :ref:`select tests ` on the command-line with the :option:`-m` option. See :ref:`mark examples` for examples which also serve as documentation. @@ -34,24 +34,26 @@ See :ref:`mark examples` for examples which also serve as documentation. Registering marks ----------------- -You can register custom marks in your ``pytest.ini`` file like this: +You can register custom marks in your configuration file like this: -.. code-block:: ini +.. tab:: toml - [pytest] - markers = - slow: marks tests as slow (deselect with '-m "not slow"') - serial + .. code-block:: toml -or in your ``pyproject.toml`` file like this: + [pytest] + markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "serial", + ] -.. code-block:: toml +.. tab:: ini - [tool.pytest.ini_options] - markers = [ - "slow: marks tests as slow (deselect with '-m \"not slow\"')", - "serial", - ] + .. code-block:: ini + + [pytest] + markers = + slow: marks tests as slow (deselect with '-m "not slow"') + serial Note that everything past the ``:`` after the mark name is an optional description. @@ -77,17 +79,30 @@ Raising errors on unknown marks Unregistered marks applied with the ``@pytest.mark.name_of_the_mark`` decorator will always emit a warning in order to avoid silently doing something surprising due to mistyped names. As described in the previous section, you can disable -the warning for custom marks by registering them in your ``pytest.ini`` file or +the warning for custom marks by registering them in your configuration file or using a custom ``pytest_configure`` hook. -When the ``--strict-markers`` command-line flag is passed, any unknown marks applied +When the :confval:`strict_markers` configuration option is set, any unknown marks applied with the ``@pytest.mark.name_of_the_mark`` decorator will trigger an error. You can -enforce this validation in your project by adding ``--strict-markers`` to ``addopts``: +enforce this validation in your project by setting :confval:`strict_markers` in your configuration: + +.. tab:: toml + + .. code-block:: toml + + [pytest] + addopts = ["--strict-markers"] + markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "serial", + ] + +.. tab:: ini -.. code-block:: ini + .. code-block:: ini - [pytest] - addopts = --strict-markers - markers = - slow: marks tests as slow (deselect with '-m "not slow"') - serial + [pytest] + strict_markers = true + markers = + slow: marks tests as slow (deselect with '-m "not slow"') + serial diff --git a/doc/en/how-to/monkeypatch.rst b/doc/en/how-to/monkeypatch.rst index a9504dcb32a..7442a85c10e 100644 --- a/doc/en/how-to/monkeypatch.rst +++ b/doc/en/how-to/monkeypatch.rst @@ -235,7 +235,7 @@ so that any attempts within tests to create http requests will fail. Be advised that it is not recommended to patch builtin functions such as ``open``, ``compile``, etc., because it might break pytest's internals. If that's - unavoidable, passing ``--tb=native``, ``--assert=plain`` and ``--capture=no`` might + unavoidable, passing :option:`--tb=native`, :option:`--assert=plain` and :option:`--capture=no` might help although there's no guarantee. .. note:: @@ -382,7 +382,7 @@ You can use the :py:meth:`monkeypatch.delitem ` to remove v def test_missing_user(monkeypatch): - # patch the DEFAULT_CONFIG t be missing the 'user' key + # patch the DEFAULT_CONFIG to be missing the 'user' key monkeypatch.delitem(app.DEFAULT_CONFIG, "user", raising=False) # Key error expected because a config is not passed, and the diff --git a/doc/en/how-to/output.rst b/doc/en/how-to/output.rst index 4994ad1af69..a594fcb3aab 100644 --- a/doc/en/how-to/output.rst +++ b/doc/en/how-to/output.rst @@ -30,8 +30,8 @@ Examples for modifying traceback printing: pytest --tb=native # Python standard library formatting pytest --tb=no # no traceback at all -The ``--full-trace`` causes very long traces to be printed on error (longer -than ``--tb=long``). It also ensures that a stack trace is printed on +The :option:`--full-trace` causes very long traces to be printed on error (longer +than :option:`--tb=long`). It also ensures that a stack trace is printed on **KeyboardInterrupt** (Ctrl+C). This is very useful if the tests are taking too long and you interrupt them with Ctrl+C to find out where the tests are *hanging*. By default no output @@ -52,8 +52,8 @@ Examples for modifying printing verbosity: pytest -vv # more verbose, display more details from the test output pytest -vvv # not a standard , but may be used for even more detail in certain setups -The ``-v`` flag controls the verbosity of pytest output in various aspects: test session progress, assertion -details when tests fail, fixtures details with ``--fixtures``, etc. +The :option:`-v` flag controls the verbosity of pytest output in various aspects: test session progress, assertion +details when tests fail, fixtures details with :option:`--fixtures`, etc. .. regendoc:wipe @@ -372,7 +372,7 @@ test inside the file gets its own line in the output. Producing a detailed summary report -------------------------------------------------- -The ``-r`` flag can be used to display a "short test summary info" at the end of the test session, +The :option:`-r` flag can be used to display a "short test summary info" at the end of the test session, making it easy in large test suites to get a clear picture of all failures, skips, xfails, etc. It defaults to ``fE`` to list failures and errors. @@ -421,7 +421,7 @@ Example: $ pytest -ra =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 6 items @@ -444,24 +444,16 @@ Example: E assert 0 test_example.py:14: AssertionError - ================================ XFAILURES ================================= - ________________________________ test_xfail ________________________________ - - def test_xfail(): - > pytest.xfail("xfailing this test") - E _pytest.outcomes.XFailed: xfailing this test - - test_example.py:26: XFailed ================================= XPASSES ================================== ========================= short test summary info ========================== SKIPPED [1] test_example.py:22: skipping this test - XFAIL test_example.py::test_xfail - reason: xfailing this test + XFAIL test_example.py::test_xfail - xfailing this test XPASS test_example.py::test_xpass - always xfail ERROR test_example.py::test_error - assert 0 FAILED test_example.py::test_fail - assert 0 == 1 failed, 1 passed, 1 skipped, 1 xfailed, 1 xpassed, 1 error in 0.12s === -The ``-r`` options accepts a number of characters after it, with ``a`` used +The :option:`-r` options accepts a number of characters after it, with ``a`` used above meaning "all except passes". Here is the full list of available characters that can be used: @@ -486,7 +478,7 @@ More than one character can be used, so for example to only see failed and skipp $ pytest -rfs =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 6 items @@ -521,7 +513,7 @@ captured output: $ pytest -rpP =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 6 items @@ -555,30 +547,39 @@ captured output: .. note:: By default, parametrized variants of skipped tests are grouped together if - they share the same skip reason. You can use ``--no-fold-skipped`` to print each skipped test separately. + they share the same skip reason. You can use :option:`--no-fold-skipped` to print each skipped test separately. + + +.. _truncation-params: -Creating resultlog format files +Modifying truncation limits -------------------------------------------------- -To create plain-text machine-readable result files you can issue: +.. versionadded: 8.4 -.. code-block:: bash +Default truncation limits are 8 lines or 640 characters, whichever comes first. +To set custom truncation limits you can use the following configuration file options: - pytest --resultlog=path +.. tab:: toml -and look at the content at the ``path`` location. Such files are used e.g. -by the `PyPy-test`_ web page to show test results over several revisions. + .. code-block:: toml -.. warning:: + [pytest] + truncation_limit_lines = 10 + truncation_limit_chars = 90 - This option is rarely used and is scheduled for removal in pytest 6.0. +.. tab:: ini - If you use this option, consider using the new `pytest-reportlog `__ plugin instead. + .. code-block:: ini - See :ref:`the deprecation docs ` for more information. + [pytest] + truncation_limit_lines = 10 + truncation_limit_chars = 90 +That will cause pytest to truncate the assertions to 10 lines or 90 characters, whichever comes first. -.. _`PyPy-test`: http://buildbot.pypy.org/summary +Setting both :confval:`truncation_limit_lines` and :confval:`truncation_limit_chars` to ``0`` will disable the truncation. +However, setting only one of those values will disable one truncation mode, but will leave the other one intact. Creating JUnitXML format files @@ -597,10 +598,19 @@ to create an XML file at ``path``. To set the name of the root test suite xml item, you can configure the ``junit_suite_name`` option in your config file: -.. code-block:: ini +.. tab:: toml + + .. code-block:: toml - [pytest] - junit_suite_name = my_suite + [pytest] + junit_suite_name = "my_suite" + +.. tab:: ini + + .. code-block:: ini + + [pytest] + junit_suite_name = my_suite .. versionadded:: 4.0 @@ -611,10 +621,19 @@ should report total test execution times, including setup and teardown It is the default pytest behavior. To report just call durations instead, configure the ``junit_duration_report`` option like this: -.. code-block:: ini +.. tab:: toml + + .. code-block:: toml + + [pytest] + junit_duration_report = "call" + +.. tab:: ini + + .. code-block:: ini - [pytest] - junit_duration_report = call + [pytest] + junit_duration_report = call .. _record_property example: @@ -752,7 +771,7 @@ record_testsuite_property .. versionadded:: 4.5 -If you want to add a properties node at the test-suite level, which may contains properties +If you want to add a properties node at the test-suite level, which may contain properties that are relevant to all tests, you can use the ``record_testsuite_property`` session-scoped fixture: The ``record_testsuite_property`` session-scoped fixture can be used to add properties relevant @@ -803,7 +822,7 @@ Sending test report to an online pastebin service This will submit test run information to a remote Paste service and provide a URL for each failure. You may select tests as usual or add -for example ``-x`` if you only want to send one particular failure. +for example :option:`-x` if you only want to send one particular failure. **Creating a URL for a whole test session log**: diff --git a/doc/en/how-to/parametrize.rst b/doc/en/how-to/parametrize.rst index b6466c491b4..5de28472705 100644 --- a/doc/en/how-to/parametrize.rst +++ b/doc/en/how-to/parametrize.rst @@ -20,6 +20,11 @@ pytest enables test parametrization at several levels: * `pytest_generate_tests`_ allows one to define custom parametrization schemes or extensions. + +.. note:: + + See :ref:`subtests` for an alternative to parametrization. + .. _parametrizemark: .. _`@pytest.mark.parametrize`: @@ -29,10 +34,6 @@ pytest enables test parametrization at several levels: .. regendoc: wipe - - - Several improvements. - The builtin :ref:`pytest.mark.parametrize ref` decorator enables parametrization of arguments for a test function. Here is a typical example of a test function that implements checking that a certain input leads @@ -56,7 +57,7 @@ them in turn: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 3 items @@ -92,12 +93,21 @@ them in turn: for the parametrization because it has several downsides. If however you would like to use unicode strings in parametrization and see them in the terminal as is (non-escaped), use this option - in your ``pytest.ini``: + in your configuration file: + + .. tab:: toml - .. code-block:: ini + .. code-block:: toml - [pytest] - disable_test_id_escaping_and_forfeit_all_rights_to_community_support = True + [pytest] + disable_test_id_escaping_and_forfeit_all_rights_to_community_support = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + disable_test_id_escaping_and_forfeit_all_rights_to_community_support = true Keep in mind however that this might cause unwanted side effects and even bugs depending on the OS used and plugins currently installed, @@ -167,7 +177,7 @@ Let's run this: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 3 items @@ -198,6 +208,7 @@ To get all combinations of multiple parametrized arguments you can stack This will run the test with the arguments set to ``x=0/y=2``, ``x=1/y=2``, ``x=0/y=3``, and ``x=1/y=3`` exhausting parameters in the order of the decorators. + .. _`pytest_generate_tests`: Basic ``pytest_generate_tests`` example @@ -244,6 +255,13 @@ command line option and the parametrization of our test function: if "stringinput" in metafunc.fixturenames: metafunc.parametrize("stringinput", metafunc.config.getoption("stringinput")) +.. note:: + + The :hook:`pytest_generate_tests` hook can also be implemented directly in a test + module or inside a test class; unlike other hooks, pytest will discover it there + as well. Other hooks must live in a :ref:`conftest.py ` or a plugin. + See :ref:`writinghooks`. + If we now pass two stringinput values, our test will run twice: .. code-block:: pytest @@ -285,7 +303,7 @@ list: $ pytest -q -rs test_strings.py s [100%] ========================= short test summary info ========================== - SKIPPED [1] test_strings.py: got empty parameter set ['stringinput'], function test_valid_string at /home/sweet/project/test_strings.py:2 + SKIPPED [1] test_strings.py: got empty parameter set for (stringinput) 1 skipped in 0.12s Note that when calling ``metafunc.parametrize`` multiple times with different parameter sets, all parameter names across diff --git a/doc/en/how-to/plugins.rst b/doc/en/how-to/plugins.rst index 7d5bcd85a31..c6641eb8484 100644 --- a/doc/en/how-to/plugins.rst +++ b/doc/en/how-to/plugins.rst @@ -120,12 +120,21 @@ This means that any subsequent try to activate/load the named plugin will not work. If you want to unconditionally disable a plugin for a project, you can add -this option to your ``pytest.ini`` file: +this option to your configuration file: -.. code-block:: ini +.. tab:: toml - [pytest] - addopts = -p no:NAME + .. code-block:: toml + + [pytest] + addopts = ["-p", "no:NAME"] + +.. tab:: ini + + .. code-block:: ini + + [pytest] + addopts = -p no:NAME Alternatively to disable it only in certain environments (for example in a CI server), you can set ``PYTEST_ADDOPTS`` environment variable to @@ -133,4 +142,69 @@ CI server), you can set ``PYTEST_ADDOPTS`` environment variable to See :ref:`findpluginname` for how to obtain the name of a plugin. -.. _`builtin plugins`: +.. _`disable_plugin_autoload`: + +Disabling plugins from autoloading +---------------------------------- + +If you want to disable plugins from loading automatically, instead of requiring you to +manually specify each plugin with :option:`-p` or :envvar:`PYTEST_PLUGINS`, you can use :option:`--disable-plugin-autoload` or :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD`. + +.. code-block:: bash + + export PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 + export PYTEST_PLUGINS=NAME,NAME2 + pytest + +.. code-block:: bash + + pytest --disable-plugin-autoload -p NAME -p NAME2 + +.. tab:: toml + + .. code-block:: toml + + [pytest] + addopts = ["--disable-plugin-autoload", "-p", "NAME", "-p", "NAME2"] + +.. tab:: ini + + .. code-block:: ini + + [pytest] + addopts = + --disable-plugin-autoload + -p NAME + -p NAME2 + +.. versionadded:: 8.4 + + The :option:`--disable-plugin-autoload` command-line flag. + +.. note:: + + :option:`-p` and :envvar:`PYTEST_PLUGINS` are both ways to explicitly control which + plugins are loaded, but they serve slightly different use-cases. + + * :option:`-p` loads (or disables with ``-p no:``) a plugin by name or entry point + for a specific pytest invocation, and is processed early during startup. + * :envvar:`PYTEST_PLUGINS` is a comma-separated list of Python modules that are imported + and registered as plugins during startup. This mechanism is commonly used by test + suites, for example when testing a plugin. + + When explicitly controlling plugin loading (especially with + :envvar:`PYTEST_DISABLE_PLUGIN_AUTOLOAD` or :option:`--disable-plugin-autoload`), + avoid specifying the same plugin via multiple mechanisms. Registering the same plugin + more than once can lead to errors during plugin registration. + +Examples: + +.. code-block:: bash + + # Disable auto-loading and load only specific plugins for this invocation + PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest -p xdist + +.. code-block:: bash + + # Disable auto-loading and load plugin modules during startup + PYTEST_DISABLE_PLUGIN_AUTOLOAD=1 PYTEST_PLUGINS=mymodule.plugin,xdist pytest diff --git a/doc/en/how-to/skipping.rst b/doc/en/how-to/skipping.rst index 09a19766f99..488f71b09f9 100644 --- a/doc/en/how-to/skipping.rst +++ b/doc/en/how-to/skipping.rst @@ -21,14 +21,14 @@ it's an **xpass** and will be reported in the test summary. ``pytest`` counts and lists *skip* and *xfail* tests separately. Detailed information about skipped/xfailed tests is not shown by default to avoid -cluttering the output. You can use the ``-r`` option to see details +cluttering the output. You can use the :option:`-r` option to see details corresponding to the "short" letters shown in the test progress: .. code-block:: bash pytest -rxXs # show extra info on xfailed, xpassed, and skipped tests -More details on the ``-r`` option can be found by running ``pytest -h``. +More details on the :option:`-r` option can be found by running ``pytest -h``. (See :ref:`how to change command line options defaults`) @@ -84,14 +84,14 @@ It is also possible to skip the whole module using If you wish to skip something conditionally then you can use ``skipif`` instead. Here is an example of marking a test function to be skipped -when run on an interpreter earlier than Python3.10: +when run on an interpreter earlier than Python3.13: .. code-block:: python import sys - @pytest.mark.skipif(sys.version_info < (3, 10), reason="requires python3.10 or higher") + @pytest.mark.skipif(sys.version_info < (3, 13), reason="requires python3.13 or higher") def test_function(): ... If the condition evaluates to ``True`` during collection, the test function will be skipped, @@ -311,7 +311,7 @@ even executed, use the ``run`` parameter as ``False``: @pytest.mark.xfail(run=False) def test_function(): ... -This is specially useful for xfailing tests that are crashing the interpreter and should be +This is particularly useful for xfailing tests that are crashing the interpreter and should be investigated later. .. _`xfail strict tutorial`: @@ -331,12 +331,21 @@ You can change this by setting the ``strict`` keyword-only parameter to ``True`` This will make ``XPASS`` ("unexpectedly passing") results from this test to fail the test suite. You can change the default value of the ``strict`` parameter using the -``xfail_strict`` ini option: +``strict_xfail`` ini option: -.. code-block:: ini +.. tab:: toml - [pytest] - xfail_strict=true + .. code-block:: toml + + [pytest] + xfail_strict = true + +.. tab:: ini + + .. code-block:: ini + + [pytest] + strict_xfail = true Ignoring xfail diff --git a/doc/en/how-to/subtests.rst b/doc/en/how-to/subtests.rst new file mode 100644 index 00000000000..93b9d052afd --- /dev/null +++ b/doc/en/how-to/subtests.rst @@ -0,0 +1,139 @@ +.. _subtests: + +How to use subtests +=================== + +.. versionadded:: 9.0 + +.. note:: + + This feature is experimental. Its behavior, particularly how failures are reported, may evolve in future releases. However, the core functionality and usage are considered stable. + +pytest allows for grouping assertions within a normal test, known as *subtests*. + +Subtests are an alternative to parametrization, particularly useful when the exact parametrization values are not known at collection time. + + +.. code-block:: python + + # content of test_subtest.py + + + def test(subtests): + for i in range(5): + with subtests.test(msg="custom message", i=i): + assert i % 2 == 0 + +Each assertion failure or error is caught by the context manager and reported individually: + +.. code-block:: pytest + + $ pytest -q test_subtest.py + uuuuuF [100%] + ================================= FAILURES ================================= + _______________________ test [custom message] (i=1) ________________________ + + subtests = <_pytest.subtests.Subtests object at 0xdeadbeef0001> + + def test(subtests): + for i in range(5): + with subtests.test(msg="custom message", i=i): + > assert i % 2 == 0 + E assert (1 % 2) == 0 + + test_subtest.py:6: AssertionError + _______________________ test [custom message] (i=3) ________________________ + + subtests = <_pytest.subtests.Subtests object at 0xdeadbeef0001> + + def test(subtests): + for i in range(5): + with subtests.test(msg="custom message", i=i): + > assert i % 2 == 0 + E assert (3 % 2) == 0 + + test_subtest.py:6: AssertionError + ___________________________________ test ___________________________________ + contains 2 failed subtests + ========================= short test summary info ========================== + SUBFAILED[custom message] (i=1) test_subtest.py::test - assert (1 % 2) == 0 + SUBFAILED[custom message] (i=3) test_subtest.py::test - assert (3 % 2) == 0 + FAILED test_subtest.py::test - contains 2 failed subtests + 3 failed, 3 subtests passed in 0.12s + +In the output above: + +* Subtest failures are reported as ``SUBFAILED``. +* Subtests are reported first and the "top-level" test is reported at the end on its own. + +Note that it is possible to use ``subtests`` multiple times in the same test, or even mix and match with normal assertions +outside the ``subtests.test`` block: + +.. code-block:: python + + def test(subtests): + for i in range(5): + with subtests.test("stage 1", i=i): + assert i % 2 == 0 + + assert func() == 10 + + for i in range(10, 20): + with subtests.test("stage 2", i=i): + assert i % 2 == 0 + +.. note:: + + See :ref:`parametrize` for an alternative to subtests. + + +Verbosity +--------- + +By default, only **subtest failures** are shown. Higher verbosity levels (:option:`-v`) will also show progress output for **passed** subtests. + +It is possible to control the verbosity of subtests by setting :confval:`verbosity_subtests`. + + +Typing +------ + +:class:`pytest.Subtests` is exported so it can be used in type annotations: + +.. code-block:: python + + def test(subtests: pytest.Subtests) -> None: ... + +.. _parametrize_vs_subtests: + +Parametrization vs Subtests +--------------------------- + +While :ref:`traditional pytest parametrization ` and ``subtests`` are similar, they have important differences and use cases. + + +Parametrization +~~~~~~~~~~~~~~~ + +* Happens at collection time. +* Generates individual tests. +* Parametrized tests can be referenced from the command line. +* Plays well with plugins that handle test execution, such as :option:`--last-failed`. +* Ideal for decision table testing. + +Subtests +~~~~~~~~ + +* Happen during test execution. +* Are not known at collection time. +* Can be generated dynamically. +* Cannot be referenced individually from the command line. +* Plugins that handle test execution cannot target individual subtests. +* An assertion failure inside a subtest does not interrupt the test, letting users see all failures in the same report. + + +.. note:: + + This feature was originally implemented as a separate plugin in `pytest-subtests `__, but since ``9.0`` has been merged into the core. + + The core implementation should be compatible with the plugin implementation, except it does not contain custom command-line options to control subtest output. diff --git a/doc/en/how-to/tmp_path.rst b/doc/en/how-to/tmp_path.rst index 3cc5152e992..e73c55878a6 100644 --- a/doc/en/how-to/tmp_path.rst +++ b/doc/en/how-to/tmp_path.rst @@ -35,7 +35,7 @@ Running this would result in a passed test except for the last $ pytest test_tmp_path.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -133,27 +133,47 @@ API for details. Temporary directory location and retention ------------------------------------------ -Temporary directories are by default created as sub-directories of -the system temporary directory. The base name will be ``pytest-NUM`` where -``NUM`` will be incremented with each test run. -By default, entries older than 3 temporary directories will be removed. -This behavior can be configured with :confval:`tmp_path_retention_count` and -:confval:`tmp_path_retention_policy`. +The temporary directories, +as returned by the :fixture:`tmp_path` and (now deprecated) :fixture:`tmpdir` fixtures, +are automatically created under a base temporary directory, +in a structure that depends on the :option:`--basetemp` option: -Using the ``--basetemp`` -option will remove the directory before every run, effectively meaning the temporary directories -of only the most recent run will be kept. +- By default (when the :option:`--basetemp` option is not set), + the temporary directories will follow this template: -You can override the default temporary directory setting like this: + .. code-block:: text -.. code-block:: bash + {temproot}/pytest-of-{user}/pytest-{num}/{testname}/ - pytest --basetemp=mydir + where: -.. warning:: + - ``{temproot}`` is the system temporary directory + as determined by :py:func:`tempfile.gettempdir`. + It can be overridden by the :envvar:`PYTEST_DEBUG_TEMPROOT` environment variable. + - ``{user}`` is the user name running the tests, + - ``{num}`` is a number that is incremented with each test suite run + - ``{testname}`` is a sanitized version of :py:attr:`the name of the current test <_pytest.nodes.Node.name>`. - The contents of ``mydir`` will be completely removed, so make sure to use a directory - for that purpose only. + The auto-incrementing ``{num}`` placeholder provides a basic retention feature + and avoids that existing results of previous test runs are blindly removed. + By default, the last 3 temporary directories are kept, + but this behavior can be configured with + :confval:`tmp_path_retention_count` and :confval:`tmp_path_retention_policy`. + +- When the :option:`--basetemp` option is used (e.g. ``pytest --basetemp=mydir``), + it will be used directly as base temporary directory: + + .. code-block:: text + + {basetemp}/{testname}/ + + Note that there is no retention feature in this case: + only the results of the most recent run will be kept. + + .. warning:: + + The directory given to :option:`--basetemp` will be cleared blindly before each test run, + so make sure to use a directory for that purpose only. When distributing tests on the local machine using ``pytest-xdist``, care is taken to automatically configure a `basetemp` directory for the sub processes such that all temporary diff --git a/doc/en/how-to/unittest.rst b/doc/en/how-to/unittest.rst index 508aebde016..0762e7d4cf8 100644 --- a/doc/en/how-to/unittest.rst +++ b/doc/en/how-to/unittest.rst @@ -22,17 +22,14 @@ their ``test`` methods in ``test_*.py`` or ``*_test.py`` files. Almost all ``unittest`` features are supported: -* ``@unittest.skip`` style decorators; -* ``setUp/tearDown``; -* ``setUpClass/tearDownClass``; -* ``setUpModule/tearDownModule``; +* :func:`unittest.skip`/:func:`unittest.skipIf` style decorators +* :meth:`unittest.TestCase.setUp`/:meth:`unittest.TestCase.tearDown` +* :meth:`unittest.TestCase.setUpClass`/:meth:`unittest.TestCase.tearDownClass` +* :func:`unittest.setUpModule`/:func:`unittest.tearDownModule` +* :meth:`unittest.TestCase.subTest` (since version ``9.0``) -.. _`pytest-subtests`: https://github.com/pytest-dev/pytest-subtests .. _`load_tests protocol`: https://docs.python.org/3/library/unittest.html#load-tests-protocol -Additionally, :ref:`subtests ` are supported by the -`pytest-subtests`_ plugin. - Up to this point pytest does not have support for the following features: * `load_tests protocol`_; @@ -45,7 +42,7 @@ in most cases without having to modify existing code: * Obtain :ref:`more informative tracebacks `; * :ref:`stdout and stderr ` capturing; -* :ref:`Test selection options ` using ``-k`` and ``-m`` flags; +* :ref:`Test selection options ` using :option:`-k` and :option:`-m` flags; * :ref:`maxfail`; * :ref:`--pdb ` command-line option for debugging on test failures (see :ref:`note ` below); @@ -109,7 +106,7 @@ achieves this by receiving a special ``request`` object which gives access to :ref:`the requesting test context ` such as the ``cls`` attribute, denoting the class from which the fixture is used. This architecture de-couples fixture writing from actual test -code and allows re-use of the fixture by a minimal reference, the fixture +code and allows reuse of the fixture by a minimal reference, the fixture name. So let's write an actual ``unittest.TestCase`` class using our fixture definition: @@ -140,7 +137,7 @@ the ``self.db`` values in the traceback: $ pytest test_unittest_db.py =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 2 items @@ -154,6 +151,7 @@ the ``self.db`` values in the traceback: def test_method1(self): assert hasattr(self, "db") > assert 0, self.db # fail for demo purposes + ^^^^^^^^^^^^^^^^^ E AssertionError: .DummyDB object at 0xdeadbeef0001> E assert 0 @@ -164,6 +162,7 @@ the ``self.db`` values in the traceback: def test_method2(self): > assert 0, self.db # fail for demo purposes + ^^^^^^^^^^^^^^^^^ E AssertionError: .DummyDB object at 0xdeadbeef0001> E assert 0 diff --git a/doc/en/how-to/usage.rst b/doc/en/how-to/usage.rst index 0e0a0310fd8..35b07bfe8c1 100644 --- a/doc/en/how-to/usage.rst +++ b/doc/en/how-to/usage.rst @@ -4,10 +4,10 @@ How to invoke pytest ========================================== -.. seealso:: :ref:`Complete pytest command-line flag reference ` +.. seealso:: :ref:`Complete pytest command-line flags reference ` In general, pytest is invoked with the command ``pytest`` (see below for :ref:`other ways to invoke pytest -`). This will execute all tests in all files whose names follow the form ``test_*.py`` or ``\*_test.py`` +`). This will execute all tests in all files whose names follow the form ``test_*.py`` or ``*_test.py`` in the current directory and its subdirectories. More generally, pytest follows :ref:`standard test discovery rules `. @@ -155,7 +155,7 @@ Managing loading of plugins Early loading plugins ~~~~~~~~~~~~~~~~~~~~~~~ -You can early-load plugins (internal and external) explicitly in the command-line with the ``-p`` option:: +You can early-load plugins (internal and external) explicitly in the command-line with the :option:`-p` option:: pytest -p mypluginmodule @@ -171,7 +171,7 @@ The option receives a ``name`` parameter, which can be: Disabling plugins ~~~~~~~~~~~~~~~~~~ -To disable loading specific plugins at invocation time, use the ``-p`` option +To disable loading specific plugins at invocation time, use the :option:`-p` option together with the prefix ``no:``. Example: to disable loading the plugin ``doctest``, which is responsible for diff --git a/doc/en/how-to/writing_hook_functions.rst b/doc/en/how-to/writing_hook_functions.rst index f4c00d04fda..d5d6d2ae4f7 100644 --- a/doc/en/how-to/writing_hook_functions.rst +++ b/doc/en/how-to/writing_hook_functions.rst @@ -94,7 +94,7 @@ around the actual hook implementations, in which case it can return the result value of the ``yield``. The simplest (though useless) hook wrapper is ``return (yield)``. -In other cases, the wrapper wants the adjust or adapt the result, in which case +In other cases, the wrapper wants to adjust or adapt the result, in which case it can return a new value. If the result of the underlying hook is a mutable object, the wrapper may modify that result, but it's probably better to avoid it. @@ -235,6 +235,12 @@ Example: """ print(config.hook) +.. note:: + + Unlike other hooks, the :hook:`pytest_generate_tests` hook is also discovered when + defined inside a test module or test class. Other hooks must live in + :ref:`conftest.py plugins ` or external plugins. + See :ref:`parametrize-basics` and the :ref:`hook-reference`. .. _`addoptionhooks`: diff --git a/doc/en/how-to/writing_plugins.rst b/doc/en/how-to/writing_plugins.rst index 1bba9644649..56043a14f97 100644 --- a/doc/en/how-to/writing_plugins.rst +++ b/doc/en/how-to/writing_plugins.rst @@ -48,7 +48,7 @@ Plugin discovery order at tool startup 5. by loading all plugins specified through the :envvar:`PYTEST_PLUGINS` environment variable. -6. by loading all "initial ":file:`conftest.py` files: +6. by loading all "initial" :file:`conftest.py` files: - determine the test paths: specified on the command line, otherwise in :confval:`testpaths` if defined and running from the rootdir, otherwise the @@ -295,7 +295,7 @@ the plugin manager like this: plugin = config.pluginmanager.get_plugin("name_of_plugin") If you want to look at the names of existing plugins, use -the ``--trace-config`` option. +the :option:`--trace-config` option. .. _registering-markers: @@ -420,13 +420,13 @@ before running pytest on it. This way we can abstract the tested logic to separa which is especially useful for longer tests and/or longer ``conftest.py`` files. Note that for ``pytester.copy_example`` to work we need to set `pytester_example_dir` -in our ``pytest.ini`` to tell pytest where to look for example files. +in our configuration file to tell pytest where to look for example files. -.. code-block:: ini +.. code-block:: toml - # content of pytest.ini - [pytest] - pytester_example_dir = . + # content of pytest.toml + [pytest] + pytester_example_dir = "." .. code-block:: python @@ -446,9 +446,9 @@ in our ``pytest.ini`` to tell pytest where to look for example files. $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project - configfile: pytest.ini + configfile: pytest.toml collected 2 items test_example.py .. [100%] diff --git a/doc/en/index.rst b/doc/en/index.rst index 95044e8a544..1140640c80a 100644 --- a/doc/en/index.rst +++ b/doc/en/index.rst @@ -2,8 +2,7 @@ .. sidebar:: **Next Open Trainings and Events** - - `pytest: Professionelles Testen (nicht nur) für Python `_, at `CH Open Workshoptage `_, **September 2nd 2024**, HSLU Rotkreuz (CH) - - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training), **March 4th -- 6th 2025**, Leipzig (DE) / Remote + - `Professional Testing with Python `_, via `Python Academy `_ (3 day in-depth training), **March 9th -- 11th 2027**, Leipzig (DE) / Remote Also see :doc:`previous talks and blogposts ` @@ -46,8 +45,6 @@ The ``pytest`` framework makes it easy to write small, readable tests, and can scale to support complex functional testing for applications and libraries. -``pytest`` requires: Python 3.8+ or PyPy3. - **PyPI package name**: :pypi:`pytest` A quick example @@ -70,7 +67,7 @@ To execute it: $ pytest =========================== test session starts ============================ - platform linux -- Python 3.x.y, pytest-8.x.y, pluggy-1.x.y + platform linux -- Python 3.x.y, pytest-9.x.y, pluggy-1.x.y rootdir: /home/sweet/project collected 1 item @@ -104,7 +101,7 @@ Features - Can run :ref:`unittest ` (including trial) test suites out of the box -- Python 3.8+ or PyPy 3 +- Python 3.10+ or PyPy 3 - Rich plugin architecture, with over 1300+ :ref:`external plugins ` and thriving community diff --git a/doc/en/reference/customize.rst b/doc/en/reference/customize.rst index 373223ec913..8903ceadf68 100644 --- a/doc/en/reference/customize.rst +++ b/doc/en/reference/customize.rst @@ -4,8 +4,7 @@ Configuration Command line options and configuration file settings ----------------------------------------------------------------- -You can get help on command line options and values in INI-style -configurations files by using the general help option: +You can get help on command line and configuration options by using the general help option: .. code-block:: bash @@ -24,51 +23,89 @@ by convention resides in the root directory of your repository. A quick example of the configuration files supported by pytest: +pytest.toml +~~~~~~~~~~~ + +.. versionadded:: 9.0 + +``pytest.toml`` files take precedence over other files, even when empty. + +Alternatively, the hidden version ``.pytest.toml`` can be used. + +.. tab:: toml + + .. code-block:: toml + + # pytest.toml or .pytest.toml + [pytest] + minversion = "9.0" + addopts = ["-ra", "-q"] + testpaths = [ + "tests", + "integration", + ] + pytest.ini ~~~~~~~~~~ -``pytest.ini`` files take precedence over other files, even when empty. +``pytest.ini`` files take precedence over other files (except ``pytest.toml`` and ``.pytest.toml``), even when empty. Alternatively, the hidden version ``.pytest.ini`` can be used. -.. code-block:: ini +.. tab:: ini - # pytest.ini or .pytest.ini - [pytest] - minversion = 6.0 - addopts = -ra -q - testpaths = - tests - integration + .. code-block:: ini + + # pytest.ini or .pytest.ini + [pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration pyproject.toml ~~~~~~~~~~~~~~ .. versionadded:: 6.0 +.. versionchanged:: 9.0 + +``pyproject.toml`` files are supported for configuration. + +.. tab:: toml + + Use ``[tool.pytest]`` to leverage native TOML types (supported since pytest 9.0): -``pyproject.toml`` are considered for configuration when they contain a ``tool.pytest.ini_options`` table. + .. code-block:: toml -.. code-block:: toml + # pyproject.toml + [tool.pytest] + minversion = "9.0" + addopts = ["-ra", "-q"] + testpaths = [ + "tests", + "integration", + ] - # pyproject.toml - [tool.pytest.ini_options] - minversion = "6.0" - addopts = "-ra -q" - testpaths = [ - "tests", - "integration", - ] +.. tab:: ini -.. note:: + Use ``[tool.pytest.ini_options]`` for INI-style configuration (supported since pytest 6.0): - One might wonder why ``[tool.pytest.ini_options]`` instead of ``[tool.pytest]`` as is the - case with other tools. + .. code-block:: toml - The reason is that the pytest team intends to fully utilize the rich TOML data format - for configuration in the future, reserving the ``[tool.pytest]`` table for that. - The ``ini_options`` table is being used, for now, as a bridge between the existing - ``.ini`` configuration system and the future configuration format. + # pyproject.toml + [tool.pytest.ini_options] + minversion = "6.0" + addopts = "-ra -q" + testpaths = [ + "tests", + "integration", + ] + + For projects that still run pytest versions older than 6.0, keep + ``minversion`` in ``pytest.ini`` or ``setup.cfg`` too. Those versions + do not read ``pyproject.toml``. tox.ini ~~~~~~~ @@ -76,15 +113,17 @@ tox.ini ``tox.ini`` files are the configuration files of the `tox `__ project, and can also be used to hold pytest configuration if they have a ``[pytest]`` section. -.. code-block:: ini +.. tab:: ini + + .. code-block:: ini - # tox.ini - [pytest] - minversion = 6.0 - addopts = -ra -q - testpaths = - tests - integration + # tox.ini + [pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration setup.cfg @@ -93,15 +132,17 @@ setup.cfg ``setup.cfg`` files are general purpose configuration files, used originally by ``distutils`` (now deprecated) and :std:doc:`setuptools `, and can also be used to hold pytest configuration if they have a ``[tool:pytest]`` section. -.. code-block:: ini +.. tab:: ini + + .. code-block:: ini - # setup.cfg - [tool:pytest] - minversion = 6.0 - addopts = -ra -q - testpaths = - tests - integration + # setup.cfg + [tool:pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration .. warning:: @@ -123,7 +164,7 @@ the command line arguments (specified test files, paths) and on the existence of configuration files. The determined ``rootdir`` and ``configfile`` are printed as part of the pytest header during startup. -Here's a summary what ``pytest`` uses ``rootdir`` for: +Here's a summary of what ``pytest`` uses ``rootdir`` for: * Construct *nodeids* during collection; each test is assigned a unique *nodeid* which is rooted at the ``rootdir`` and takes into account @@ -136,9 +177,9 @@ Here's a summary what ``pytest`` uses ``rootdir`` for: ``rootdir`` is **NOT** used to modify ``sys.path``/``PYTHONPATH`` or influence how modules are imported. See :ref:`pythonpath` for more details. -The ``--rootdir=path`` command-line option can be used to force a specific directory. +The :option:`--rootdir=path` command-line option can be used to force a specific directory. Note that contrary to other command-line options, ``--rootdir`` cannot be used with -:confval:`addopts` inside ``pytest.ini`` because the ``rootdir`` is used to *find* ``pytest.ini`` +:confval:`addopts` inside a configuration file because the ``rootdir`` is used to *find* the configuration file already. Finding the ``rootdir`` @@ -146,20 +187,20 @@ Finding the ``rootdir`` Here is the algorithm which finds the rootdir from ``args``: -- If ``-c`` is passed in the command-line, use that as configuration file, and its directory as ``rootdir``. +- If :option:`-c` is passed in the command-line, use that as configuration file, and its directory as ``rootdir``. - Determine the common ancestor directory for the specified ``args`` that are recognised as paths that exist in the file system. If no such paths are found, the common ancestor directory is set to the current working directory. -- Look for ``pytest.ini``, ``pyproject.toml``, ``tox.ini``, and ``setup.cfg`` files in the ancestor +- Look for ``pytest.toml``, ``.pytest.toml``, ``pytest.ini``, ``.pytest.ini``, ``pyproject.toml``, ``tox.ini``, and ``setup.cfg`` files in the ancestor directory and upwards. If one is matched, it becomes the ``configfile`` and its directory becomes the ``rootdir``. - If no configuration file was found, look for ``setup.py`` upwards from the common ancestor directory to determine the ``rootdir``. -- If no ``setup.py`` was found, look for ``pytest.ini``, ``pyproject.toml``, ``tox.ini``, and +- If no ``setup.py`` was found, look for ``pytest.toml``, ``.pytest.toml``, ``pytest.ini``, ``.pytest.ini``, ``pyproject.toml``, ``tox.ini``, and ``setup.cfg`` in each of the specified ``args`` and upwards. If one is matched, it becomes the ``configfile`` and its directory becomes the ``rootdir``. @@ -167,18 +208,20 @@ Here is the algorithm which finds the rootdir from ``args``: directory. This allows the use of pytest in structures that are not part of a package and don't have any particular configuration file. -If no ``args`` are given, pytest collects test below the current working +If no ``args`` are given, pytest collects tests below the current working directory and also starts determining the ``rootdir`` from there. Files will only be matched for configuration if: -* ``pytest.ini``: will always match and take precedence, even if empty. -* ``pyproject.toml``: contains a ``[tool.pytest.ini_options]`` table. +* ``pytest.toml``: will always match and take highest precedence, even if empty. +* ``pytest.ini``: will always match and take precedence (after ``pytest.toml`` and ``.pytest.toml``), even if empty. +* ``pyproject.toml``: contains a ``[tool.pytest]`` or ``[tool.pytest.ini_options]`` table. * ``tox.ini``: contains a ``[pytest]`` section. * ``setup.cfg``: contains a ``[tool:pytest]`` section. Finally, a ``pyproject.toml`` file will be considered the ``configfile`` if no other match was found, in this case -even if it does not contain a ``[tool.pytest.ini_options]`` table (this was added in ``8.1``). +even if it does not contain a ``[tool.pytest]`` table (since version ``9.0``) or a ``[tool.pytest.ini_options]`` +table (since version ``8.1``). The files are considered in the order above. Options from multiple ``configfiles`` candidates are never merged - the first match wins. @@ -213,11 +256,13 @@ check for configuration files as follows: .. code-block:: text - # first look for pytest.ini files + # first look for path/pytest.toml + path/pytest.toml path/pytest.ini - path/pyproject.toml # must contain a [tool.pytest.ini_options] table to match + path/pyproject.toml # must contain a [tool.pytest] table to match path/tox.ini # must contain [pytest] section to match path/setup.cfg # must contain [tool:pytest] section to match + pytest.toml pytest.ini ... # all the way up to the root @@ -233,7 +278,7 @@ check for configuration files as follows: ``pytest --log-output ../../test.log args``. Then ``args`` is mandatory, otherwise pytest uses the folder of test.log for rootdir determination (see also :issue:`1435`). - A dot ``.`` for referencing to the current working directory is also + A dot ``.`` for referencing the current working directory is also possible. diff --git a/doc/en/reference/exit-codes.rst b/doc/en/reference/exit-codes.rst index b695ca3702e..49aaca19121 100644 --- a/doc/en/reference/exit-codes.rst +++ b/doc/en/reference/exit-codes.rst @@ -20,7 +20,7 @@ They are represented by the :class:`pytest.ExitCode` enum. The exit codes being .. note:: - If you would like to customize the exit code in some scenarios, specially when + If you would like to customize the exit code in some scenarios, specifically when no tests are collected, consider using the `pytest-custom_exit_code `__ plugin. diff --git a/doc/en/reference/fixtures.rst b/doc/en/reference/fixtures.rst index dff93a035ef..c4a8d01ff0e 100644 --- a/doc/en/reference/fixtures.rst +++ b/doc/en/reference/fixtures.rst @@ -32,6 +32,10 @@ Built-in fixtures :fixture:`capsys` Capture, as text, output to ``sys.stdout`` and ``sys.stderr``. + :fixture:`capteesys` + Capture in the same manner as :fixture:`capsys`, but also pass text + through according to :option:`--capture`. + :fixture:`capsysbinary` Capture, as bytes, output to ``sys.stdout`` and ``sys.stderr``. @@ -48,6 +52,9 @@ Built-in fixtures :fixture:`pytestconfig` Access to configuration values, pluginmanager and plugin hooks. + :fixture:`subtests` + Enable declaring subtests inside test functions. + :fixture:`record_property` Add extra properties to the test. @@ -270,13 +277,13 @@ the test's search for fixtures would look like: pytest will only search for ``a_fix`` and ``b_fix`` in the plugins after searching for them first in the scopes inside ``tests/``. -.. note: +.. note:: pytest can tell you what fixtures are available for a given test if you call - ``pytests`` along with the test's name (or the scope it's in), and provide - the ``--fixtures`` flag, e.g. ``pytest --fixtures test_something.py`` + ``pytest`` along with the test's name (or the scope it's in), and provide + the :option:`--fixtures` flag, e.g. ``pytest --fixtures test_something.py`` (fixtures with names that start with ``_`` will only be shown if you also - provide the ``-v`` flag). + provide the :option:`-v` flag). .. _`fixture order`: @@ -347,7 +354,7 @@ an order of operations for a given test. If there's any ambiguity, and the order of operations can be interpreted more than one way, you should assume pytest could go with any one of those interpretations at any point. -For example, if ``d`` didn't request ``c``, i.e.the graph would look like this: +For example, if ``d`` didn't request ``c``, i.e. the graph would look like this: .. image:: /example/fixtures/test_fixtures_order_dependencies_unclear.* :align: center @@ -441,10 +448,10 @@ for the tests inside ``TestClassWithoutAutouse``, since they can reference can't see ``c3``. -.. note: +.. note:: pytest can tell you what order the fixtures will execute in for a given test - if you call ``pytests`` along with the test's name (or the scope it's in), - and provide the ``--setup-plan`` flag, e.g. + if you call ``pytest`` along with the test's name (or the scope it's in), + and provide the :option:`--setup-plan` flag, e.g. ``pytest --setup-plan test_something.py`` (fixtures with names that start - with ``_`` will only be shown if you also provide the ``-v`` flag). + with ``_`` will only be shown if you also provide the :option:`-v` flag). diff --git a/doc/en/reference/plugin_list.rst b/doc/en/reference/plugin_list.rst index 7526b055943..c13600133f6 100644 --- a/doc/en/reference/plugin_list.rst +++ b/doc/en/reference/plugin_list.rst @@ -27,21 +27,22 @@ please refer to `the update script =7; extra == "pytest" - :pypi:`nuts` Network Unit Testing System May 28, 2024 N/A pytest<8,>=7 + :pypi:`databricks-labs-pytester` Python Testing for Databricks Oct 17, 2025 4 - Beta pytest>=8.3 + :pypi:`logassert` Simple but powerful assertion and verification of logged lines Aug 14, 2025 5 - Production/Stable pytest; extra == "dev" + :pypi:`logot` Test whether your code is logging correctly 🪵 Jul 28, 2025 5 - Production/Stable pytest; extra == "pytest" + :pypi:`nuts` Network Unit Testing System May 10, 2025 N/A pytest<8,>=7 :pypi:`pytest-abq` Pytest integration for the ABQ universal test runner. Apr 07, 2023 N/A N/A :pypi:`pytest-abstracts` A contextmanager pytest fixture for handling multiple mock abstracts May 25, 2022 N/A N/A - :pypi:`pytest-accept` A pytest-plugin for updating doctest outputs Feb 10, 2024 N/A pytest (>=6) + :pypi:`pytest-accept` Aug 19, 2025 N/A pytest>=7 :pypi:`pytest-adaptavist` pytest plugin for generating test execution results within Jira Test Management (tm4j) Oct 13, 2022 N/A pytest (>=5.4.0) - :pypi:`pytest-adaptavist-fixed` pytest plugin for generating test execution results within Jira Test Management (tm4j) Nov 08, 2023 N/A pytest >=5.4.0 + :pypi:`pytest-adaptavist-fixed` pytest plugin for generating test execution results within Jira Test Management (tm4j) Jan 17, 2025 N/A pytest>=5.4.0 :pypi:`pytest-addons-test` 用于测试pytest的插件 Aug 02, 2021 N/A pytest (>=6.2.4,<7.0.0) :pypi:`pytest-adf` Pytest plugin for writing Azure Data Factory integration tests May 10, 2021 4 - Beta pytest (>=3.5.0) :pypi:`pytest-adf-azure-identity` Pytest plugin for writing Azure Data Factory integration tests Mar 06, 2021 4 - Beta pytest (>=3.5.0) @@ -49,59 +50,75 @@ This list contains 1487 plugins. :pypi:`pytest-affected` Nov 06, 2023 N/A N/A :pypi:`pytest-agent` Service that exposes a REST API that can be used to interract remotely with Pytest. It is shipped with a dashboard that enables running tests in a more convenient way. Nov 25, 2021 N/A N/A :pypi:`pytest-aggreport` pytest plugin for pytest-repeat that generate aggregate report of the same test cases with additional statistics details. Mar 07, 2021 4 - Beta pytest (>=6.2.2) + :pypi:`pytest-ai` A Python package to generate regular, edge-case, and security HTTP tests. Jan 22, 2025 N/A N/A :pypi:`pytest-ai1899` pytest plugin for connecting to ai1899 smart system stack Mar 13, 2024 5 - Production/Stable N/A - :pypi:`pytest-aio` Pytest plugin for testing async python code Apr 08, 2024 5 - Production/Stable pytest + :pypi:`pytest-aio` Pytest plugin for testing async python code Jul 31, 2024 5 - Production/Stable pytest + :pypi:`pytest-aioboto3` Aioboto3 Pytest with Moto Jan 17, 2025 N/A N/A :pypi:`pytest-aiofiles` pytest fixtures for writing aiofiles tests with pyfakefs May 14, 2017 5 - Production/Stable N/A :pypi:`pytest-aiogram` May 06, 2023 N/A N/A - :pypi:`pytest-aiohttp` Pytest plugin for aiohttp support Sep 06, 2023 4 - Beta pytest >=6.1.0 + :pypi:`pytest-aiohttp` Pytest plugin for aiohttp support Jan 23, 2025 4 - Beta pytest>=6.1.0 :pypi:`pytest-aiohttp-client` Pytest \`client\` fixture for the Aiohttp Jan 10, 2023 N/A pytest (>=7.2.0,<8.0.0) + :pypi:`pytest-aiohttp-mock` Send responses to aiohttp. Sep 13, 2025 3 - Alpha pytest>=8 :pypi:`pytest-aiomoto` pytest-aiomoto Jun 24, 2023 N/A pytest (>=7.0,<8.0) - :pypi:`pytest-aioresponses` py.test integration for aioresponses Jul 29, 2021 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-aioworkers` A plugin to test aioworkers project with pytest May 01, 2023 5 - Production/Stable pytest>=6.1.0 + :pypi:`pytest-aioresponses` py.test integration for aioresponses Jan 02, 2025 4 - Beta pytest>=3.5.0 + :pypi:`pytest-aioworkers` A plugin to test aioworkers project with pytest Dec 26, 2024 5 - Production/Stable pytest>=8.3.4 :pypi:`pytest-airflow` pytest support for airflow. Apr 03, 2019 3 - Alpha pytest (>=4.4.0) :pypi:`pytest-airflow-utils` Nov 15, 2021 N/A N/A - :pypi:`pytest-alembic` A pytest plugin for verifying alembic migrations. Mar 04, 2024 N/A pytest (>=6.0) + :pypi:`pytest-alembic` A pytest plugin for verifying alembic migrations. May 27, 2025 N/A pytest>=7.0 + :pypi:`pytest-alerts` A pytest plugin for sending test results to Slack and Telegram Feb 21, 2025 4 - Beta pytest>=7.4.0 :pypi:`pytest-allclose` Pytest fixture extending Numpy's allclose function Jul 30, 2019 5 - Production/Stable pytest :pypi:`pytest-allure-adaptor` Plugin for py.test to generate allure xml reports Jan 10, 2018 N/A pytest (>=2.7.3) :pypi:`pytest-allure-adaptor2` Plugin for py.test to generate allure xml reports Oct 14, 2020 N/A pytest (>=2.7.3) :pypi:`pytest-allure-collection` pytest plugin to collect allure markers without running any tests Apr 13, 2023 N/A pytest :pypi:`pytest-allure-dsl` pytest plugin to test case doc string dls instructions Oct 25, 2020 4 - Beta pytest + :pypi:`pytest-allure-host` Publish Allure static reports to private S3 behind CloudFront with history preservation Oct 21, 2025 3 - Alpha N/A :pypi:`pytest-allure-id2history` Overwrite allure history id with testcase full name and testcase id if testcase has id, exclude parameters. May 14, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-allure-intersection` Oct 27, 2022 N/A pytest (<5) :pypi:`pytest-allure-spec-coverage` The pytest plugin aimed to display test coverage of the specs(requirements) in Allure Oct 26, 2021 N/A pytest + :pypi:`pytest-allure-step` Enhanced logging integration with Allure reports for pytest Jul 13, 2025 3 - Alpha pytest>=6.0.0 :pypi:`pytest-alphamoon` Static code checks used at Alphamoon Dec 30, 2021 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-amaranth-sim` Fixture to automate running Amaranth simulations Sep 21, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-analyzer` this plugin allows to analyze tests in pytest project, collect test metadata and sync it with testomat.io TCM system Feb 21, 2024 N/A pytest <8.0.0,>=7.3.1 :pypi:`pytest-android` This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. Feb 21, 2019 3 - Alpha pytest :pypi:`pytest-anki` A pytest plugin for testing Anki add-ons Jul 31, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-annotate` pytest-annotate: Generate PyAnnotate annotations from your pytest tests. Jun 07, 2022 3 - Alpha pytest (<8.0.0,>=3.2.0) - :pypi:`pytest-ansible` Plugin for pytest to simplify calling ansible modules from tests or fixtures Jul 10, 2024 5 - Production/Stable pytest>=6 + :pypi:`pytest-annotated` Pytest plugin to allow use of Annotated in tests to resolve fixtures Sep 30, 2024 N/A pytest>=8.3.3 + :pypi:`pytest-ansible` Plugin for pytest to simplify calling ansible modules from tests or fixtures Aug 21, 2025 5 - Production/Stable pytest>=6 :pypi:`pytest-ansible-playbook` Pytest fixture which runs given ansible playbook file. Mar 08, 2019 4 - Beta N/A :pypi:`pytest-ansible-playbook-runner` Pytest fixture which runs given ansible playbook file. Dec 02, 2020 4 - Beta pytest (>=3.1.0) :pypi:`pytest-ansible-units` A pytest plugin for running unit tests within an ansible collection Apr 14, 2022 N/A N/A - :pypi:`pytest-antilru` Bust functools.lru_cache when running pytest to avoid test pollution Jul 05, 2022 5 - Production/Stable pytest + :pypi:`pytest-antilru` Bust functools.lru_cache when running pytest to avoid test pollution Jul 28, 2024 5 - Production/Stable pytest>=7; python_version >= "3.10" :pypi:`pytest-anyio` The pytest anyio plugin is built into anyio. You don't need this package. Jun 29, 2021 N/A pytest :pypi:`pytest-anything` Pytest fixtures to assert anything and something Jan 18, 2024 N/A pytest :pypi:`pytest-aoc` Downloads puzzle inputs for Advent of Code and synthesizes PyTest fixtures Dec 02, 2023 5 - Production/Stable pytest ; extra == 'test' :pypi:`pytest-aoreporter` pytest report Jun 27, 2022 N/A N/A :pypi:`pytest-api` An ASGI middleware to populate OpenAPI Specification examples from pytest functions May 12, 2022 N/A pytest (>=7.1.1,<8.0.0) + :pypi:`pytest-api-cov` Pytest Plugin to provide API Coverage statistics for Python Web Frameworks Oct 28, 2025 N/A pytest>=6.0.0 + :pypi:`pytest-api-framework` pytest framework Jun 22, 2025 N/A pytest==7.2.2 + :pypi:`pytest-api-framework-alpha` Oct 29, 2025 N/A pytest==7.2.2 :pypi:`pytest-api-soup` Validate multiple endpoints with unit testing using a single source of truth. Aug 27, 2022 N/A N/A :pypi:`pytest-apistellar` apistellar plugin for pytest. Jun 18, 2019 N/A N/A :pypi:`pytest-apiver` Jun 21, 2024 N/A pytest :pypi:`pytest-appengine` AppEngine integration that works well with pytest-django Feb 27, 2017 N/A N/A :pypi:`pytest-appium` Pytest plugin for appium Dec 05, 2019 N/A N/A + :pypi:`pytest-approval` A simple approval test library utilizing external diff programs such as PyCharm and Visual Studio Code to compare approved and received output. Oct 27, 2025 N/A pytest>=8.3.5 :pypi:`pytest-approvaltests` A plugin to use approvaltests with pytest May 08, 2022 4 - Beta pytest (>=7.0.1) - :pypi:`pytest-approvaltests-geo` Extension for ApprovalTests.Python specific to geo data verification Feb 05, 2024 5 - Production/Stable pytest - :pypi:`pytest-archon` Rule your architecture like a real developer Dec 18, 2023 5 - Production/Stable pytest >=7.2 + :pypi:`pytest-approvaltests-geo` Extension for ApprovalTests.Python specific to geo data verification Jul 14, 2025 5 - Production/Stable pytest + :pypi:`pytest-archon` Rule your architecture like a real developer Sep 19, 2025 5 - Production/Stable pytest>=7.2 :pypi:`pytest-argus` pyest results colection plugin Jun 24, 2021 5 - Production/Stable pytest (>=6.2.4) + :pypi:`pytest-argus-reporter` A simple plugin to report results of test into argus Sep 17, 2025 4 - Beta pytest>=3.0; extra == "dev" + :pypi:`pytest-argus-server` A plugin that provides a running Argus API server for tests Mar 24, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-arraydiff` pytest plugin to help with comparing array output from tests Nov 27, 2023 4 - Beta pytest >=4.6 + :pypi:`pytest-asdf-plugin` Pytest plugin for testing ASDF schemas Aug 18, 2025 5 - Production/Stable pytest>=7 :pypi:`pytest-asgi-server` Convenient ASGI client/server fixtures for Pytest Dec 12, 2020 N/A pytest (>=5.4.1) :pypi:`pytest-aspec` A rspec format reporter for pytest Dec 20, 2023 4 - Beta N/A :pypi:`pytest-asptest` test Answer Set Programming programs Apr 28, 2018 4 - Beta N/A :pypi:`pytest-assertcount` Plugin to count actual number of asserts in pytest Oct 23, 2022 N/A pytest (>=5.0.0) :pypi:`pytest-assertions` Pytest Assertions Apr 27, 2022 N/A N/A + :pypi:`pytest-assert-type` Use typing.assert_type() to test runtime behavior Oct 26, 2025 3 - Alpha pytest>=6.2.0 :pypi:`pytest-assertutil` pytest-assertutil May 10, 2019 N/A N/A :pypi:`pytest-assert-utils` Useful assertion utilities for use with pytest Apr 14, 2022 3 - Alpha N/A - :pypi:`pytest-assist` load testing library Jun 24, 2024 N/A pytest + :pypi:`pytest-assist` pytest plugin library Oct 29, 2025 4 - Beta pytest :pypi:`pytest-assume` A pytest plugin that allows multiple failures per test Jun 24, 2021 N/A pytest (>=2.7) :pypi:`pytest-assurka` A pytest plugin for Assurka Studio Aug 04, 2022 N/A N/A :pypi:`pytest-ast-back-to-python` A plugin for pytest devs to view how assertion rewriting recodes the AST Sep 29, 2019 4 - Beta N/A @@ -110,86 +127,98 @@ This list contains 1487 plugins. :pypi:`pytest-astropy-header` pytest plugin to add diagnostic information to the header of the test output Sep 06, 2022 3 - Alpha pytest (>=4.6) :pypi:`pytest-ast-transformer` May 04, 2019 3 - Alpha pytest :pypi:`pytest_async` pytest-async - Run your coroutine in event loop without decorator Feb 26, 2020 N/A N/A + :pypi:`pytest-async-benchmark` pytest-async-benchmark: Modern pytest benchmarking for async code. 🚀 May 28, 2025 N/A pytest>=8.3.5 :pypi:`pytest-async-generators` Pytest fixtures for async generators Jul 05, 2023 N/A N/A - :pypi:`pytest-asyncio` Pytest support for asyncio May 19, 2024 4 - Beta pytest<9,>=7.0.0 - :pypi:`pytest-asyncio-cooperative` Run all your asynchronous tests cooperatively. Jul 04, 2024 N/A N/A + :pypi:`pytest-asyncio` Pytest support for asyncio Sep 12, 2025 5 - Production/Stable pytest<9,>=8.2 + :pypi:`pytest-asyncio-concurrent` Pytest plugin to execute python async tests concurrently. May 17, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-asyncio-cooperative` Run all your asynchronous tests cooperatively. Jun 24, 2025 N/A N/A :pypi:`pytest-asyncio-network-simulator` pytest-asyncio-network-simulator: Plugin for pytest for simulator the network in tests Jul 31, 2018 3 - Alpha pytest (<3.7.0,>=3.3.2) :pypi:`pytest-async-mongodb` pytest plugin for async MongoDB Oct 18, 2017 5 - Production/Stable pytest (>=2.5.2) :pypi:`pytest-async-sqlalchemy` Database testing fixtures using the SQLAlchemy asyncio API Oct 07, 2021 4 - Beta pytest (>=6.0.0) :pypi:`pytest-atf-allure` 基于allure-pytest进行自定义 Nov 29, 2023 N/A pytest (>=7.4.2,<8.0.0) :pypi:`pytest-atomic` Skip rest of tests if previous test failed. Nov 24, 2018 4 - Beta N/A + :pypi:`pytest-atstack` A simple plugin to use with pytest Jan 02, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-attrib` pytest plugin to select tests based on attributes similar to the nose-attrib plugin May 24, 2016 4 - Beta N/A :pypi:`pytest-attributes` A plugin that allows users to add attributes to their tests. These attributes can then be referenced by fixtures or the test itself. Jun 24, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-austin` Austin plugin for pytest Oct 11, 2020 4 - Beta N/A :pypi:`pytest-autocap` automatically capture test & fixture stdout/stderr to files May 15, 2022 N/A pytest (<7.2,>=7.1.2) :pypi:`pytest-autochecklog` automatically check condition and log all the checks Apr 25, 2015 4 - Beta N/A + :pypi:`pytest-autofixture` simplify pytest fixtures Aug 01, 2024 N/A pytest>=8 :pypi:`pytest-automation` pytest plugin for building a test suite, using YAML files to extend pytest parameterize functionality. Apr 24, 2024 N/A pytest>=7.0.0 :pypi:`pytest-automock` Pytest plugin for automatical mocks creation May 16, 2023 N/A pytest ; extra == 'dev' :pypi:`pytest-auto-parametrize` pytest plugin: avoid repeating arguments in parametrize Oct 02, 2016 3 - Alpha N/A + :pypi:`pytest-autoprofile` \`line_profiler.autoprofile\`-ing your \`pytest\` test suite Aug 06, 2025 4 - Beta pytest>=7.0 :pypi:`pytest-autotest` This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. Aug 25, 2021 N/A pytest - :pypi:`pytest-aux` templates/examples and aux for pytest Jul 05, 2024 N/A N/A :pypi:`pytest-aviator` Aviator's Flakybot pytest plugin that automatically reruns flaky tests. Nov 04, 2022 4 - Beta pytest :pypi:`pytest-avoidance` Makes pytest skip tests that don not need rerunning May 23, 2019 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-awaiting-fix` A simple plugin to use with pytest for traceability across Jira and disabled automated tests Aug 09, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-aws` pytest plugin for testing AWS resource configurations Oct 04, 2017 4 - Beta N/A :pypi:`pytest-aws-apigateway` pytest plugin for AWS ApiGateway May 24, 2024 4 - Beta pytest :pypi:`pytest-aws-config` Protect your AWS credentials in unit tests May 28, 2021 N/A N/A - :pypi:`pytest-aws-fixtures` A series of fixtures to use in integration tests involving actual AWS services. Feb 02, 2024 N/A pytest (>=8.0.0,<9.0.0) + :pypi:`pytest-aws-fixtures` A series of fixtures to use in integration tests involving actual AWS services. Apr 06, 2025 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-axe` pytest plugin for axe-selenium-python Nov 12, 2018 N/A pytest (>=3.0.0) :pypi:`pytest-axe-playwright-snapshot` A pytest plugin that runs Axe-core on Playwright pages and takes snapshots of the results. Jul 25, 2023 N/A pytest :pypi:`pytest-azure` Pytest utilities and mocks for Azure Jan 18, 2023 3 - Alpha pytest - :pypi:`pytest-azure-devops` Simplifies using azure devops parallel strategy (https://docs.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner) with pytest. Jun 20, 2022 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-azure-devops` Simplifies using azure devops parallel strategy (https://docs.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner) with pytest. Jul 16, 2025 4 - Beta pytest>=3.5.0 :pypi:`pytest-azurepipelines` Formatting PyTest output for Azure Pipelines UI Oct 06, 2023 5 - Production/Stable pytest (>=5.0.0) :pypi:`pytest-bandit` A bandit plugin for pytest Feb 23, 2021 4 - Beta pytest (>=3.5.0) :pypi:`pytest-bandit-xayon` A bandit plugin for pytest Oct 17, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-base-url` pytest plugin for URL based testing Jan 31, 2024 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-bashdoctest` A pytest plugin for testing bash command examples in markdown documentation Oct 03, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-batch-regression` A pytest plugin to repeat the entire test suite in batches. May 08, 2024 N/A pytest>=6.0.0 - :pypi:`pytest-bazel` A pytest runner with bazel support Jul 12, 2024 4 - Beta pytest - :pypi:`pytest-bdd` BDD for pytest Jun 04, 2024 6 - Mature pytest>=6.2.0 + :pypi:`pytest-bazel` A pytest runner with bazel support Oct 31, 2025 4 - Beta pytest + :pypi:`pytest-bdd` BDD for pytest Dec 05, 2024 6 - Mature pytest>=7.0.0 :pypi:`pytest-bdd-html` pytest plugin to display BDD info in HTML test report Nov 22, 2022 3 - Alpha pytest (!=6.0.0,>=5.0) - :pypi:`pytest-bdd-ng` BDD for pytest Dec 31, 2023 4 - Beta pytest >=5.0 - :pypi:`pytest-bdd-report` A pytest-bdd plugin for generating useful and informative BDD test reports May 20, 2024 N/A pytest >=7.1.3 + :pypi:`pytest-bdd-ng` BDD for pytest Nov 26, 2024 4 - Beta pytest>=5.2 + :pypi:`pytest-bdd-report` A pytest-bdd plugin for generating useful and informative BDD test reports Aug 19, 2025 N/A pytest>=7.1.3 + :pypi:`pytest-bdd-reporter` Enterprise-grade BDD test reporting with interactive dashboards, suite management, and comprehensive email integration Oct 14, 2025 5 - Production/Stable pytest>=6.0.0 :pypi:`pytest-bdd-splinter` Common steps for pytest bdd and splinter integration Aug 12, 2019 5 - Production/Stable pytest (>=4.0.0) :pypi:`pytest-bdd-web` A simple plugin to use with pytest Jan 02, 2020 4 - Beta pytest (>=3.5.0) :pypi:`pytest-bdd-wrappers` Feb 11, 2020 2 - Pre-Alpha N/A :pypi:`pytest-beakerlib` A pytest plugin that reports test results to the BeakerLib framework Mar 17, 2017 5 - Production/Stable pytest - :pypi:`pytest-beartype` Pytest plugin to run your tests with beartype checking enabled. Jan 25, 2024 N/A pytest - :pypi:`pytest-bec-e2e` BEC pytest plugin for end-to-end tests Jul 08, 2024 3 - Alpha pytest + :pypi:`pytest-beartype` Pytest plugin to run your tests with beartype checking enabled. Oct 31, 2024 N/A pytest + :pypi:`pytest-bec-e2e` BEC pytest plugin for end-to-end tests Oct 31, 2025 3 - Alpha pytest :pypi:`pytest-beds` Fixtures for testing Google Appengine (GAE) apps Jun 07, 2016 4 - Beta N/A :pypi:`pytest-beeprint` use icdiff for better error messages in pytest assertions Jul 04, 2023 4 - Beta N/A :pypi:`pytest-bench` Benchmark utility that plugs into pytest. Jul 21, 2014 3 - Alpha N/A - :pypi:`pytest-benchmark` A \`\`pytest\`\` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. Oct 25, 2022 5 - Production/Stable pytest (>=3.8) + :pypi:`pytest-benchmark` A \`\`pytest\`\` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. Oct 30, 2025 5 - Production/Stable pytest>=8.1 :pypi:`pytest-better-datadir` A small example package Mar 13, 2023 N/A N/A :pypi:`pytest-better-parametrize` Better description of parametrized test cases Mar 05, 2024 4 - Beta pytest >=6.2.0 :pypi:`pytest-bg-process` Pytest plugin to initialize background process Jan 24, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-bigchaindb` A BigchainDB plugin for pytest. Jan 24, 2022 4 - Beta N/A :pypi:`pytest-bigquery-mock` Provides a mock fixture for python bigquery client Dec 28, 2022 N/A pytest (>=5.0) :pypi:`pytest-bisect-tests` Find tests leaking state and affecting other Jun 09, 2024 N/A N/A - :pypi:`pytest-black` A pytest plugin to enable format checking with black Oct 05, 2020 4 - Beta N/A + :pypi:`pytest-black` A pytest plugin to enable format checking with black Dec 15, 2024 4 - Beta pytest>=7.0.0 :pypi:`pytest-black-multipy` Allow '--black' on older Pythons Jan 14, 2021 5 - Production/Stable pytest (!=3.7.3,>=3.5) ; extra == 'testing' :pypi:`pytest-black-ng` A pytest plugin to enable format checking with black Oct 20, 2022 4 - Beta pytest (>=7.0.0) :pypi:`pytest-blame` A pytest plugin helps developers to debug by providing useful commits history. May 04, 2019 N/A pytest (>=4.4.0) - :pypi:`pytest-blender` Blender Pytest plugin. Aug 10, 2023 N/A pytest ; extra == 'dev' + :pypi:`pytest-blender` Blender Pytest plugin. Jun 25, 2025 N/A pytest :pypi:`pytest-blink1` Pytest plugin to emit notifications via the Blink(1) RGB LED Jan 07, 2018 4 - Beta N/A :pypi:`pytest-blockage` Disable network requests during a test run. Dec 21, 2021 N/A pytest :pypi:`pytest-blocker` pytest plugin to mark a test as blocker and skip all other tests Sep 07, 2015 4 - Beta N/A + :pypi:`pytest-b-logger` BLogger is a Pytest plugin for enhanced test logging and generating convenient and lightweight reports. Oct 28, 2025 N/A pytest :pypi:`pytest-blue` A pytest plugin that adds a \`blue\` fixture for printing stuff in blue. Sep 05, 2022 N/A N/A :pypi:`pytest-board` Local continuous test runner with pytest and watchdog. Jan 20, 2019 N/A N/A + :pypi:`pytest-boardfarm3` Integrate boardfarm as a pytest plugin. Sep 15, 2025 N/A pytest + :pypi:`pytest-boilerplate` The pytest plugin for your Django Boilerplate. Sep 12, 2024 5 - Production/Stable pytest>=4.0.0 + :pypi:`pytest-bonsai` Apr 08, 2025 N/A pytest>=6 :pypi:`pytest-boost-xml` Plugin for pytest to generate boost xml reports Nov 30, 2022 4 - Beta N/A :pypi:`pytest-bootstrap` Mar 04, 2022 N/A N/A - :pypi:`pytest-boto-mock` Thin-wrapper around the mock package for easier use with pytest Jun 05, 2024 5 - Production/Stable pytest>=8.2.0 + :pypi:`pytest-boto-mock` Thin-wrapper around the mock package for easier use with pytest Jul 16, 2024 5 - Production/Stable pytest>=8.2.0 :pypi:`pytest-bpdb` A py.test plug-in to enable drop to bpdb debugger on test failure. Jan 19, 2015 2 - Pre-Alpha N/A :pypi:`pytest-bq` BigQuery fixtures and fixture factories for Pytest. May 08, 2024 5 - Production/Stable pytest>=6.2 :pypi:`pytest-bravado` Pytest-bravado automatically generates from OpenAPI specification client fixtures. Feb 15, 2022 N/A N/A :pypi:`pytest-breakword` Use breakword with pytest Aug 04, 2021 N/A pytest (>=6.2.4,<7.0.0) :pypi:`pytest-breed-adapter` A simple plugin to connect with breed-server Nov 07, 2018 4 - Beta pytest (>=3.5.0) :pypi:`pytest-briefcase` A pytest plugin for running tests on a Briefcase project. Jun 14, 2020 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-broadcaster` Pytest plugin to broadcast pytest output to various destinations Apr 06, 2024 3 - Alpha pytest + :pypi:`pytest-brightest` Bright ideas for improving your pytest experience Jul 15, 2025 3 - Alpha pytest>=8.4.1 + :pypi:`pytest-broadcaster` Pytest plugin to broadcast pytest output to various destinations Mar 02, 2025 3 - Alpha pytest :pypi:`pytest-browser` A pytest plugin for console based browser test selection just after the collection phase Dec 10, 2016 3 - Alpha N/A :pypi:`pytest-browsermob-proxy` BrowserMob proxy plugin for py.test. Jun 11, 2013 4 - Beta N/A :pypi:`pytest_browserstack` Py.test plugin for BrowserStack Jan 27, 2016 4 - Beta N/A :pypi:`pytest-browserstack-local` \`\`py.test\`\` plugin to run \`\`BrowserStackLocal\`\` in background. Feb 09, 2018 N/A N/A :pypi:`pytest-budosystems` Budo Systems is a martial arts school management system. This module is the Budo Systems Pytest Plugin. May 07, 2023 3 - Alpha pytest - :pypi:`pytest-bug` Pytest plugin for marking tests as a bug Jun 05, 2024 5 - Production/Stable pytest>=8.0.0 + :pypi:`pytest-bug` Pytest plugin for marking tests as a bug Jun 17, 2025 5 - Production/Stable pytest>=8.4.0 :pypi:`pytest-bugtong-tag` pytest-bugtong-tag is a plugin for pytest Jan 16, 2022 N/A N/A :pypi:`pytest-bugzilla` py.test bugzilla integration plugin May 05, 2010 4 - Beta N/A :pypi:`pytest-bugzilla-notifier` A plugin that allows you to execute create, update, and read information from BugZilla bugs Jun 15, 2018 4 - Beta pytest (>=2.9.2) @@ -203,14 +232,22 @@ This list contains 1487 plugins. :pypi:`pytest-call-checker` Small pytest utility to easily create test doubles Oct 16, 2022 4 - Beta pytest (>=7.1.3,<8.0.0) :pypi:`pytest-camel-collect` Enable CamelCase-aware pytest class collection Aug 02, 2020 N/A pytest (>=2.9) :pypi:`pytest-canonical-data` A plugin which allows to compare results with canonical results, based on previous runs May 08, 2020 2 - Pre-Alpha pytest (>=3.5.0) + :pypi:`pytest-canvas` A minimal pytest plugin that streamlines testing for projects using the Canvas SDK. Jul 22, 2025 N/A pytest<9,>=8.4 :pypi:`pytest-caprng` A plugin that replays pRNG state on failure. May 02, 2018 4 - Beta N/A + :pypi:`pytest-capsqlalchemy` Pytest plugin to allow capturing SQLAlchemy queries. Mar 19, 2025 4 - Beta N/A :pypi:`pytest-capture-deprecatedwarnings` pytest plugin to capture all deprecatedwarnings and put them in one file Apr 30, 2019 N/A N/A :pypi:`pytest-capture-warnings` pytest plugin to capture all warnings and put them in one file of your choice May 03, 2022 N/A pytest - :pypi:`pytest-cases` Separate test code from test cases in pytest. Apr 04, 2024 5 - Production/Stable N/A + :pypi:`pytest-case` A clean, modern, wrapper for pytest.mark.parametrize Nov 25, 2024 N/A pytest<9.0.0,>=8.3.3 + :pypi:`pytest-case-provider` Advanced pytest parametrization plugin that generates test case instances from sync or async factories. Oct 26, 2025 3 - Alpha pytest<9,>=8 + :pypi:`pytest-cases` Separate test code from test cases in pytest. Jun 09, 2025 5 - Production/Stable pytest + :pypi:`pytest-case-start-from` A pytest plugin to start test execution from a specific test case Oct 28, 2025 4 - Beta pytest>=6.0.0 + :pypi:`pytest-casewise-package-install` A pytest plugin for test case-level dynamic dependency management Oct 31, 2025 3 - Alpha pytest>=6.0.0 :pypi:`pytest-cassandra` Cassandra CCM Test Fixtures for pytest Nov 04, 2017 1 - Planning N/A :pypi:`pytest-catchlog` py.test plugin to catch log messages. This is a fork of pytest-capturelog. Jan 24, 2016 4 - Beta pytest (>=2.6) :pypi:`pytest-catch-server` Pytest plugin with server for catching HTTP requests. Dec 12, 2019 5 - Production/Stable N/A - :pypi:`pytest-celery` Pytest plugin for Celery Apr 11, 2024 4 - Beta N/A + :pypi:`pytest-cdist` A pytest plugin to split your test suite into multiple parts Jan 30, 2025 N/A pytest>=7 + :pypi:`pytest-celery` Pytest plugin for Celery Jul 30, 2025 5 - Production/Stable N/A + :pypi:`pytest-celery-py37` Pytest plugin for Celery (compatible with python 3.7) May 23, 2025 5 - Production/Stable N/A :pypi:`pytest-cfg-fetcher` Pass config options to your unit tests. Feb 26, 2024 N/A N/A :pypi:`pytest-chainmaker` pytest plugin for chainmaker Oct 15, 2021 N/A N/A :pypi:`pytest-chalice` A set of py.test fixtures for AWS Chalice Jul 01, 2020 4 - Beta N/A @@ -219,18 +256,20 @@ This list contains 1487 plugins. :pypi:`pytest-change-report` turn . into √,turn F into x Sep 14, 2020 N/A pytest :pypi:`pytest-change-xds` turn . into √,turn F into x Apr 16, 2022 N/A pytest :pypi:`pytest-chdir` A pytest fixture for changing current working directory Jan 28, 2020 N/A pytest (>=5.0.0,<6.0.0) - :pypi:`pytest-check` A pytest plugin that allows multiple failures per test. Jan 18, 2024 N/A pytest>=7.0.0 + :pypi:`pytest-check` A pytest plugin that allows multiple failures per test. Oct 07, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-checkdocs` check the README when running tests Apr 30, 2024 5 - Production/Stable pytest!=8.1.*,>=6; extra == "testing" :pypi:`pytest-checkipdb` plugin to check if there are ipdb debugs left Dec 04, 2023 5 - Production/Stable pytest >=2.9.2 :pypi:`pytest-check-library` check your missing library Jul 17, 2022 N/A N/A :pypi:`pytest-check-libs` check your missing library Jul 17, 2022 N/A N/A :pypi:`pytest-check-links` Check links in files Jul 29, 2020 N/A pytest<9,>=7.0 - :pypi:`pytest-checklist` Pytest plugin to track and report unit/function coverage. Jun 10, 2024 N/A N/A + :pypi:`pytest-checklist` Pytest plugin to track and report unit/function coverage. May 23, 2025 N/A N/A :pypi:`pytest-check-mk` pytest plugin to test Check_MK checks Nov 19, 2015 4 - Beta pytest - :pypi:`pytest-check-requirements` A package to prevent Dependency Confusion attacks against Yandex. Feb 20, 2024 N/A N/A + :pypi:`pytest-checkpoint` Restore a checkpoint in pytest Oct 04, 2025 N/A pytest>=8.0.0 :pypi:`pytest-ch-framework` My pytest framework Apr 17, 2024 N/A pytest==8.0.1 - :pypi:`pytest-chic-report` A pytest plugin to send a report and printing summary of tests. Jan 31, 2023 5 - Production/Stable N/A + :pypi:`pytest-chic-report` Simple pytest plugin for generating and sending report to messengers. Nov 01, 2024 N/A pytest>=6.0 + :pypi:`pytest-chinesereport` Apr 16, 2025 4 - Beta pytest>=3.5.0 :pypi:`pytest-choose` Provide the pytest with the ability to collect use cases based on rules in text files Feb 04, 2024 N/A pytest >=7.0.0 + :pypi:`pytest-chronicle` Reusable pytest results ingestion tooling with database export and CLI helpers. Oct 30, 2025 N/A pytest>=8.0; extra == "dev" :pypi:`pytest-chunks` Run only a chunk of your test suite Jul 05, 2022 N/A pytest (>=6.0.0) :pypi:`pytest_cid` Compare data structures containing matching CIDs of different versions and encoding Sep 01, 2023 4 - Beta pytest >= 5.0, < 7.0 :pypi:`pytest-circleci` py.test plugin for CircleCI May 03, 2019 N/A N/A @@ -238,28 +277,34 @@ This list contains 1487 plugins. :pypi:`pytest-circleci-parallelized-rjp` Parallelize pytest across CircleCI workers. Jun 21, 2022 N/A pytest :pypi:`pytest-ckan` Backport of CKAN 2.9 pytest plugin and fixtures to CAKN 2.8 Apr 28, 2020 4 - Beta pytest :pypi:`pytest-clarity` A plugin providing an alternative, colourful diff output for failing assertions. Jun 11, 2021 N/A N/A + :pypi:`pytest-class-fixtures` Class as PyTest fixtures (and BDD steps) Nov 15, 2024 N/A pytest<9.0.0,>=8.3.3 :pypi:`pytest-cldf` Easy quality control for CLDF datasets using pytest Nov 07, 2022 N/A pytest (>=3.6) - :pypi:`pytest-cleanslate` Collects and executes pytest tests separately Jun 17, 2024 N/A pytest + :pypi:`pytest-clean-database` A pytest plugin that cleans your database up after every test. Mar 14, 2025 3 - Alpha pytest<9,>=7.0 + :pypi:`pytest-cleanslate` Collects and executes pytest tests separately Apr 10, 2025 N/A pytest :pypi:`pytest_cleanup` Automated, comprehensive and well-organised pytest test cases. Jan 28, 2020 N/A N/A - :pypi:`pytest-cleanuptotal` A cleanup plugin for pytest Mar 19, 2024 5 - Production/Stable N/A - :pypi:`pytest-clerk` A set of pytest fixtures to help with integration testing with Clerk. Jun 27, 2024 N/A pytest<9.0.0,>=8.0.0 + :pypi:`pytest-cleanuptotal` A cleanup plugin for pytest Jul 22, 2025 5 - Production/Stable N/A + :pypi:`pytest-clerk` A set of pytest fixtures to help with integration testing with Clerk. Aug 30, 2025 N/A pytest<9.0.0,>=8.0.0 + :pypi:`pytest-cli2-ansible` Mar 05, 2025 N/A N/A :pypi:`pytest-click` Pytest plugin for Click Feb 11, 2022 5 - Production/Stable pytest (>=5.0) :pypi:`pytest-cli-fixtures` Automatically register fixtures for custom CLI arguments Jul 28, 2022 N/A pytest (~=7.0) - :pypi:`pytest-clld` Jul 06, 2022 N/A pytest (>=3.6) + :pypi:`pytest-clld` Oct 23, 2024 N/A pytest>=3.9 :pypi:`pytest-cloud` Distributed tests planner plugin for pytest testing framework. Oct 05, 2020 6 - Mature N/A :pypi:`pytest-cloudflare-worker` pytest plugin for testing cloudflare workers Mar 30, 2021 4 - Beta pytest (>=6.0.0) :pypi:`pytest-cloudist` Distribute tests to cloud machines without fuss Sep 02, 2022 4 - Beta pytest (>=7.1.2,<8.0.0) - :pypi:`pytest-cmake` Provide CMake module for Pytest May 31, 2024 N/A pytest<9,>=4 + :pypi:`pytest-cmake` Provide CMake module for Pytest Aug 14, 2025 N/A pytest<9,>=4 :pypi:`pytest-cmake-presets` Execute CMake Presets via pytest Dec 26, 2022 N/A pytest (>=7.2.0,<8.0.0) + :pypi:`pytest-cmdline-add-args` Pytest plugin for custom argument handling and Allure reporting. This plugin allows you to add arguments before running a test. Sep 01, 2024 N/A N/A :pypi:`pytest-cobra` PyTest plugin for testing Smart Contracts for Ethereum blockchain. Jun 29, 2019 3 - Alpha pytest (<4.0.0,>=3.7.1) + :pypi:`pytest-cocotb` Pytest plugin to integrate Cocotb Mar 15, 2025 5 - Production/Stable pytest; extra == "test" + :pypi:`pytest-codeblock` Pytest plugin to collect and test code blocks in reStructuredText and Markdown files. May 10, 2025 4 - Beta pytest :pypi:`pytest_codeblocks` Test code blocks in your READMEs Sep 17, 2023 5 - Production/Stable pytest >= 7.0.0 :pypi:`pytest-codecarbon` Pytest plugin for measuring carbon emissions Jun 15, 2022 N/A pytest :pypi:`pytest-codecheckers` pytest plugin to add source code sanity checks (pep8 and friends) Feb 13, 2010 N/A N/A - :pypi:`pytest-codecov` Pytest plugin for uploading pytest-cov results to codecov.io Nov 29, 2022 4 - Beta pytest (>=4.6.0) + :pypi:`pytest-codecov` Pytest plugin for uploading pytest-cov results to codecov.io Mar 25, 2025 4 - Beta pytest>=4.6.0 :pypi:`pytest-codegen` Automatically create pytest test signatures Aug 23, 2020 2 - Pre-Alpha N/A :pypi:`pytest-codeowners` Pytest plugin for selecting tests by GitHub CODEOWNERS. Mar 30, 2022 4 - Beta pytest (>=6.0.0) :pypi:`pytest-codestyle` pytest plugin to run pycodestyle Mar 23, 2020 3 - Alpha N/A - :pypi:`pytest-codspeed` Pytest plugin to create CodSpeed benchmarks Mar 19, 2024 5 - Production/Stable pytest>=3.8 + :pypi:`pytest-codspeed` Pytest plugin to create CodSpeed benchmarks Oct 24, 2025 5 - Production/Stable pytest>=3.8 :pypi:`pytest-collect-appoint-info` set your encoding Aug 03, 2023 N/A pytest :pypi:`pytest-collect-formatter` Formatter for pytest collect output Mar 29, 2021 5 - Production/Stable N/A :pypi:`pytest-collect-formatter2` Formatter for pytest collect output May 31, 2021 5 - Production/Stable N/A @@ -268,41 +313,45 @@ This list contains 1487 plugins. :pypi:`pytest-collect-pytest-interinfo` A simple plugin to use with pytest Sep 26, 2023 4 - Beta N/A :pypi:`pytest-colordots` Colorizes the progress indicators Oct 06, 2017 5 - Production/Stable N/A :pypi:`pytest-commander` An interactive GUI test runner for PyTest Aug 17, 2021 N/A pytest (<7.0.0,>=6.2.4) - :pypi:`pytest-common-subject` pytest framework for testing different aspects of a common method Jun 12, 2024 N/A pytest<9,>=3.6 + :pypi:`pytest-common-subject` pytest framework for testing different aspects of a common method Oct 22, 2025 N/A pytest<9,>=3.6 :pypi:`pytest-compare` pytest plugin for comparing call arguments. Jun 22, 2023 5 - Production/Stable N/A :pypi:`pytest-concurrent` Concurrently execute test cases with multithread, multiprocess and gevent Jan 12, 2019 4 - Beta pytest (>=3.1.1) + :pypi:`pytest-conductor` Pytest plugin for coordinating the order in which marked tests run. Jul 30, 2025 N/A pytest<8.4; python_version == "3.8" :pypi:`pytest-config` Base configurations and utilities for developing your Python project test suite with pytest. Nov 07, 2014 5 - Production/Stable N/A :pypi:`pytest-confluence-report` Package stands for pytest plugin to upload results into Confluence page. Apr 17, 2022 N/A N/A :pypi:`pytest-console-scripts` Pytest plugin for testing console scripts May 31, 2023 4 - Beta pytest (>=4.0.0) :pypi:`pytest-consul` pytest plugin with fixtures for testing consul aware apps Nov 24, 2018 3 - Alpha pytest - :pypi:`pytest-container` Pytest fixtures for writing container based tests Apr 10, 2024 4 - Beta pytest>=3.10 + :pypi:`pytest-container` Pytest fixtures for writing container based tests Jun 30, 2025 4 - Beta pytest>=3.10 :pypi:`pytest-contextfixture` Define pytest fixtures as context managers. Mar 12, 2013 4 - Beta N/A :pypi:`pytest-contexts` A plugin to run tests written with the Contexts framework using pytest May 19, 2021 4 - Beta N/A :pypi:`pytest-continuous` A pytest plugin to run tests continuously until failure or interruption. Apr 23, 2024 N/A N/A :pypi:`pytest-cookies` The pytest plugin for your Cookiecutter templates. 🍪 Mar 22, 2023 5 - Production/Stable pytest (>=3.9.0) - :pypi:`pytest-copie` The pytest plugin for your copier templates 📒 Jun 26, 2024 3 - Alpha pytest + :pypi:`pytest-copie` The pytest plugin for your copier templates 📒 Sep 29, 2025 3 - Alpha pytest :pypi:`pytest-copier` A pytest plugin to help testing Copier templates Dec 11, 2023 4 - Beta pytest>=7.3.2 :pypi:`pytest-couchdbkit` py.test extension for per-test couchdb databases using couchdbkit Apr 17, 2012 N/A N/A :pypi:`pytest-count` count erros and send email Jan 12, 2018 4 - Beta N/A - :pypi:`pytest-cov` Pytest plugin for measuring coverage. Mar 24, 2024 5 - Production/Stable pytest>=4.6 + :pypi:`pytest-cov` Pytest plugin for measuring coverage. Sep 09, 2025 5 - Production/Stable pytest>=7 :pypi:`pytest-cover` Pytest plugin for measuring coverage. Forked from \`pytest-cov\`. Aug 01, 2015 5 - Production/Stable N/A :pypi:`pytest-coverage` Jun 17, 2015 N/A N/A :pypi:`pytest-coverage-context` Coverage dynamic context support for PyTest, including sub-processes Jun 28, 2023 4 - Beta N/A - :pypi:`pytest-coveragemarkers` Using pytest markers to track functional coverage and filtering of tests Jun 04, 2024 N/A pytest<8.0.0,>=7.1.2 + :pypi:`pytest-coveragemarkers` Using pytest markers to track functional coverage and filtering of tests May 15, 2025 N/A pytest<8.0.0,>=7.1.2 :pypi:`pytest-cov-exclude` Pytest plugin for excluding tests based on coverage data Apr 29, 2016 4 - Beta pytest (>=2.8.0,<2.9.0); extra == 'dev' :pypi:`pytest_covid` Too many faillure, less tests. Jun 24, 2020 N/A N/A - :pypi:`pytest-cpp` Use pytest's runner to discover and execute C++ tests Nov 01, 2023 5 - Production/Stable pytest >=7.0 - :pypi:`pytest-cppython` A pytest plugin that imports CPPython testing types Mar 14, 2024 N/A N/A + :pypi:`pytest-cpp` Use pytest's runner to discover and execute C++ tests Sep 18, 2024 5 - Production/Stable pytest :pypi:`pytest-cqase` Custom qase pytest plugin Aug 22, 2022 N/A pytest (>=7.1.2,<8.0.0) :pypi:`pytest-cram` Run cram tests with pytest. Aug 08, 2020 N/A N/A :pypi:`pytest-crate` Manages CrateDB instances during your integration tests May 28, 2019 3 - Alpha pytest (>=4.0) - :pypi:`pytest-crayons` A pytest plugin for colorful print statements Oct 08, 2023 N/A pytest + :pypi:`pytest-cratedb` Manage CrateDB instances for integration tests Oct 08, 2024 4 - Beta pytest<9 + :pypi:`pytest-cratedb-reporter` A pytest plugin for reporting test results to CrateDB Mar 11, 2025 N/A pytest>=6.0.0 + :pypi:`pytest-crayons` A pytest plugin for colorful print statements Oct 14, 2025 5 - Production/Stable pytest + :pypi:`pytest-cream` The cream of test execution - smooth pytest workflows with intelligent orchestration Oct 26, 2025 N/A pytest :pypi:`pytest-create` pytest-create Feb 15, 2023 1 - Planning N/A :pypi:`pytest-cricri` A Cricri plugin for pytest. Jan 27, 2018 N/A pytest :pypi:`pytest-crontab` add crontab task in crontab Dec 09, 2019 N/A N/A :pypi:`pytest-csv` CSV output for pytest. Apr 22, 2021 N/A pytest (>=6.0) - :pypi:`pytest-csv-params` Pytest plugin for Test Case Parametrization with CSV files Jul 01, 2023 5 - Production/Stable pytest (>=7.4.0,<8.0.0) - :pypi:`pytest-curio` Pytest support for curio. Oct 07, 2020 N/A N/A + :pypi:`pytest-csv-params` Pytest plugin for Test Case Parametrization with CSV files May 29, 2025 5 - Production/Stable pytest<9,>=8.3 + :pypi:`pytest-culprit` Find the last Git commit where a pytest test started failing May 15, 2025 N/A N/A + :pypi:`pytest-curio` Pytest support for curio. Oct 06, 2024 N/A pytest :pypi:`pytest-curl-report` pytest plugin to generate curl command line report Dec 11, 2016 4 - Beta N/A :pypi:`pytest-custom-concurrency` Custom grouping concurrence for pytest Feb 08, 2021 N/A N/A :pypi:`pytest-custom-exit-code` Exit pytest test session with custom exit code in different scenarios Aug 07, 2019 4 - Beta pytest (>=4.0.2) @@ -310,61 +359,70 @@ This list contains 1487 plugins. :pypi:`pytest-custom-outputs` A plugin that allows users to create and use custom outputs instead of the standard Pass and Fail. Also allows users to retrieve test results in fixtures. Jul 10, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-custom-report` Configure the symbols displayed for test outcomes Jan 30, 2019 N/A pytest :pypi:`pytest-custom-scheduling` Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report Mar 01, 2021 N/A N/A + :pypi:`pytest-custom-timeout` Use custom logic when a test times out. Based on pytest-timeout. Jan 08, 2025 4 - Beta pytest>=8.0.0 :pypi:`pytest-cython` A plugin for testing Cython extension modules Apr 05, 2024 5 - Production/Stable pytest>=8 :pypi:`pytest-cython-collect` Jun 17, 2022 N/A pytest :pypi:`pytest-darker` A pytest plugin for checking of modified code using Darker Feb 25, 2024 N/A pytest <7,>=6.0.1 :pypi:`pytest-dash` pytest fixtures to run dash applications. Mar 18, 2019 N/A N/A - :pypi:`pytest-dashboard` May 30, 2024 N/A pytest<8.0.0,>=7.4.3 + :pypi:`pytest-dashboard` Jun 02, 2025 N/A pytest<8.0.0,>=7.4.3 :pypi:`pytest-data` Useful functions for managing data for pytest fixtures Nov 01, 2016 5 - Production/Stable N/A - :pypi:`pytest-databases` Reusable database fixtures for any and all databases. Jul 02, 2024 4 - Beta pytest + :pypi:`pytest-databases` Reusable database fixtures for any and all databases. Oct 06, 2025 4 - Beta pytest :pypi:`pytest-databricks` Pytest plugin for remote Databricks notebooks testing Jul 29, 2020 N/A pytest - :pypi:`pytest-datadir` pytest plugin for test data directories and files Oct 03, 2023 5 - Production/Stable pytest >=5.0 + :pypi:`pytest-datadir` pytest plugin for test data directories and files Jul 30, 2025 5 - Production/Stable pytest>=7.0 :pypi:`pytest-datadir-mgr` Manager for test data: downloads, artifact caching, and a tmpdir context. Apr 06, 2023 5 - Production/Stable pytest (>=7.1) :pypi:`pytest-datadir-ng` Fixtures for pytest allowing test functions/methods to easily retrieve test resources from the local filesystem. Dec 25, 2019 5 - Production/Stable pytest :pypi:`pytest-datadir-nng` Fixtures for pytest allowing test functions/methods to easily retrieve test resources from the local filesystem. Nov 09, 2022 5 - Production/Stable pytest (>=7.0.0,<8.0.0) :pypi:`pytest-data-extractor` A pytest plugin to extract relevant metadata about tests into an external file (currently only json support) Jul 19, 2022 N/A pytest (>=7.0.1) :pypi:`pytest-data-file` Fixture "data" and "case_data" for test from yaml file Dec 04, 2019 N/A N/A :pypi:`pytest-datafiles` py.test plugin to create a 'tmp_path' containing predefined files/directories. Feb 24, 2023 5 - Production/Stable pytest (>=3.6) - :pypi:`pytest-datafixtures` Data fixtures for pytest made simple Dec 05, 2020 5 - Production/Stable N/A + :pypi:`pytest-datafixtures` Data fixtures for pytest made simple. May 15, 2025 5 - Production/Stable N/A :pypi:`pytest-data-from-files` pytest plugin to provide data from files loaded automatically Oct 13, 2021 4 - Beta pytest + :pypi:`pytest-dataguard` Data validation and integrity testing for your datasets using pytest. Oct 08, 2025 N/A pytest>=8.4.2 + :pypi:`pytest-data-loader` Pytest plugin for loading test data for data-driven testing (DDT) Oct 29, 2025 4 - Beta pytest<9,>=7.0.0 :pypi:`pytest-dataplugin` A pytest plugin for managing an archive of test data. Sep 16, 2017 1 - Planning N/A - :pypi:`pytest-datarecorder` A py.test plugin recording and comparing test output. Feb 15, 2024 5 - Production/Stable pytest + :pypi:`pytest-datarecorder` A py.test plugin recording and comparing test output. Jul 31, 2024 5 - Production/Stable pytest :pypi:`pytest-dataset` Plugin for loading different datasets for pytest by prefix from json or yaml files Sep 01, 2023 5 - Production/Stable N/A :pypi:`pytest-data-suites` Class-based pytest parametrization Apr 06, 2024 N/A pytest<9.0,>=6.0 :pypi:`pytest-datatest` A pytest plugin for test driven data-wrangling (this is the development version of datatest's pytest integration). Oct 15, 2020 4 - Beta pytest (>=3.3) - :pypi:`pytest-db` Session scope fixture "db" for mysql query or change Dec 04, 2019 N/A N/A + :pypi:`pytest-db` Session scope fixture "db" for mysql query or change Aug 22, 2024 N/A pytest :pypi:`pytest-dbfixtures` Databases fixtures plugin for py.test. Dec 07, 2016 4 - Beta N/A :pypi:`pytest-db-plugin` Nov 27, 2021 N/A pytest (>=5.0) :pypi:`pytest-dbt` Unit test dbt models with standard python tooling Jun 08, 2023 2 - Pre-Alpha pytest (>=7.0.0,<8.0.0) :pypi:`pytest-dbt-adapter` A pytest plugin for testing dbt adapter plugins Nov 24, 2021 N/A pytest (<7,>=6) :pypi:`pytest-dbt-conventions` A pytest plugin for linting a dbt project's conventions Mar 02, 2022 N/A pytest (>=6.2.5,<7.0.0) :pypi:`pytest-dbt-core` Pytest extension for dbt. Jun 04, 2024 N/A pytest>=6.2.5; extra == "test" - :pypi:`pytest-dbt-postgres` Pytest tooling to unittest DBT & Postgres models Jan 02, 2024 N/A pytest (>=7.4.3,<8.0.0) + :pypi:`pytest-dbt-duckdb` Fearless testing for dbt models, powered by DuckDB. Oct 28, 2025 4 - Beta pytest>=8.3.4 + :pypi:`pytest-dbt-postgres` Pytest tooling to unittest DBT & Postgres models Sep 03, 2024 N/A pytest<9.0.0,>=8.3.2 :pypi:`pytest-dbus-notification` D-BUS notifications for pytest results. Mar 05, 2014 5 - Production/Stable N/A :pypi:`pytest-dbx` Pytest plugin to run unit tests for dbx (Databricks CLI extensions) related code Nov 29, 2022 N/A pytest (>=7.1.3,<8.0.0) :pypi:`pytest-dc` Manages Docker containers during your integration tests Aug 16, 2023 5 - Production/Stable pytest >=3.3 :pypi:`pytest-deadfixtures` A simple plugin to list unused fixtures in pytest Jul 23, 2020 5 - Production/Stable N/A :pypi:`pytest-deduplicate` Identifies duplicate unit tests Aug 12, 2023 4 - Beta pytest + :pypi:`pytest-deepassert` A pytest plugin for enhanced assertion reporting with detailed diffs Sep 02, 2025 3 - Alpha pytest>=7.0.0 :pypi:`pytest-deepcov` deepcov Mar 30, 2021 N/A N/A - :pypi:`pytest-defer` Aug 24, 2021 N/A N/A + :pypi:`pytest_defer` A 'defer' fixture for pytest Nov 13, 2024 N/A pytest>=8.3 + :pypi:`pytest-delta` Run only tests impacted by your code changes (delta-based selection) for pytest. Oct 27, 2025 4 - Beta pytest>=7.0 :pypi:`pytest-demo-plugin` pytest示例插件 May 15, 2021 N/A N/A :pypi:`pytest-dependency` Manage dependencies of tests Dec 31, 2023 4 - Beta N/A :pypi:`pytest-depends` Tests that depend on other tests Apr 05, 2020 5 - Production/Stable pytest (>=3) + :pypi:`pytest-depper` Smart test selection based on AST-level code dependency analysis Oct 23, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-deprecate` Mark tests as testing a deprecated feature with a warning note. Jul 01, 2019 N/A N/A - :pypi:`pytest-describe` Describe-style plugin for pytest Feb 10, 2024 5 - Production/Stable pytest <9,>=4.6 + :pypi:`pytest-deprecator` A simple plugin to use with pytest Dec 02, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-describe` Describe-style plugin for pytest Oct 23, 2025 5 - Production/Stable pytest<9,>=6 :pypi:`pytest-describe-it` plugin for rich text descriptions Jul 19, 2019 4 - Beta pytest - :pypi:`pytest-deselect-if` A plugin to deselect pytests tests rather than using skipif Mar 24, 2024 4 - Beta pytest>=6.2.0 - :pypi:`pytest-devpi-server` DevPI server fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-deselect-if` A plugin to deselect pytests tests rather than using skipif Dec 26, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-devpi-server` DevPI server fixture for py.test Oct 17, 2024 5 - Production/Stable pytest + :pypi:`pytest-dfm` pytest-dfm provides a pytest integration for DV Flow Manager, a build system for silicon design Sep 13, 2025 N/A pytest :pypi:`pytest-dhos` Common fixtures for pytest in DHOS services and libraries Sep 07, 2022 N/A N/A :pypi:`pytest-diamond` pytest plugin for diamond Aug 31, 2015 4 - Beta N/A :pypi:`pytest-dicom` pytest plugin to provide DICOM fixtures Dec 19, 2018 3 - Alpha pytest :pypi:`pytest-dictsdiff` Jul 26, 2019 N/A N/A :pypi:`pytest-diff` A simple plugin to use with pytest Mar 30, 2019 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-diffeo` A package to prevent Dependency Confusion attacks against Yandex. Feb 20, 2024 N/A N/A :pypi:`pytest-diff-selector` Get tests affected by code changes (using git) Feb 24, 2022 4 - Beta pytest (>=6.2.2) ; extra == 'all' :pypi:`pytest-difido` PyTest plugin for generating Difido reports Oct 23, 2022 4 - Beta pytest (>=4.0.0) + :pypi:`pytest-directives` Control your tests flow Aug 11, 2025 3 - Alpha pytest :pypi:`pytest-dir-equal` pytest-dir-equals is a pytest plugin providing helpers to assert directories equality allowing golden testing Dec 11, 2023 4 - Beta pytest>=7.3.2 - :pypi:`pytest-dirty` Static import analysis for thrifty testing. Jul 11, 2024 3 - Alpha pytest>=8.2; extra == "dev" + :pypi:`pytest-dirty` Static import analysis for thrifty testing. Jun 08, 2025 3 - Alpha pytest>=8.2; extra == "dev" :pypi:`pytest-disable` pytest plugin to disable a test and skip it from testrun Sep 10, 2015 4 - Beta N/A :pypi:`pytest-disable-plugin` Disable plugins per test Feb 28, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-discord` A pytest plugin to notify test results to a Discord channel. May 11, 2024 4 - Beta pytest!=6.0.0,<9,>=3.3.2 @@ -372,9 +430,9 @@ This list contains 1487 plugins. :pypi:`pytest-ditto` Snapshot testing pytest plugin with minimal ceremony and flexible persistence formats. Jun 09, 2024 4 - Beta pytest>=3.5.0 :pypi:`pytest-ditto-pandas` pytest-ditto plugin for pandas snapshots. May 29, 2024 4 - Beta pytest>=3.5.0 :pypi:`pytest-ditto-pyarrow` pytest-ditto plugin for pyarrow tables. Jun 09, 2024 4 - Beta pytest>=3.5.0 - :pypi:`pytest-django` A Django plugin for pytest. Jan 30, 2024 5 - Production/Stable pytest >=7.0.0 + :pypi:`pytest-django` A Django plugin for pytest. Apr 03, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-django-ahead` A Django plugin for pytest. Oct 27, 2016 5 - Production/Stable pytest (>=2.9) - :pypi:`pytest-djangoapp` Nice pytest plugin to help you with Django pluggable application testing. May 19, 2023 4 - Beta pytest + :pypi:`pytest-djangoapp` Nice pytest plugin to help you with Django pluggable application testing. Sep 28, 2025 5 - Production/Stable pytest :pypi:`pytest-django-cache-xdist` A djangocachexdist plugin for pytest May 12, 2020 4 - Beta N/A :pypi:`pytest-django-casperjs` Integrate CasperJS with your django tests as a pytest fixture. Mar 15, 2015 2 - Pre-Alpha N/A :pypi:`pytest-django-class` A pytest plugin for running django in class-scoped fixtures Aug 08, 2023 4 - Beta N/A @@ -384,9 +442,9 @@ This list contains 1487 plugins. :pypi:`pytest-django-filefield` Replaces FileField.storage with something you can patch globally. May 09, 2022 5 - Production/Stable pytest >= 5.2 :pypi:`pytest-django-gcir` A Django plugin for pytest. Mar 06, 2018 5 - Production/Stable N/A :pypi:`pytest-django-haystack` Cleanup your Haystack indexes between tests Sep 03, 2017 5 - Production/Stable pytest (>=2.3.4) - :pypi:`pytest-django-ifactory` A model instance factory for pytest-django Aug 27, 2023 5 - Production/Stable N/A + :pypi:`pytest-django-ifactory` A model instance factory for pytest-django Apr 30, 2025 5 - Production/Stable N/A :pypi:`pytest-django-lite` The bare minimum to integrate py.test with Django. Jan 30, 2014 N/A N/A - :pypi:`pytest-django-liveserver-ssl` Jan 20, 2022 3 - Alpha N/A + :pypi:`pytest-django-liveserver-ssl` Jan 09, 2025 3 - Alpha N/A :pypi:`pytest-django-model` A Simple Way to Test your Django Models Feb 14, 2019 4 - Beta N/A :pypi:`pytest-django-ordering` A pytest plugin for preserving the order in which Django runs tests. Jul 25, 2019 5 - Production/Stable pytest (>=2.3.0) :pypi:`pytest-django-queries` Generate performance reports from your django database performance tests. Mar 01, 2021 N/A N/A @@ -397,30 +455,31 @@ This list contains 1487 plugins. :pypi:`pytest-doc` A documentation plugin for py.test. Jun 28, 2015 5 - Production/Stable N/A :pypi:`pytest-docfiles` pytest plugin to test codeblocks in your documentation. Dec 22, 2021 4 - Beta pytest (>=3.7.0) :pypi:`pytest-docgen` An RST Documentation Generator for pytest-based test suites Apr 17, 2020 N/A N/A - :pypi:`pytest-docker` Simple pytest fixtures for Docker and Docker Compose based tests Feb 02, 2024 N/A pytest <9.0,>=4.0 - :pypi:`pytest-docker-apache-fixtures` Pytest fixtures for testing with apache2 (httpd). Feb 16, 2022 4 - Beta pytest + :pypi:`pytest-docker` Simple pytest fixtures for Docker and Docker Compose based tests Jul 04, 2025 N/A pytest<9.0,>=4.0 + :pypi:`pytest-docker-apache-fixtures` Pytest fixtures for testing with apache2 (httpd). Aug 12, 2024 4 - Beta pytest :pypi:`pytest-docker-butla` Jun 16, 2019 3 - Alpha N/A :pypi:`pytest-dockerc` Run, manage and stop Docker Compose project from Docker API Oct 09, 2020 5 - Production/Stable pytest (>=3.0) :pypi:`pytest-docker-compose` Manages Docker containers during your integration tests Jan 26, 2021 5 - Production/Stable pytest (>=3.3) - :pypi:`pytest-docker-compose-v2` Manages Docker containers during your integration tests Feb 28, 2024 4 - Beta pytest<8,>=7.2.2 + :pypi:`pytest-docker-compose-v2` Manages Docker containers during your integration tests Dec 11, 2024 4 - Beta pytest>=7.2.2 :pypi:`pytest-docker-db` A plugin to use docker databases for pytests Mar 20, 2021 5 - Production/Stable pytest (>=3.1.1) - :pypi:`pytest-docker-fixtures` pytest docker fixtures Apr 03, 2024 3 - Alpha N/A - :pypi:`pytest-docker-git-fixtures` Pytest fixtures for testing with git scm. Feb 09, 2022 4 - Beta pytest - :pypi:`pytest-docker-haproxy-fixtures` Pytest fixtures for testing with haproxy. Feb 09, 2022 4 - Beta pytest + :pypi:`pytest-docker-fixtures` pytest docker fixtures Jun 25, 2025 3 - Alpha pytest + :pypi:`pytest-docker-git-fixtures` Pytest fixtures for testing with git scm. Aug 12, 2024 4 - Beta pytest + :pypi:`pytest-docker-haproxy-fixtures` Pytest fixtures for testing with haproxy. Aug 12, 2024 4 - Beta pytest :pypi:`pytest-docker-pexpect` pytest plugin for writing functional tests with pexpect and docker Jan 14, 2019 N/A pytest :pypi:`pytest-docker-postgresql` A simple plugin to use with pytest Sep 24, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-docker-py` Easy to use, simple to extend, pytest plugin that minimally leverages docker-py. Nov 27, 2018 N/A pytest (==4.0.0) - :pypi:`pytest-docker-registry-fixtures` Pytest fixtures for testing with docker registries. Apr 08, 2022 4 - Beta pytest + :pypi:`pytest-docker-registry-fixtures` Pytest fixtures for testing with docker registries. Aug 12, 2024 4 - Beta pytest :pypi:`pytest-docker-service` pytest plugin to start docker container Jan 03, 2024 3 - Alpha pytest (>=7.1.3) - :pypi:`pytest-docker-squid-fixtures` Pytest fixtures for testing with squid. Feb 09, 2022 4 - Beta pytest - :pypi:`pytest-docker-tools` Docker integration tests for pytest Feb 17, 2022 4 - Beta pytest (>=6.0.1) + :pypi:`pytest-docker-squid-fixtures` Pytest fixtures for testing with squid. Aug 12, 2024 4 - Beta pytest + :pypi:`pytest-docker-tools` Docker integration tests for pytest Mar 16, 2025 4 - Beta pytest>=6.0.1 :pypi:`pytest-docs` Documentation tool for pytest Nov 11, 2018 4 - Beta pytest (>=3.5.0) :pypi:`pytest-docstyle` pytest plugin to run pydocstyle Mar 23, 2020 3 - Alpha N/A :pypi:`pytest-doctest-custom` A py.test plugin for customizing string representations of doctest results. Jul 25, 2016 4 - Beta N/A :pypi:`pytest-doctest-ellipsis-markers` Setup additional values for ELLIPSIS_MARKER for doctests Jan 12, 2018 4 - Beta N/A :pypi:`pytest-doctest-import` A simple pytest plugin to import names and add them to the doctest namespace. Nov 13, 2018 4 - Beta pytest (>=3.3.0) :pypi:`pytest-doctest-mkdocstrings` Run pytest --doctest-modules with markdown docstrings in code blocks (\`\`\`) Mar 02, 2024 N/A pytest - :pypi:`pytest-doctestplus` Pytest plugin with advanced doctest features. Mar 10, 2024 5 - Production/Stable pytest >=4.6 + :pypi:`pytest-doctest-only` A plugin to run only doctest Jul 30, 2025 4 - Beta pytest>=8.3.0 + :pypi:`pytest-doctestplus` Pytest plugin with advanced doctest features. Oct 18, 2025 5 - Production/Stable pytest>=4.6 :pypi:`pytest-documentary` A simple pytest plugin to generate test documentation Jul 11, 2024 N/A pytest :pypi:`pytest-dogu-report` pytest plugin for dogu report Jul 07, 2023 N/A N/A :pypi:`pytest-dogu-sdk` pytest plugin for the Dogu Dec 14, 2023 N/A N/A @@ -428,111 +487,132 @@ This list contains 1487 plugins. :pypi:`pytest-donde` record pytest session characteristics per test item (coverage and duration) into a persistent file and use them in your own plugin or script. Oct 01, 2023 4 - Beta pytest >=7.3.1 :pypi:`pytest-doorstop` A pytest plugin for adding test results into doorstop items. Jun 09, 2020 4 - Beta pytest (>=3.5.0) :pypi:`pytest-dotenv` A py.test plugin that parses environment files before running tests Jun 16, 2020 4 - Beta pytest (>=5.0.0) + :pypi:`pytest-dotenv-modern` A modern pytest plugin that loads environment variables from dotenv files Sep 27, 2025 4 - Beta pytest>=6.0.0 :pypi:`pytest-dot-only-pkcopley` A Pytest marker for only running a single test Oct 27, 2023 N/A N/A + :pypi:`pytest-dparam` A more readable alternative to @pytest.mark.parametrize. Aug 27, 2024 6 - Mature pytest + :pypi:`pytest-dpg` pytest-dpg is a pytest plugin for testing Dear PyGui (DPG) applications Aug 13, 2024 N/A N/A :pypi:`pytest-draw` Pytest plugin for randomly selecting a specific number of tests Mar 21, 2023 3 - Alpha pytest :pypi:`pytest-drf` A Django REST framework plugin for pytest. Jul 12, 2022 5 - Production/Stable pytest (>=3.7) + :pypi:`pytest-drill-sergeant` A pytest plugin that enforces test quality standards through automatic marker detection and AAA structure validation Sep 12, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-drivings` Tool to allow webdriver automation to be ran locally or remotely Jan 13, 2021 N/A N/A :pypi:`pytest-drop-dup-tests` A Pytest plugin to drop duplicated tests during collection Mar 04, 2024 5 - Production/Stable pytest >=7 - :pypi:`pytest-dryrun` A Pytest plugin to ignore tests during collection without reporting them in the test summary. Jul 18, 2023 5 - Production/Stable pytest (>=7.4.0,<8.0.0) + :pypi:`pytest-dryci` Test caching plugin for pytest Sep 27, 2024 4 - Beta N/A + :pypi:`pytest-dryrun` A Pytest plugin to ignore tests during collection without reporting them in the test summary. Jan 19, 2025 5 - Production/Stable pytest<9,>=7.40 + :pypi:`pytest-dsl` A DSL testing framework based on pytest Oct 31, 2025 N/A pytest>=7.0.0 + :pypi:`pytest-dsl-ssh` SSH/SFTP关键字插件,为pytest-dsl提供SSH和SFTP操作能力 Jul 25, 2025 4 - Beta pytest>=7.0.0 + :pypi:`pytest-dsl-ui` Playwright-based UI automation keywords for pytest-dsl framework Aug 21, 2025 N/A pytest>=7.0.0; extra == "dev" :pypi:`pytest-dummynet` A py.test plugin providing access to a dummynet. Dec 15, 2021 5 - Production/Stable pytest :pypi:`pytest-dump2json` A pytest plugin for dumping test results to json. Jun 29, 2015 N/A N/A - :pypi:`pytest-duration-insights` Jun 25, 2021 N/A N/A - :pypi:`pytest-durations` Pytest plugin reporting fixtures and test functions execution time. Apr 22, 2022 5 - Production/Stable pytest (>=4.6) + :pypi:`pytest-duration-insights` Jul 15, 2024 N/A N/A + :pypi:`pytest-durations` Pytest plugin reporting fixtures and test functions execution time. Aug 29, 2025 5 - Production/Stable pytest>=4.6 + :pypi:`pytest-dynamic-parameterize` A Python package for managing pytest plugins. Oct 14, 2025 N/A pytest :pypi:`pytest-dynamicrerun` A pytest plugin to rerun tests dynamically based off of test outcome and output. Aug 15, 2020 4 - Beta N/A - :pypi:`pytest-dynamodb` DynamoDB fixtures for pytest Mar 12, 2024 5 - Production/Stable pytest + :pypi:`pytest-dynamodb` DynamoDB fixtures for pytest Apr 04, 2025 5 - Production/Stable pytest :pypi:`pytest-easy-addoption` pytest-easy-addoption: Easy way to work with pytest addoption Jan 22, 2020 N/A N/A - :pypi:`pytest-easy-api` A package to prevent Dependency Confusion attacks against Yandex. Feb 16, 2024 N/A N/A :pypi:`pytest-easyMPI` Package that supports mpi tests in pytest Oct 21, 2020 N/A N/A :pypi:`pytest-easyread` pytest plugin that makes terminal printouts of the reports easier to read Nov 17, 2017 N/A N/A :pypi:`pytest-easy-server` Pytest plugin for easy testing against servers May 01, 2021 4 - Beta pytest (<5.0.0,>=4.3.1) ; python_version < "3.5" :pypi:`pytest-ebics-sandbox` A pytest plugin for testing against an EBICS sandbox server. Requires docker. Aug 15, 2022 N/A N/A :pypi:`pytest-ec2` Pytest execution on EC2 instance Oct 22, 2019 3 - Alpha N/A - :pypi:`pytest-echo` pytest plugin with mechanisms for echoing environment variables, package version and generic attributes Dec 05, 2023 5 - Production/Stable pytest >=2.2 - :pypi:`pytest-edit` Edit the source code of a failed test with \`pytest --edit\`. Jun 09, 2024 N/A pytest + :pypi:`pytest-echo` pytest plugin that allows to dump environment variables, package version and generic attributes Apr 27, 2025 5 - Production/Stable pytest>=8.3.3 + :pypi:`pytest-edit` Edit the source code of a failed test with \`pytest --edit\`. Nov 17, 2024 N/A pytest :pypi:`pytest-ekstazi` Pytest plugin to select test using Ekstazi algorithm Sep 10, 2022 N/A pytest - :pypi:`pytest-elasticsearch` Elasticsearch fixtures and fixture factories for Pytest. Mar 15, 2024 5 - Production/Stable pytest >=7.0 + :pypi:`pytest-elasticsearch` Elasticsearch fixtures and fixture factories for Pytest. Dec 03, 2024 5 - Production/Stable pytest>=7.0 + :pypi:`pytest-elasticsearch-test` Elasticsearch fixtures and fixture factories for Pytest. Apr 20, 2025 5 - Production/Stable pytest>=7.0 :pypi:`pytest-elements` Tool to help automate user interfaces Jan 13, 2021 N/A pytest (>=5.4,<6.0) :pypi:`pytest-eliot` An eliot plugin for pytest. Aug 31, 2022 1 - Planning pytest (>=5.4.0) - :pypi:`pytest-elk-reporter` A simple plugin to use with pytest Apr 04, 2024 4 - Beta pytest>=3.5.0 + :pypi:`pytest-elk-reporter` A simple plugin to use with pytest Jul 25, 2024 4 - Beta pytest>=3.5.0 :pypi:`pytest-email` Send execution result email Jul 08, 2020 N/A pytest - :pypi:`pytest-embedded` A pytest plugin that designed for embedded testing. May 31, 2024 5 - Production/Stable pytest>=7.0 - :pypi:`pytest-embedded-arduino` Make pytest-embedded plugin work with Arduino. May 23, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-idf` Make pytest-embedded plugin work with ESP-IDF. May 23, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-jtag` Make pytest-embedded plugin work with JTAG. May 23, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-qemu` Make pytest-embedded plugin work with QEMU. May 23, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-serial` Make pytest-embedded plugin work with Serial. May 31, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-serial-esp` Make pytest-embedded plugin work with Espressif target boards. May 31, 2024 5 - Production/Stable N/A - :pypi:`pytest-embedded-wokwi` Make pytest-embedded plugin work with the Wokwi CLI. May 23, 2024 5 - Production/Stable N/A + :pypi:`pytest-embedded` A pytest plugin that designed for embedded testing. Oct 27, 2025 5 - Production/Stable pytest>=7.0 + :pypi:`pytest-embedded-arduino` Make pytest-embedded plugin work with Arduino. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-idf` Make pytest-embedded plugin work with ESP-IDF. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-jtag` Make pytest-embedded plugin work with JTAG. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-nuttx` Make pytest-embedded plugin work with NuttX. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-qemu` Make pytest-embedded plugin work with QEMU. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-serial` Make pytest-embedded plugin work with Serial. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-serial-esp` Make pytest-embedded plugin work with Espressif target boards. Oct 27, 2025 5 - Production/Stable N/A + :pypi:`pytest-embedded-wokwi` Make pytest-embedded plugin work with the Wokwi CLI. Oct 27, 2025 5 - Production/Stable N/A :pypi:`pytest-embrace` 💝 Dataclasses-as-tests. Describe the runtime once and multiply coverage with no boilerplate. Mar 25, 2023 N/A pytest (>=7.0,<8.0) :pypi:`pytest-emoji` A pytest plugin that adds emojis to your test result report Feb 19, 2019 4 - Beta pytest (>=4.2.1) :pypi:`pytest-emoji-output` Pytest plugin to represent test output with emoji support Apr 09, 2023 4 - Beta pytest (==7.0.1) - :pypi:`pytest-enabler` Enable installed pytest plugins Mar 21, 2024 5 - Production/Stable pytest>=6; extra == "testing" + :pypi:`pytest-enabler` Enable installed pytest plugins May 16, 2025 5 - Production/Stable pytest!=8.1.*,>=6; extra == "test" :pypi:`pytest-encode` set your encoding and logger Nov 06, 2021 N/A N/A :pypi:`pytest-encode-kane` set your encoding and logger Nov 16, 2021 N/A pytest :pypi:`pytest-encoding` set your encoding and logger Aug 11, 2023 N/A pytest :pypi:`pytest_energy_reporter` An energy estimation reporter for pytest Mar 28, 2024 3 - Alpha pytest<9.0.0,>=8.1.1 :pypi:`pytest-enhanced-reports` Enhanced test reports for pytest Dec 15, 2022 N/A N/A :pypi:`pytest-enhancements` Improvements for pytest (rejected upstream) Oct 30, 2019 4 - Beta N/A - :pypi:`pytest-env` pytest plugin that allows you to add environment variables. Nov 28, 2023 5 - Production/Stable pytest>=7.4.3 + :pypi:`pytest-env` pytest plugin that allows you to add environment variables. Oct 09, 2025 5 - Production/Stable pytest>=8.4.2 :pypi:`pytest-envfiles` A py.test plugin that parses environment files before running tests Oct 08, 2015 3 - Alpha N/A :pypi:`pytest-env-info` Push information about the running pytest into envvars Nov 25, 2017 4 - Beta pytest (>=3.1.1) :pypi:`pytest-environment` Pytest Environment Mar 17, 2024 1 - Planning N/A :pypi:`pytest-envraw` py.test plugin that allows you to add environment variables. Aug 27, 2020 4 - Beta pytest (>=2.6.0) :pypi:`pytest-envvars` Pytest plugin to validate use of envvars on your tests Jun 13, 2020 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-envx` Pytest plugin for managing environment variables with interpolation and .env file support. Jun 28, 2025 4 - Beta pytest>=8.4.1 :pypi:`pytest-env-yaml` Apr 02, 2019 N/A N/A :pypi:`pytest-eradicate` pytest plugin to check for commented out code Sep 08, 2020 N/A pytest (>=2.4.2) :pypi:`pytest_erp` py.test plugin to send test info to report portal dynamically Jan 13, 2015 N/A N/A :pypi:`pytest-error-for-skips` Pytest plugin to treat skipped tests a test failure Dec 19, 2019 4 - Beta pytest (>=4.6) + :pypi:`pytest-errxfail` pytest plugin to mark a test as xfailed if it fails with the specified error message in the captured output Jan 06, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-essentials` A Pytest plugin providing essential utilities like soft assertions. May 19, 2025 3 - Alpha pytest>=7.0 :pypi:`pytest-eth` PyTest plugin for testing Smart Contracts for Ethereum Virtual Machine (EVM). Aug 14, 2020 1 - Planning N/A :pypi:`pytest-ethereum` pytest-ethereum: Pytest library for ethereum projects. Jun 24, 2019 3 - Alpha pytest (==3.3.2); extra == 'dev' :pypi:`pytest-eucalyptus` Pytest Plugin for BDD Jun 28, 2022 N/A pytest (>=4.2.0) + :pypi:`pytest-evals` A pytest plugin for running and analyzing LLM evaluation tests Feb 02, 2025 N/A pytest>=7.0.0 :pypi:`pytest-eventlet` Applies eventlet monkey-patch as a pytest plugin. Oct 04, 2021 N/A pytest ; extra == 'dev' - :pypi:`pytest-evm` The testing package containing tools to test Web3-based projects Apr 22, 2024 4 - Beta pytest<9.0.0,>=8.1.1 + :pypi:`pytest-everyfunc` A pytest plugin to detect completely untested functions using coverage Apr 30, 2025 4 - Beta pytest + :pypi:`pytest_evm` The testing package containing tools to test Web3-based projects Sep 23, 2024 4 - Beta pytest<9.0.0,>=8.1.1 :pypi:`pytest_exact_fixtures` Parse queries in Lucene and Elasticsearch syntaxes Feb 04, 2019 N/A N/A - :pypi:`pytest-examples` Pytest plugin for testing examples in docstrings and markdown files. Jul 02, 2024 4 - Beta pytest>=7 - :pypi:`pytest-exasol-itde` Jul 01, 2024 N/A pytest<9,>=7 - :pypi:`pytest-exasol-saas` Jun 07, 2024 N/A pytest<9,>=7 - :pypi:`pytest-excel` pytest plugin for generating excel reports Jun 18, 2024 5 - Production/Stable pytest>3.6 + :pypi:`pytest-examples` Pytest plugin for testing examples in docstrings and markdown files. May 06, 2025 N/A pytest>=7 + :pypi:`pytest-exasol-backend` Oct 29, 2025 N/A pytest<9,>=7 + :pypi:`pytest-exasol-extension` Oct 29, 2025 N/A pytest<9,>=7 + :pypi:`pytest-exasol-itde` Nov 22, 2024 N/A pytest<9,>=7 + :pypi:`pytest-exasol-saas` Nov 22, 2024 N/A pytest<9,>=7 + :pypi:`pytest-exasol-slc` Oct 30, 2025 N/A pytest<9,>=7 + :pypi:`pytest-excel` pytest plugin for generating excel reports Jul 22, 2025 5 - Production/Stable pytest :pypi:`pytest-exceptional` Better exceptions Mar 16, 2017 4 - Beta N/A :pypi:`pytest-exception-script` Walk your code through exception script to check it's resiliency to failures. Aug 04, 2020 3 - Alpha pytest :pypi:`pytest-executable` pytest plugin for testing executables Oct 07, 2023 N/A pytest <8,>=5 :pypi:`pytest-execution-timer` A timer for the phases of Pytest's execution. Dec 24, 2021 4 - Beta N/A :pypi:`pytest-exit-code` A pytest plugin that overrides the built-in exit codes to retain more information about the test results. May 06, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-exit-status` Enhance. Jan 25, 2025 N/A pytest>=8.0.0 :pypi:`pytest-expect` py.test plugin to store test expectations and mark tests based on them Apr 21, 2016 4 - Beta N/A :pypi:`pytest-expectdir` A pytest plugin to provide initial/expected directories, and check a test transforms the initial directory to the expected one Mar 19, 2023 5 - Production/Stable pytest (>=5.0) + :pypi:`pytest-expected` Record and play back your expectations Feb 26, 2025 N/A pytest :pypi:`pytest-expecter` Better testing with expecter and pytest. Sep 18, 2022 5 - Production/Stable N/A :pypi:`pytest-expectr` This plugin is used to expect multiple assert using pytest framework. Oct 05, 2018 N/A pytest (>=2.4.2) :pypi:`pytest-expect-test` A fixture to support expect tests in pytest Apr 10, 2023 4 - Beta pytest (>=3.5.0) :pypi:`pytest-experiments` A pytest plugin to help developers of research-oriented software projects keep track of the results of their numerical experiments. Dec 13, 2021 4 - Beta pytest (>=6.2.5,<7.0.0) :pypi:`pytest-explicit` A Pytest plugin to ignore certain marked tests by default Jun 15, 2021 5 - Production/Stable pytest - :pypi:`pytest-exploratory` Interactive console for pytest. Aug 18, 2023 N/A pytest (>=6.2) + :pypi:`pytest-exploratory` Interactive console for pytest. Sep 18, 2024 N/A pytest>=6.2 :pypi:`pytest-explorer` terminal ui for exploring and running tests Aug 01, 2023 N/A N/A :pypi:`pytest-ext` pytest plugin for automation test Mar 31, 2024 N/A pytest>=5.3 + :pypi:`pytest-extended-mock` a pytest extension for easy mock setup Mar 12, 2025 N/A pytest<9.0.0,>=8.3.5 :pypi:`pytest-extensions` A collection of helpers for pytest to ease testing Aug 17, 2022 4 - Beta pytest ; extra == 'testing' :pypi:`pytest-external-blockers` a special outcome for tests that are blocked for external reasons Oct 05, 2021 N/A pytest :pypi:`pytest_extra` Some helpers for writing tests with pytest. Aug 14, 2014 N/A N/A :pypi:`pytest-extra-durations` A pytest plugin to get durations on a per-function basis and per module basis. Apr 21, 2020 4 - Beta pytest (>=3.5.0) :pypi:`pytest-extra-markers` Additional pytest markers to dynamically enable/disable tests viia CLI flags Mar 05, 2023 4 - Beta pytest + :pypi:`pytest-f3ts` Pytest Plugin for communicating test results and information to a FixturFab Test Runner GUI Jul 15, 2025 N/A pytest<8.0.0,>=7.2.1 :pypi:`pytest-fabric` Provides test utilities to run fabric task tests by using docker containers Sep 12, 2018 5 - Production/Stable N/A - :pypi:`pytest-factor` A package to prevent Dependency Confusion attacks against Yandex. Feb 20, 2024 N/A N/A :pypi:`pytest-factory` Use factories for test setup with py.test Sep 06, 2020 3 - Alpha pytest (>4.3) - :pypi:`pytest-factoryboy` Factory Boy support for pytest. Mar 05, 2024 6 - Mature pytest (>=6.2) + :pypi:`pytest-factoryboy` Factory Boy support for pytest. Jul 01, 2025 6 - Mature pytest>=7.0 :pypi:`pytest-factoryboy-fixtures` Generates pytest fixtures that allow the use of type hinting Jun 25, 2020 N/A N/A :pypi:`pytest-factoryboy-state` Simple factoryboy random state management Mar 22, 2022 5 - Production/Stable pytest (>=5.0) :pypi:`pytest-failed-screen-record` Create a video of the screen when pytest fails Jan 05, 2023 4 - Beta pytest (>=7.1.2d,<8.0.0) :pypi:`pytest-failed-screenshot` Test case fails,take a screenshot,save it,attach it to the allure Apr 21, 2021 N/A N/A :pypi:`pytest-failed-to-verify` A pytest plugin that helps better distinguishing real test failures from setup flakiness. Aug 08, 2019 5 - Production/Stable pytest (>=4.1.0) :pypi:`pytest-fail-slow` Fail tests that take too long to run Jun 01, 2024 N/A pytest>=7.0 + :pypi:`pytest-failure-tracker` A pytest plugin for tracking test failures over multiple runs Jul 17, 2024 N/A pytest>=6.0.0 :pypi:`pytest-faker` Faker integration with the pytest framework. Dec 19, 2016 6 - Mature N/A :pypi:`pytest-falcon` Pytest helpers for Falcon. Sep 07, 2016 4 - Beta N/A - :pypi:`pytest-falcon-client` A package to prevent Dependency Confusion attacks against Yandex. Feb 21, 2024 N/A N/A :pypi:`pytest-fantasy` Pytest plugin for Flask Fantasy Framework Mar 14, 2019 N/A N/A :pypi:`pytest-fastapi` Dec 27, 2020 N/A N/A :pypi:`pytest-fastapi-deps` A fixture which allows easy replacement of fastapi dependencies for testing Jul 20, 2022 5 - Production/Stable pytest :pypi:`pytest-fastest` Use SCM and coverage to run only needed tests Oct 04, 2023 4 - Beta pytest (>=4.4) :pypi:`pytest-fast-first` Pytest plugin that runs fast tests first Jan 19, 2023 3 - Alpha pytest :pypi:`pytest-faulthandler` py.test plugin that activates the fault handler module for tests (dummy package) Jul 04, 2019 6 - Mature pytest (>=5.0) - :pypi:`pytest-fauna` A collection of helpful test fixtures for Fauna DB. May 30, 2024 N/A N/A + :pypi:`pytest-fauna` A collection of helpful test fixtures for Fauna DB. Jan 03, 2025 N/A N/A :pypi:`pytest-fauxfactory` Integration of fauxfactory into pytest. Dec 06, 2017 5 - Production/Stable pytest (>=3.2) :pypi:`pytest-figleaf` py.test figleaf coverage plugin Jan 18, 2010 5 - Production/Stable N/A :pypi:`pytest-file` Pytest File Mar 18, 2024 1 - Planning N/A @@ -542,24 +622,29 @@ This list contains 1487 plugins. :pypi:`pytest-file-watcher` Pytest-File-Watcher is a CLI tool that watches for changes in your code and runs pytest on the changed files. Mar 23, 2023 N/A pytest :pypi:`pytest-filter-case` run test cases filter by mark Nov 05, 2020 N/A N/A :pypi:`pytest-filter-subpackage` Pytest plugin for filtering based on sub-packages Mar 04, 2024 5 - Production/Stable pytest >=4.6 - :pypi:`pytest-find-dependencies` A pytest plugin to find dependencies between tests Mar 16, 2024 4 - Beta pytest >=4.3.0 + :pypi:`pytest-find-dependencies` A pytest plugin to find dependencies between tests Jul 16, 2025 5 - Production/Stable pytest>=6.2.4 :pypi:`pytest-finer-verdicts` A pytest plugin to treat non-assertion failures as test errors. Jun 18, 2020 N/A pytest (>=5.4.3) - :pypi:`pytest-firefox` pytest plugin to manipulate firefox Aug 08, 2017 3 - Alpha pytest (>=3.0.2) - :pypi:`pytest-fixture-classes` Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers Sep 02, 2023 5 - Production/Stable pytest + :pypi:`pytest-firefox` Feb 28, 2025 N/A N/A + :pypi:`pytest-fixturecheck` A pytest plugin to check fixture validity before test execution Jun 02, 2025 3 - Alpha pytest>=6.0.0 + :pypi:`pytest-fixture-classes` Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers Oct 12, 2025 5 - Production/Stable N/A + :pypi:`pytest-fixture-collect` A utility to collect pytest fixture file paths. Jul 25, 2025 N/A pytest; extra == "test" :pypi:`pytest-fixturecollection` A pytest plugin to collect tests based on fixtures being used by tests Feb 22, 2024 4 - Beta pytest >=3.5.0 - :pypi:`pytest-fixture-config` Fixture configuration utils for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-fixture-config` Fixture configuration utils for py.test Oct 17, 2024 5 - Production/Stable pytest + :pypi:`pytest-fixture-forms` A pytest plugin for creating fixtures that holds different forms between tests. Dec 06, 2024 N/A pytest<9.0.0,>=7.0.0 :pypi:`pytest-fixture-maker` Pytest plugin to load fixtures from YAML files Sep 21, 2021 N/A N/A :pypi:`pytest-fixture-marker` A pytest plugin to add markers based on fixtures used. Oct 11, 2020 5 - Production/Stable N/A - :pypi:`pytest-fixture-order` pytest plugin to control fixture evaluation order May 16, 2022 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-fixture-order` pytest plugin to control fixture evaluation order Oct 22, 2025 5 - Production/Stable pytest>=3.0 :pypi:`pytest-fixture-ref` Lets users reference fixtures without name matching magic. Nov 17, 2022 4 - Beta N/A :pypi:`pytest-fixture-remover` A LibCST codemod to remove pytest fixtures applied via the usefixtures decorator, as well as its parametrizations. Feb 14, 2024 5 - Production/Stable N/A :pypi:`pytest-fixture-rtttg` Warn or fail on fixture name clash Feb 23, 2022 N/A pytest (>=7.0.1,<8.0.0) :pypi:`pytest-fixtures` Common fixtures for pytest May 01, 2019 5 - Production/Stable N/A - :pypi:`pytest-fixture-tools` Plugin for pytest which provides tools for fixtures Aug 18, 2020 6 - Mature pytest + :pypi:`pytest-fixtures-fixtures` Handy fixtues to access your fixtures from your _pytest tests. Sep 14, 2025 4 - Beta pytest>=8.4.1 + :pypi:`pytest-fixture-tools` Plugin for pytest which provides tools for fixtures Apr 30, 2025 6 - Mature pytest :pypi:`pytest-fixture-typecheck` A pytest plugin to assert type annotations at runtime. Aug 24, 2021 N/A pytest - :pypi:`pytest-flake8` pytest plugin to check FLAKE8 requirements Mar 18, 2022 4 - Beta pytest (>=7.0) - :pypi:`pytest-flake8-path` A pytest fixture for testing flake8 plugins. Jul 10, 2023 5 - Production/Stable pytest + :pypi:`pytest-flake8` pytest plugin to check FLAKE8 requirements Nov 09, 2024 5 - Production/Stable pytest>=7.0 + :pypi:`pytest-flake8-path` A pytest fixture for testing flake8 plugins. Sep 09, 2025 5 - Production/Stable pytest :pypi:`pytest-flake8-v2` pytest plugin to check FLAKE8 requirements Mar 01, 2022 5 - Production/Stable pytest (>=7.0) + :pypi:`pytest-flake-detection` Continuously runs your tests to detect flaky tests Nov 29, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-flakefinder` Runs tests multiple times to expose flakiness. Oct 26, 2022 4 - Beta pytest (>=2.7.1) :pypi:`pytest-flakes` pytest plugin to check source code with pyflakes Dec 02, 2021 5 - Production/Stable pytest (>=5) :pypi:`pytest-flaptastic` Flaptastic py.test plugin Mar 17, 2019 N/A N/A @@ -568,69 +653,82 @@ This list contains 1487 plugins. :pypi:`pytest-flask-sqlalchemy` A pytest plugin for preserving test isolation in Flask-SQlAlchemy using database transactions. Apr 30, 2022 4 - Beta pytest (>=3.2.1) :pypi:`pytest-flask-sqlalchemy-transactions` Run tests in transactions using pytest, Flask, and SQLalchemy. Aug 02, 2018 4 - Beta pytest (>=3.2.1) :pypi:`pytest-flexreport` Apr 15, 2023 4 - Beta pytest - :pypi:`pytest-fluent` A pytest plugin in order to provide logs via fluentd Jun 05, 2024 4 - Beta pytest>=7.0.0 + :pypi:`pytest-fluent` A pytest plugin in order to provide logs via fluentd Aug 14, 2024 4 - Beta pytest>=7.0.0 :pypi:`pytest-fluentbit` A pytest plugin in order to provide logs via fluentbit Jun 16, 2023 4 - Beta pytest (>=7.0.0) - :pypi:`pytest-fly` pytest observer Apr 14, 2024 3 - Alpha pytest + :pypi:`pytest-fly` pytest runner and observer Jun 07, 2025 3 - Alpha pytest :pypi:`pytest-flyte` Pytest fixtures for simplifying Flyte integration testing May 03, 2021 N/A pytest + :pypi:`pytest-fmu-filter` A pytest plugin to filter fmus Jun 23, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-focus` A pytest plugin that alerts user of failed test cases with screen notifications May 04, 2019 4 - Beta pytest :pypi:`pytest-forbid` Mar 07, 2023 N/A pytest (>=7.2.2,<8.0.0) :pypi:`pytest-forcefail` py.test plugin to make the test failing regardless of pytest.mark.xfail May 15, 2018 4 - Beta N/A - :pypi:`pytest-forks` Fork helper for pytest Mar 05, 2024 N/A N/A :pypi:`pytest-forward-compatability` A name to avoid typosquating pytest-foward-compatibility Sep 06, 2020 N/A N/A :pypi:`pytest-forward-compatibility` A pytest plugin to shim pytest commandline options for fowards compatibility Sep 29, 2020 N/A N/A - :pypi:`pytest-frappe` Pytest Frappe Plugin - A set of pytest fixtures to test Frappe applications Oct 29, 2023 4 - Beta pytest>=7.0.0 - :pypi:`pytest-freezeblaster` Wrap tests with fixtures in freeze_time Jul 10, 2024 N/A pytest>=6.2.5 + :pypi:`pytest-frappe` Pytest Frappe Plugin - A set of pytest fixtures to test Frappe applications Jul 30, 2024 4 - Beta pytest>=7.0.0 + :pypi:`pytest-freethreaded` pytest plugin for running parallel tests Oct 03, 2024 5 - Production/Stable pytest + :pypi:`pytest-freezeblaster` Wrap tests with fixtures in freeze_time Oct 13, 2025 N/A pytest>=6.2.5 :pypi:`pytest-freezegun` Wrap tests with fixtures in freeze_time Jul 19, 2020 4 - Beta pytest (>=3.0.0) - :pypi:`pytest-freezer` Pytest plugin providing a fixture interface for spulec/freezegun Jun 21, 2023 N/A pytest >= 3.6 + :pypi:`pytest-freezer` Pytest plugin providing a fixture interface for spulec/freezegun Dec 12, 2024 N/A pytest>=3.6 :pypi:`pytest-freeze-reqs` Check if requirement files are frozen Apr 29, 2021 N/A N/A :pypi:`pytest-frozen-uuids` Deterministically frozen UUID's for your tests Apr 17, 2022 N/A pytest (>=3.0) :pypi:`pytest-func-cov` Pytest plugin for measuring function coverage Apr 15, 2021 3 - Alpha pytest (>=5) + :pypi:`pytest-funcnodes` Testing plugin for funcnodes Mar 19, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-funparam` An alternative way to parametrize test cases. Dec 02, 2021 4 - Beta pytest >=4.6.0 + :pypi:`pytest-fv` pytest extensions to support running functional-verification jobs Jun 06, 2025 N/A pytest :pypi:`pytest-fxa` pytest plugin for Firefox Accounts Aug 28, 2018 5 - Production/Stable N/A + :pypi:`pytest-fxa-mte` pytest plugin for Firefox Accounts Oct 02, 2024 5 - Production/Stable N/A :pypi:`pytest-fxtest` Oct 27, 2020 N/A N/A - :pypi:`pytest-fzf` fzf-based test selector for pytest Jul 03, 2024 4 - Beta pytest>=6.0.0 + :pypi:`pytest-fzf` fzf-based test selector for pytest Jan 06, 2025 4 - Beta pytest>=6.0.0 :pypi:`pytest_gae` pytest plugin for apps written with Google's AppEngine Aug 03, 2016 3 - Alpha N/A - :pypi:`pytest-gather-fixtures` set up asynchronous pytest fixtures concurrently Apr 12, 2022 N/A pytest (>=6.0.0) + :pypi:`pytest-gak` A Pytest plugin and command line tool for interactive testing with Pytest Apr 10, 2025 N/A N/A + :pypi:`pytest-gather-fixtures` set up asynchronous pytest fixtures concurrently Aug 18, 2024 N/A pytest>=7.0.0 :pypi:`pytest-gc` The garbage collector plugin for py.test Feb 01, 2018 N/A N/A :pypi:`pytest-gcov` Uses gcov to measure test coverage of a C library Feb 01, 2018 3 - Alpha N/A - :pypi:`pytest-gcs` GCS fixtures and fixture factories for Pytest. Mar 01, 2024 5 - Production/Stable pytest >=6.2 - :pypi:`pytest-gee` The Python plugin for your GEE based packages. Jun 30, 2024 3 - Alpha pytest + :pypi:`pytest-gcs` GCS fixtures and fixture factories for Pytest. Jan 24, 2025 5 - Production/Stable pytest>=6.2 + :pypi:`pytest-gee` The Python plugin for your GEE based packages. Oct 16, 2025 3 - Alpha pytest :pypi:`pytest-gevent` Ensure that gevent is properly patched when invoking pytest Feb 25, 2020 N/A pytest :pypi:`pytest-gherkin` A flexible framework for executing BDD gherkin tests Jul 27, 2019 3 - Alpha pytest (>=5.0.0) :pypi:`pytest-gh-log-group` pytest plugin for gh actions Jan 11, 2022 3 - Alpha pytest :pypi:`pytest-ghostinspector` For finding/executing Ghost Inspector tests May 17, 2016 3 - Alpha N/A - :pypi:`pytest-girder` A set of pytest fixtures for testing Girder applications. Jul 08, 2024 N/A pytest>=3.6 - :pypi:`pytest-git` Git repository fixture for py.test May 28, 2019 5 - Production/Stable pytest - :pypi:`pytest-gitconfig` Provide a gitconfig sandbox for testing Oct 15, 2023 4 - Beta pytest>=7.1.2 + :pypi:`pytest-girder` A set of pytest fixtures for testing Girder applications. Sep 30, 2025 N/A pytest>=3.6 + :pypi:`pytest-git` Git repository fixture for py.test Oct 17, 2024 5 - Production/Stable pytest + :pypi:`pytest-gitconfig` Provide a Git config sandbox for testing Oct 12, 2025 4 - Beta pytest>=7.1.2 :pypi:`pytest-gitcov` Pytest plugin for reporting on coverage of the last git commit. Jan 11, 2020 2 - Pre-Alpha N/A :pypi:`pytest-git-diff` Pytest plugin that allows the user to select the tests affected by a range of git commits Apr 02, 2024 N/A N/A :pypi:`pytest-git-fixtures` Pytest fixtures for testing with git. Mar 11, 2021 4 - Beta pytest :pypi:`pytest-github` Plugin for py.test that associates tests with github issues using a marker. Mar 07, 2019 5 - Production/Stable N/A - :pypi:`pytest-github-actions-annotate-failures` pytest plugin to annotate failed tests with a workflow command for GitHub Actions May 04, 2023 5 - Production/Stable pytest (>=4.0.0) + :pypi:`pytest-github-actions-annotate-failures` pytest plugin to annotate failed tests with a workflow command for GitHub Actions Jan 17, 2025 5 - Production/Stable pytest>=6.0.0 :pypi:`pytest-github-report` Generate a GitHub report using pytest in GitHub Workflows Jun 03, 2022 4 - Beta N/A :pypi:`pytest-gitignore` py.test plugin to ignore the same files as git Jul 17, 2015 4 - Beta N/A + :pypi:`pytest-gitlab` Pytest Plugin for Gitlab Oct 16, 2024 N/A N/A :pypi:`pytest-gitlabci-parallelized` Parallelize pytest across GitLab CI workers. Mar 08, 2023 N/A N/A - :pypi:`pytest-gitlab-code-quality` Collects warnings while testing and generates a GitLab Code Quality Report. Apr 03, 2024 N/A pytest>=8.1.1 + :pypi:`pytest-gitlab-code-quality` Collects warnings while testing and generates a GitLab Code Quality Report. Sep 09, 2024 N/A pytest>=8.1.1 :pypi:`pytest-gitlab-fold` Folds output sections in GitLab CI build log Dec 31, 2023 4 - Beta pytest >=2.6.0 + :pypi:`pytest-gitscope` A pragmatic pytest plugin that runs only the tests that matter, and ship faster Sep 24, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-git-selector` Utility to select tests that have had its dependencies modified (as identified by git diff) Nov 17, 2022 N/A N/A - :pypi:`pytest-glamor-allure` Extends allure-pytest functionality Apr 30, 2024 4 - Beta pytest<=8.2.0 + :pypi:`pytest-glamor-allure` Extends allure-pytest functionality Jul 20, 2025 4 - Beta pytest<=8.4.1 :pypi:`pytest-gnupg-fixtures` Pytest fixtures for testing with gnupg. Mar 04, 2021 4 - Beta pytest :pypi:`pytest-golden` Plugin for pytest that offloads expected outputs to data files Jul 18, 2022 N/A pytest (>=6.1.2) :pypi:`pytest-goldie` A plugin to support golden tests with pytest. May 23, 2023 4 - Beta pytest (>=3.5.0) :pypi:`pytest-google-chat` Notify google chat channel for test results Mar 27, 2022 4 - Beta pytest + :pypi:`pytest-google-cloud-storage` Pytest custom features, e.g. fixtures and various tests. Aimed to emulate Google Cloud Storage service Sep 11, 2025 N/A pytest>=8.0.0 + :pypi:`pytest-grader` Pytest extension for scoring programming assignments. Aug 25, 2025 N/A pytest>=8 + :pypi:`pytest-gradescope` A pytest plugin for Gradescope integration Apr 29, 2025 N/A N/A :pypi:`pytest-graphql-schema` Get graphql schema as fixture for pytest Oct 18, 2019 N/A N/A :pypi:`pytest-greendots` Green progress dots Feb 08, 2014 3 - Alpha N/A + :pypi:`pytest-greener` Pytest plugin for Greener Oct 18, 2025 N/A pytest<9.0.0,>=8.3.3 + :pypi:`pytest-greet` Oct 21, 2025 N/A N/A :pypi:`pytest-group-by-class` A Pytest plugin for running a subset of your tests by splitting them in to groups of classes. Jun 27, 2023 5 - Production/Stable pytest (>=2.5) :pypi:`pytest-growl` Growl notifications for pytest results. Jan 13, 2014 5 - Production/Stable N/A :pypi:`pytest-grpc` pytest plugin for grpc May 01, 2020 N/A pytest (>=3.6.0) - :pypi:`pytest-grunnur` Py.Test plugin for Grunnur-based packages. Feb 05, 2023 N/A N/A + :pypi:`pytest-grpc-aio` pytest plugin for grpc.aio Oct 28, 2025 N/A pytest>=3.6.0 + :pypi:`pytest-grunnur` Py.Test plugin for Grunnur-based packages. Jul 26, 2024 N/A pytest>=6 :pypi:`pytest_gui_status` Show pytest status in gui Jan 23, 2016 N/A pytest :pypi:`pytest-hammertime` Display "🔨 " instead of "." for passed pytest tests. Jul 28, 2018 N/A pytest :pypi:`pytest-hardware-test-report` A simple plugin to use with pytest Apr 01, 2024 4 - Beta pytest<9.0.0,>=8.0.0 :pypi:`pytest-harmony` Chain tests and data with pytest Jan 17, 2023 N/A pytest (>=7.2.1,<8.0.0) :pypi:`pytest-harvest` Store data created during your pytest tests execution, and retrieve it at the end of the session, e.g. for applicative benchmarking purposes. Mar 16, 2024 5 - Production/Stable N/A - :pypi:`pytest-helm-charts` A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. Feb 07, 2024 4 - Beta pytest (>=8.0.0,<9.0.0) - :pypi:`pytest-helm-templates` Pytest fixtures for unit testing the output of helm templates May 08, 2024 N/A pytest~=7.4.0; extra == "dev" + :pypi:`pytest-helm-charts` A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. Oct 31, 2024 4 - Beta pytest<9.0.0,>=8.0.0 + :pypi:`pytest-helm-templates` Pytest fixtures for unit testing the output of helm templates Aug 07, 2024 N/A pytest~=7.4.0; extra == "dev" :pypi:`pytest-helper` Functions to help in using the pytest testing framework May 31, 2019 5 - Production/Stable N/A :pypi:`pytest-helpers` pytest helpers May 17, 2020 N/A pytest :pypi:`pytest-helpers-namespace` Pytest Helpers Namespace Plugin Dec 29, 2021 5 - Production/Stable pytest (>=6.0.0) @@ -640,266 +738,328 @@ This list contains 1487 plugins. :pypi:`pytest-historic` Custom report to display pytest historical execution records Apr 08, 2020 N/A pytest :pypi:`pytest-historic-hook` Custom listener to store execution results into MYSQL DB, which is used for pytest-historic report Apr 08, 2020 N/A pytest :pypi:`pytest-history` Pytest plugin to keep a history of your pytest runs Jan 14, 2024 N/A pytest (>=7.4.3,<8.0.0) - :pypi:`pytest-home` Home directory fixtures Oct 09, 2023 5 - Production/Stable pytest + :pypi:`pytest-home` Home directory fixtures Jul 28, 2024 5 - Production/Stable pytest :pypi:`pytest-homeassistant` A pytest plugin for use with homeassistant custom components. Aug 12, 2020 4 - Beta N/A - :pypi:`pytest-homeassistant-custom-component` Experimental package to automatically extract test plugins for Home Assistant custom components Jul 11, 2024 3 - Alpha pytest==8.2.0 + :pypi:`pytest-homeassistant-custom-component` Experimental package to automatically extract test plugins for Home Assistant custom components Oct 31, 2025 3 - Alpha pytest==8.4.2 :pypi:`pytest-honey` A simple plugin to use with pytest Jan 07, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-honors` Report on tests that honor constraints, and guard against regressions Mar 06, 2020 4 - Beta N/A - :pypi:`pytest-hot-reloading` Apr 18, 2024 N/A N/A + :pypi:`pytest-hot-reloading` Sep 23, 2024 N/A N/A :pypi:`pytest-hot-test` A plugin that tracks test changes Dec 10, 2022 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-houdini` pytest plugin for testing code in Houdini. Jul 05, 2024 N/A pytest + :pypi:`pytest-houdini` pytest plugin for testing code in Houdini. Jul 15, 2024 N/A pytest :pypi:`pytest-hoverfly` Simplify working with Hoverfly from pytest Jan 30, 2023 N/A pytest (>=5.0) :pypi:`pytest-hoverfly-wrapper` Integrates the Hoverfly HTTP proxy into Pytest Feb 27, 2023 5 - Production/Stable pytest (>=3.7.0) :pypi:`pytest-hpfeeds` Helpers for testing hpfeeds in your python project Feb 28, 2023 4 - Beta pytest (>=6.2.4,<7.0.0) :pypi:`pytest-html` pytest plugin for generating HTML reports Nov 07, 2023 5 - Production/Stable pytest>=7.0.0 - :pypi:`pytest-html-cn` pytest plugin for generating HTML reports Aug 01, 2023 5 - Production/Stable N/A + :pypi:`pytest-html5` the best report for pytest Oct 11, 2025 N/A N/A + :pypi:`pytest-html-cn` pytest plugin for generating HTML reports Aug 19, 2024 5 - Production/Stable pytest!=6.0.0,>=5.0 :pypi:`pytest-html-lee` optimized pytest plugin for generating HTML reports Jun 30, 2020 5 - Production/Stable pytest (>=5.0) :pypi:`pytest-html-merger` Pytest HTML reports merging utility Jul 12, 2024 N/A N/A + :pypi:`pytest-html-nova-act` A Pytest Plugin for Amazon Nova Act Python SDK. Sep 05, 2025 N/A N/A :pypi:`pytest-html-object-storage` Pytest report plugin for send HTML report on object-storage Jan 17, 2024 5 - Production/Stable N/A + :pypi:`pytest-html-plus` Get started with rich pytest reports in under 3 seconds. Just install the plugin — no setup required. The simplest, fastest reporter for pytest. Oct 30, 2025 N/A N/A :pypi:`pytest-html-profiling` Pytest plugin for generating HTML reports with per-test profiling and optionally call graph visualizations. Based on pytest-html by Dave Hunt. Feb 11, 2020 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-html-report` Enhanced HTML reporting for pytest with categories, specifications, and detailed logging Jun 24, 2025 4 - Beta pytest>=6.0 :pypi:`pytest-html-reporter` Generates a static html report based on pytest framework Feb 13, 2022 N/A N/A :pypi:`pytest-html-report-merger` May 22, 2024 N/A N/A :pypi:`pytest-html-thread` pytest plugin for generating HTML reports Dec 29, 2020 5 - Production/Stable N/A - :pypi:`pytest-http` Fixture "http" for http requests Dec 05, 2019 N/A N/A - :pypi:`pytest-httpbin` Easily test your HTTP library against a local copy of httpbin May 08, 2023 5 - Production/Stable pytest ; extra == 'test' - :pypi:`pytest-httpdbg` A pytest plugin to record HTTP(S) requests with stack trace Jan 10, 2024 3 - Alpha pytest >=7.0.0 + :pypi:`pytest-htmlx` Custom HTML report plugin for Pytest with charts and tables Sep 09, 2025 4 - Beta pytest + :pypi:`pytest-http` Fixture "http" for http requests Aug 22, 2024 N/A pytest + :pypi:`pytest-httpbin` Easily test your HTTP library against a local copy of httpbin Sep 18, 2024 5 - Production/Stable pytest; extra == "test" + :pypi:`pytest-httpchain` pytest plugin for HTTP testing using JSON files Aug 16, 2025 5 - Production/Stable N/A + :pypi:`pytest-httpchain-jsonref` JSON reference ($ref) support for pytest-httpchain Aug 16, 2025 N/A N/A + :pypi:`pytest-httpchain-mcp` MCP server for pytest-httpchain Aug 16, 2025 N/A N/A + :pypi:`pytest-httpchain-models` Pydantic models for pytest-httpchain Aug 16, 2025 N/A N/A + :pypi:`pytest-httpchain-templates` Templating support for pytest-httpchain Aug 16, 2025 N/A N/A + :pypi:`pytest-httpchain-userfunc` User functions support for pytest-httpchain Aug 16, 2025 N/A N/A + :pypi:`pytest-httpdbg` A pytest plugin to record HTTP(S) requests with stack trace. Oct 26, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-http-mocker` Pytest plugin for http mocking (via https://github.com/vilus/mocker) Oct 20, 2019 N/A N/A :pypi:`pytest-httpretty` A thin wrapper of HTTPretty for pytest Feb 16, 2014 3 - Alpha N/A - :pypi:`pytest_httpserver` pytest-httpserver is a httpserver for pytest Feb 24, 2024 3 - Alpha N/A - :pypi:`pytest-httptesting` http_testing framework on top of pytest May 08, 2024 N/A pytest<9.0.0,>=8.2.0 - :pypi:`pytest-httpx` Send responses to httpx. Feb 21, 2024 5 - Production/Stable pytest <9,>=7 + :pypi:`pytest_httpserver` pytest-httpserver is a httpserver for pytest Apr 10, 2025 3 - Alpha N/A + :pypi:`pytest-httptesting` http_testing framework on top of pytest Dec 19, 2024 N/A pytest>=8.2.0 + :pypi:`pytest-httpx` Send responses to httpx. Nov 28, 2024 5 - Production/Stable pytest==8.* :pypi:`pytest-httpx-blockage` Disable httpx requests during a test run Feb 16, 2023 N/A pytest (>=7.2.1) :pypi:`pytest-httpx-recorder` Recorder feature based on pytest_httpx, like recorder feature in responses. Jan 04, 2024 5 - Production/Stable pytest :pypi:`pytest-hue` Visualise PyTest status via your Phillips Hue lights May 09, 2019 N/A N/A :pypi:`pytest-hylang` Pytest plugin to allow running tests written in hylang Mar 28, 2021 N/A pytest :pypi:`pytest-hypo-25` help hypo module for pytest Jan 12, 2020 3 - Alpha N/A - :pypi:`pytest-iam` A fully functional OAUTH2 / OpenID Connect (OIDC) server to be used in your testsuite Apr 22, 2024 3 - Alpha pytest>=7.0.0 - :pypi:`pytest-ibutsu` A plugin to sent pytest results to an Ibutsu server Aug 05, 2022 4 - Beta pytest>=7.1 + :pypi:`pytest-iam` A fully functional OAUTH2 / OpenID Connect (OIDC) / SCIM server to be used in your testsuite Jul 25, 2025 4 - Beta pytest>=7.0.0 + :pypi:`pytest-ibutsu` A plugin to sent pytest results to an Ibutsu server Oct 21, 2025 4 - Beta pytest :pypi:`pytest-icdiff` use icdiff for better error messages in pytest assertions Dec 05, 2023 4 - Beta pytest :pypi:`pytest-idapro` A pytest plugin for idapython. Allows a pytest setup to run tests outside and inside IDA in an automated manner by runnig pytest inside IDA and by mocking idapython api Nov 03, 2018 N/A N/A :pypi:`pytest-idem` A pytest plugin to help with testing idem projects Dec 13, 2023 5 - Production/Stable N/A :pypi:`pytest-idempotent` Pytest plugin for testing function idempotence. Jul 25, 2022 N/A N/A :pypi:`pytest-ignore-flaky` ignore failures from flaky tests (pytest plugin) Apr 20, 2024 5 - Production/Stable pytest>=6.0 - :pypi:`pytest-ignore-test-results` A pytest plugin to ignore test results. Aug 17, 2023 2 - Pre-Alpha pytest>=7.0 - :pypi:`pytest-image-diff` Mar 09, 2023 3 - Alpha pytest - :pypi:`pytest-image-snapshot` A pytest plugin for image snapshot management and comparison. Jul 01, 2024 4 - Beta pytest>=3.5.0 + :pypi:`pytest-ignore-test-results` A pytest plugin to ignore test results. Feb 03, 2025 5 - Production/Stable pytest>=7.0 + :pypi:`pytest-image-diff` Dec 31, 2024 3 - Alpha pytest + :pypi:`pytest-image-snapshot` A pytest plugin for image snapshot management and comparison. Jul 16, 2025 4 - Beta pytest>=3.5.0 + :pypi:`pytest-impacted` A pytest plugin that selectively runs tests impacted by codechanges via git introspection, ASL parsing, and dependency graph analysis. Sep 11, 2025 4 - Beta pytest>=8.0.0 + :pypi:`pytest-import-check` pytest plugin to check whether Python modules can be imported Jul 19, 2024 3 - Alpha pytest>=8.1 :pypi:`pytest-incremental` an incremental test runner (pytest plugin) Apr 24, 2021 5 - Production/Stable N/A :pypi:`pytest-infinity` Jun 09, 2024 N/A pytest<9.0.0,>=8.0.0 + :pypi:`pytest-influx` Pytest plugin for managing your influx instance between test runs Oct 16, 2024 N/A pytest<9.0.0,>=8.3.3 :pypi:`pytest-influxdb` Plugin for influxdb and pytest integration. Apr 20, 2021 N/A N/A :pypi:`pytest-info-collector` pytest plugin to collect information from tests May 26, 2019 3 - Alpha N/A :pypi:`pytest-info-plugin` Get executed interface information in pytest interface automation framework Sep 14, 2023 N/A N/A :pypi:`pytest-informative-node` display more node ininformation. Apr 25, 2019 4 - Beta N/A + :pypi:`pytest-infrahouse` A set of fixtures to use with pytest Oct 29, 2025 4 - Beta pytest~=8.3 :pypi:`pytest-infrastructure` pytest stack validation prior to testing executing Apr 12, 2020 4 - Beta N/A :pypi:`pytest-ini` Reuse pytest.ini to store env variables Apr 26, 2022 N/A N/A :pypi:`pytest-initry` Plugin for sending automation test data from Pytest to the initry Apr 30, 2024 N/A pytest<9.0.0,>=8.1.1 - :pypi:`pytest-inline` A pytest plugin for writing inline tests. Oct 19, 2023 4 - Beta pytest >=7.0.0 - :pypi:`pytest-inmanta` A py.test plugin providing fixtures to simplify inmanta modules testing. Jul 05, 2024 5 - Production/Stable pytest - :pypi:`pytest-inmanta-extensions` Inmanta tests package Jul 05, 2024 5 - Production/Stable N/A - :pypi:`pytest-inmanta-lsm` Common fixtures for inmanta LSM related modules Jul 06, 2024 5 - Production/Stable N/A - :pypi:`pytest-inmanta-yang` Common fixtures used in inmanta yang related modules Feb 22, 2024 4 - Beta pytest + :pypi:`pytest-inline` A pytest plugin for writing inline tests Oct 24, 2024 4 - Beta pytest<9.0,>=7.0 + :pypi:`pytest-inmanta` A py.test plugin providing fixtures to simplify inmanta modules testing. Apr 09, 2025 5 - Production/Stable pytest + :pypi:`pytest-inmanta-extensions` Inmanta tests package Jul 04, 2025 5 - Production/Stable N/A + :pypi:`pytest-inmanta-lsm` Common fixtures for inmanta LSM related modules Aug 26, 2025 5 - Production/Stable N/A + :pypi:`pytest-inmanta-srlinux` Pytest library to facilitate end to end testing of inmanta projects Apr 22, 2025 3 - Alpha N/A + :pypi:`pytest-inmanta-yang` Common fixtures used in inmanta yang related modules Oct 28, 2025 4 - Beta pytest :pypi:`pytest-Inomaly` A simple image diff plugin for pytest Feb 13, 2018 4 - Beta N/A - :pypi:`pytest-in-robotframework` The extension enables easy execution of pytest tests within the Robot Framework environment. Mar 02, 2024 N/A pytest + :pypi:`pytest-in-robotframework` The extension enables easy execution of pytest tests within the Robot Framework environment. Nov 23, 2024 N/A pytest :pypi:`pytest-insper` Pytest plugin for courses at Insper Mar 21, 2024 N/A pytest :pypi:`pytest-insta` A practical snapshot testing plugin for pytest Feb 19, 2024 N/A pytest (>=7.2.0,<9.0.0) :pypi:`pytest-instafail` pytest plugin to show failures instantly Mar 31, 2023 4 - Beta pytest (>=5) :pypi:`pytest-instrument` pytest plugin to instrument tests Apr 05, 2020 5 - Production/Stable pytest (>=5.1.0) + :pypi:`pytest-insubprocess` A pytest plugin to execute test cases in a subprocess Jul 01, 2025 4 - Beta pytest>=7.4 :pypi:`pytest-integration` Organizing pytests by integration or not Nov 17, 2022 N/A N/A :pypi:`pytest-integration-mark` Automatic integration test marking and excluding plugin for pytest May 22, 2023 N/A pytest (>=5.2) :pypi:`pytest-interactive` A pytest plugin for console based interactive test selection just after the collection phase Nov 30, 2017 3 - Alpha N/A :pypi:`pytest-intercept-remote` Pytest plugin for intercepting outgoing connection requests during pytest run. May 24, 2021 4 - Beta pytest (>=4.6) - :pypi:`pytest-interface-tester` Pytest plugin for checking charm relation interface protocol compliance. Feb 09, 2024 4 - Beta pytest - :pypi:`pytest-invenio` Pytest fixtures for Invenio. Jun 27, 2024 5 - Production/Stable pytest<7.2.0,>=6 + :pypi:`pytest-interface-tester` Pytest plugin for checking charm relation interface protocol compliance. Oct 09, 2025 4 - Beta pytest + :pypi:`pytest-invenio` Pytest fixtures for Invenio. Jul 09, 2025 5 - Production/Stable pytest<9.0.0,>=6 :pypi:`pytest-involve` Run tests covering a specific file or changeset Feb 02, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-iovis` A Pytest plugin to enable Jupyter Notebook testing with Papermill Nov 06, 2024 4 - Beta pytest>=7.1.0 :pypi:`pytest-ipdb` A py.test plug-in to enable drop to ipdb debugger on test failure. Mar 20, 2013 2 - Pre-Alpha N/A :pypi:`pytest-ipynb` THIS PROJECT IS ABANDONED Jan 29, 2019 3 - Alpha N/A - :pypi:`pytest-ipywidgets` Jul 11, 2024 N/A pytest - :pypi:`pytest-isolate` Feb 20, 2023 4 - Beta pytest + :pypi:`pytest-ipynb2` Pytest plugin to run tests in Jupyter Notebooks Mar 09, 2025 N/A pytest + :pypi:`pytest-ipywidgets` Oct 24, 2025 N/A pytest + :pypi:`pytest-isolate` Run pytest tests in isolated subprocesses Sep 08, 2025 4 - Beta pytest + :pypi:`pytest-isolate-mpi` pytest-isolate-mpi allows for MPI-parallel tests being executed in a segfault and MPI_Abort safe manner Feb 24, 2025 4 - Beta pytest>=5 :pypi:`pytest-isort` py.test plugin to check import ordering using isort Mar 05, 2024 5 - Production/Stable pytest (>=5.0) :pypi:`pytest-it` Pytest plugin to display test reports as a plaintext spec, inspired by Rspec: https://github.com/mattduck/pytest-it. Jan 29, 2024 4 - Beta N/A + :pypi:`pytest-item-dict` Get a hierarchical dict of session.items Nov 14, 2024 4 - Beta pytest>=8.3.0 :pypi:`pytest-iterassert` Nicer list and iterable assertion messages for pytest May 11, 2020 3 - Alpha N/A + :pypi:`pytest-iteration` Add iteration mark for tests Aug 22, 2024 N/A pytest :pypi:`pytest-iters` A contextmanager pytest fixture for handling multiple mock iters May 24, 2022 N/A N/A :pypi:`pytest_jar_yuan` A allure and pytest used package Dec 12, 2022 N/A N/A :pypi:`pytest-jasmine` Run jasmine tests from your pytest test suite Nov 04, 2017 1 - Planning N/A :pypi:`pytest-jelastic` Pytest plugin defining the necessary command-line options to pass to pytests testing a Jelastic environment. Nov 16, 2022 N/A pytest (>=7.2.0,<8.0.0) :pypi:`pytest-jest` A custom jest-pytest oriented Pytest reporter May 22, 2018 4 - Beta pytest (>=3.3.2) :pypi:`pytest-jinja` A plugin to generate customizable jinja-based HTML reports in pytest Oct 04, 2022 3 - Alpha pytest (>=6.2.5,<7.0.0) - :pypi:`pytest-jira` py.test JIRA integration plugin, using markers Apr 30, 2024 3 - Alpha N/A + :pypi:`pytest-jira` py.test JIRA integration plugin, using markers Apr 15, 2025 3 - Alpha N/A :pypi:`pytest-jira-xfail` Plugin skips (xfail) tests if unresolved Jira issue(s) linked Jul 09, 2024 N/A pytest>=7.2.0 - :pypi:`pytest-jira-xray` pytest plugin to integrate tests with JIRA XRAY Mar 27, 2024 4 - Beta pytest>=6.2.4 + :pypi:`pytest-jira-xray` pytest plugin to integrate tests with JIRA XRAY Oct 11, 2025 4 - Beta pytest>=6.2.4 :pypi:`pytest-job-selection` A pytest plugin for load balancing test suites Jan 30, 2023 4 - Beta pytest (>=3.5.0) :pypi:`pytest-jobserver` Limit parallel tests with posix jobserver. May 15, 2019 5 - Production/Stable pytest :pypi:`pytest-joke` Test failures are better served with humor. Oct 08, 2019 4 - Beta pytest (>=4.2.1) :pypi:`pytest-json` Generate JSON test reports Jan 18, 2016 4 - Beta N/A - :pypi:`pytest-json-ctrf` Pytest plugin to generate json report in CTRF (Common Test Report Format) Jun 15, 2024 N/A pytest>6.0.0 + :pypi:`pytest-json-ctrf` Pytest plugin to generate json report in CTRF (Common Test Report Format) Oct 10, 2024 N/A pytest>6.0.0 :pypi:`pytest-json-fixtures` JSON output for the --fixtures flag Mar 14, 2023 4 - Beta N/A :pypi:`pytest-jsonlint` UNKNOWN Aug 04, 2016 N/A N/A :pypi:`pytest-json-report` A pytest plugin to report test results as JSON files Mar 15, 2022 4 - Beta pytest (>=3.8.0) - :pypi:`pytest-json-report-wip` A pytest plugin to report test results as JSON files Oct 28, 2023 4 - Beta pytest >=3.8.0 - :pypi:`pytest-jsonschema` A pytest plugin to perform JSONSchema validations Mar 27, 2024 4 - Beta pytest>=6.2.0 - :pypi:`pytest-jtr` pytest plugin supporting json test report output Jun 04, 2024 N/A pytest<8.0.0,>=7.1.2 - :pypi:`pytest-jupyter` A pytest plugin for testing Jupyter libraries and extensions. Apr 04, 2024 4 - Beta pytest>=7.0 + :pypi:`pytest-json-report-wip` A pytest plugin to report test results as JSON files Jul 23, 2025 4 - Beta pytest >=3.8.0 + :pypi:`pytest-jsonschema` A pytest plugin to perform JSONSchema validations Apr 20, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-jsonschema-snapshot` Pytest plugin for automatic JSON Schema generation and validation from examples Sep 13, 2025 N/A pytest + :pypi:`pytest-jtr` pytest plugin supporting json test report output Jul 21, 2024 N/A pytest<8.0.0,>=7.1.2 + :pypi:`pytest-jubilant` Add your description here Jul 28, 2025 N/A pytest>=8.3.5 + :pypi:`pytest-junit-xray-xml` Export test results in an augmented JUnit format for usage with Xray () Jan 01, 2025 4 - Beta pytest + :pypi:`pytest-jupyter` A pytest plugin for testing Jupyter libraries and extensions. Oct 16, 2025 4 - Beta pytest>=7.0 :pypi:`pytest-jupyterhub` A reusable JupyterHub pytest plugin Apr 25, 2023 5 - Production/Stable pytest - :pypi:`pytest-kafka` Zookeeper, Kafka server, and Kafka consumer fixtures for Pytest Jun 14, 2023 N/A pytest + :pypi:`pytest-jux` A pytest plugin for signing and publishing JUnit XML test reports to the Jux REST API Oct 24, 2025 3 - Alpha pytest>=7.4 + :pypi:`pytest-k8s` Kubernetes-based testing for pytest Jul 07, 2025 N/A pytest>=8.4.1 + :pypi:`pytest-kafka` Zookeeper, Kafka server, and Kafka consumer fixtures for Pytest Aug 14, 2024 N/A pytest :pypi:`pytest-kafkavents` A plugin to send pytest events to Kafka Sep 08, 2021 4 - Beta pytest + :pypi:`pytest-kairos` Pytest plugin with random number generation, reproducibility, and test repetition Aug 08, 2024 5 - Production/Stable pytest>=5.0.0 :pypi:`pytest-kasima` Display horizontal lines above and below the captured standard output for easy viewing. Jan 26, 2023 5 - Production/Stable pytest (>=7.2.1,<8.0.0) :pypi:`pytest-keep-together` Pytest plugin to customize test ordering by running all 'related' tests together Dec 07, 2022 5 - Production/Stable pytest :pypi:`pytest-kexi` Apr 29, 2022 N/A pytest (>=7.1.2,<8.0.0) - :pypi:`pytest-keyring` A Pytest plugin to access the system's keyring to provide credentials for tests Oct 01, 2023 N/A pytest (>=7.1) + :pypi:`pytest-keyring` A Pytest plugin to access the system's keyring to provide credentials for tests Dec 08, 2024 N/A pytest>=8.0.2 :pypi:`pytest-kind` Kubernetes test support with KIND for pytest Nov 30, 2022 5 - Production/Stable N/A :pypi:`pytest-kivy` Kivy GUI tests fixtures using pytest Jul 06, 2021 4 - Beta pytest (>=3.6) :pypi:`pytest-knows` A pytest plugin that can automaticly skip test case based on dependence info calculated by trace Aug 22, 2014 N/A N/A :pypi:`pytest-konira` Run Konira DSL tests with py.test Oct 09, 2011 N/A N/A - :pypi:`pytest-kookit` Your simple but kooky integration testing with pytest May 16, 2024 N/A N/A + :pypi:`pytest-kookit` Your simple but kooky integration testing with pytest Sep 10, 2024 N/A N/A :pypi:`pytest-koopmans` A plugin for testing the koopmans package Nov 21, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-krtech-common` pytest krtech common library Nov 28, 2016 4 - Beta N/A - :pypi:`pytest-kubernetes` Sep 14, 2023 N/A pytest (>=7.2.1,<8.0.0) + :pypi:`pytest-kubernetes` Oct 23, 2025 N/A pytest<9.0.0,>=8.3.0 + :pypi:`pytest_kustomize` Parse and validate kustomize output Oct 02, 2025 N/A N/A :pypi:`pytest-kuunda` pytest plugin to help with test data setup for PySpark tests Feb 25, 2024 4 - Beta pytest >=6.2.0 :pypi:`pytest-kwparametrize` Alternate syntax for @pytest.mark.parametrize with test cases as dictionaries and default value fallbacks Jan 22, 2021 N/A pytest (>=6) :pypi:`pytest-lambda` Define pytest fixtures with lambda functions. May 27, 2024 5 - Production/Stable pytest<9,>=3.6 :pypi:`pytest-lamp` Jan 06, 2017 3 - Alpha N/A :pypi:`pytest-langchain` Pytest-style test runner for langchain agents Feb 26, 2023 N/A pytest :pypi:`pytest-lark` Create fancy and clear HTML test reports. Nov 05, 2023 N/A N/A + :pypi:`pytest-latin-hypercube` Implementation of Latin Hypercube Sampling for pytest. Jun 26, 2025 N/A pytest :pypi:`pytest-launchable` Launchable Pytest Plugin Apr 05, 2023 N/A pytest (>=4.2.0) :pypi:`pytest-layab` Pytest fixtures for layab. Oct 05, 2020 5 - Production/Stable N/A :pypi:`pytest-lazy-fixture` It helps to use fixtures in pytest.mark.parametrize Feb 01, 2020 4 - Beta pytest (>=3.2.5) - :pypi:`pytest-lazy-fixtures` Allows you to use fixtures in @pytest.mark.parametrize. Mar 16, 2024 N/A pytest (>=7) + :pypi:`pytest-lazy-fixtures` Allows you to use fixtures in @pytest.mark.parametrize. Sep 16, 2025 N/A pytest>=7 :pypi:`pytest-ldap` python-ldap fixtures for pytest Aug 18, 2020 N/A pytest :pypi:`pytest-leak-finder` Find the test that's leaking before the one that fails Feb 15, 2023 4 - Beta pytest (>=3.5.0) :pypi:`pytest-leaks` A pytest plugin to trace resource leaks. Nov 27, 2019 1 - Planning N/A :pypi:`pytest-leaping` A simple plugin to use with pytest Mar 27, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-leo-interface` Pytest extension tool for leo projects. Mar 19, 2025 N/A N/A :pypi:`pytest-level` Select tests of a given level or lower Oct 21, 2019 N/A pytest + :pypi:`pytest-lf-skip` A pytest plugin which makes \`--last-failed\` skip instead of deselect tests. Oct 14, 2025 4 - Beta pytest>=8.3.5 :pypi:`pytest-libfaketime` A python-libfaketime plugin for pytest Apr 12, 2024 4 - Beta pytest>=3.0.0 - :pypi:`pytest-libiio` A pytest plugin to manage interfacing with libiio contexts Dec 22, 2023 4 - Beta N/A + :pypi:`pytest-libiio` A pytest plugin for testing libiio based devices Aug 15, 2025 N/A pytest>=3.5.0 :pypi:`pytest-libnotify` Pytest plugin that shows notifications about the test run Apr 02, 2021 3 - Alpha pytest :pypi:`pytest-ligo` Jan 16, 2020 4 - Beta N/A :pypi:`pytest-lineno` A pytest plugin to show the line numbers of test functions Dec 04, 2020 N/A pytest :pypi:`pytest-line-profiler` Profile code executed by pytest Aug 10, 2023 4 - Beta pytest >=3.5.0 :pypi:`pytest-line-profiler-apn` Profile code executed by pytest Dec 05, 2022 N/A pytest (>=3.5.0) :pypi:`pytest-lisa` Pytest plugin for organizing tests. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) - :pypi:`pytest-listener` A simple network listener May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-listener` A simple network listener Nov 29, 2024 5 - Production/Stable pytest :pypi:`pytest-litf` A pytest plugin that stream output in LITF format Jan 18, 2021 4 - Beta pytest (>=3.1.1) :pypi:`pytest-litter` Pytest plugin which verifies that tests do not modify file trees. Nov 23, 2023 4 - Beta pytest >=6.1 :pypi:`pytest-live` Live results for pytest Mar 08, 2020 N/A pytest + :pypi:`pytest-llm` pytest-llm: A pytest plugin for testing LLM outputs with success rate thresholds. Oct 03, 2025 3 - Alpha pytest>=7.0.0 + :pypi:`pytest-llmeval` A pytest plugin to evaluate/benchmark LLM prompts Mar 19, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-lobster` Pytest to generate lobster tracing files Jul 26, 2025 N/A pytest>=7.0 :pypi:`pytest-local-badge` Generate local badges (shields) reporting your test suite status. Jan 15, 2023 N/A pytest (>=6.1.0) :pypi:`pytest-localftpserver` A PyTest plugin which provides an FTP fixture for your tests May 19, 2024 5 - Production/Stable pytest - :pypi:`pytest-localserver` pytest plugin to test server connections locally. Oct 12, 2023 4 - Beta N/A + :pypi:`pytest-localserver` pytest plugin to test server connections locally. Oct 06, 2024 4 - Beta N/A :pypi:`pytest-localstack` Pytest plugin for AWS integration tests Jun 07, 2023 4 - Beta pytest (>=6.0.0,<7.0.0) :pypi:`pytest-lock` pytest-lock is a pytest plugin that allows you to "lock" the results of unit tests, storing them in a local cache. This is particularly useful for tests that are resource-intensive or don't need to be run every time. When the tests are run subsequently, pytest-lock will compare the current results with the locked results and issue a warning if there are any discrepancies. Feb 03, 2024 N/A pytest (>=7.4.3,<8.0.0) - :pypi:`pytest-lockable` lockable resource plugin for pytest Jan 24, 2024 5 - Production/Stable pytest - :pypi:`pytest-locker` Used to lock object during testing. Essentially changing assertions from being hard coded to asserting that nothing changed Oct 29, 2021 N/A pytest (>=5.4) + :pypi:`pytest-lockable` lockable resource plugin for pytest Sep 08, 2025 5 - Production/Stable pytest + :pypi:`pytest-locker` Used to lock object during testing. Essentially changing assertions from being hard coded to asserting that nothing changed Dec 20, 2024 N/A pytest>=5.4 :pypi:`pytest-log` print log Aug 15, 2021 N/A pytest (>=3.8) :pypi:`pytest-logbook` py.test plugin to capture logbook log messages Nov 23, 2015 5 - Production/Stable pytest (>=2.8) :pypi:`pytest-logdog` Pytest plugin to test logging Jun 15, 2021 1 - Planning pytest (>=6.2.0) :pypi:`pytest-logfest` Pytest plugin providing three logger fixtures with basic or full writing to log files Jul 21, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-logger` Plugin configuring handlers for loggers from Python logging module. Mar 10, 2024 5 - Production/Stable pytest (>=3.2) + :pypi:`pytest-logger-db` Add your description here Sep 14, 2025 N/A N/A :pypi:`pytest-logging` Configures logging and allows tweaking the log level with a py.test flag Nov 04, 2015 4 - Beta N/A :pypi:`pytest-logging-end-to-end-test-tool` Sep 23, 2022 N/A pytest (>=7.1.2,<8.0.0) - :pypi:`pytest-logikal` Common testing environment Jun 27, 2024 5 - Production/Stable pytest==8.2.2 + :pypi:`pytest-logging-strict` pytest fixture logging configured from packaged YAML May 20, 2025 3 - Alpha pytest + :pypi:`pytest-logikal` Common testing environment Sep 11, 2025 5 - Production/Stable pytest==8.4.2 :pypi:`pytest-log-report` Package for creating a pytest test run reprot Dec 26, 2019 N/A N/A + :pypi:`pytest-logscanner` Pytest plugin for logscanner (A logger for python logging outputting to easily viewable (and filterable) html files. Good for people not grep savey, and color higlighting and quickly changing filters might even bye useful for commandline wizards.) Sep 30, 2024 4 - Beta pytest>=8.2.2 :pypi:`pytest-loguru` Pytest Loguru Mar 20, 2024 5 - Production/Stable pytest; extra == "test" - :pypi:`pytest-loop` pytest plugin for looping tests Mar 30, 2024 5 - Production/Stable pytest - :pypi:`pytest-lsp` A pytest plugin for end-to-end testing of language servers May 22, 2024 3 - Alpha pytest + :pypi:`pytest-loop` pytest plugin for looping tests Oct 17, 2024 5 - Production/Stable pytest + :pypi:`pytest-lsp` A pytest plugin for end-to-end testing of language servers Oct 25, 2025 5 - Production/Stable pytest>=8.0 + :pypi:`pytest-lw-realtime-result` Pytest plugin to generate realtime test results to a file Mar 13, 2025 N/A pytest>=3.5.0 + :pypi:`pytest-manifest` PyTest plugin for recording and asserting against a manifest file Apr 07, 2025 N/A pytest :pypi:`pytest-manual-marker` pytest marker for marking manual tests Aug 04, 2022 3 - Alpha pytest>=7 + :pypi:`pytest-mark-count` Get a count of the number of tests marked, unmarked, and unique tests if tests have multiple markers Nov 13, 2024 4 - Beta pytest>=8.0.0 :pypi:`pytest-markdoctest` A pytest plugin to doctest your markdown files Jul 22, 2022 4 - Beta pytest (>=6) :pypi:`pytest-markdown` Test your markdown docs with pytest Jan 15, 2021 4 - Beta pytest (>=6.0.1,<7.0.0) - :pypi:`pytest-markdown-docs` Run markdown code fences through pytest Mar 05, 2024 N/A pytest (>=7.0.0) - :pypi:`pytest-marker-bugzilla` py.test bugzilla integration plugin, using markers Jan 09, 2020 N/A N/A - :pypi:`pytest-markers-presence` A simple plugin to detect missed pytest tags and markers" Feb 04, 2021 4 - Beta pytest (>=6.0) + :pypi:`pytest-markdown-docs` Run markdown code fences through pytest Apr 09, 2025 N/A pytest>=7.0.0 + :pypi:`pytest-marker-bugzilla` py.test bugzilla integration plugin, using markers Apr 02, 2025 5 - Production/Stable pytest>=2.2.4 + :pypi:`pytest-markers-presence` A simple plugin to detect missed pytest tags and markers" Oct 30, 2024 4 - Beta pytest>=6.0 + :pypi:`pytest-mark-filter` Filter pytest marks by name using match kw May 11, 2025 N/A pytest>=8.3.0 :pypi:`pytest-markfiltration` UNKNOWN Nov 08, 2011 3 - Alpha N/A - :pypi:`pytest-mark-manage` 用例标签化管理 Jul 08, 2024 N/A pytest + :pypi:`pytest-mark-manage` 用例标签化管理 Aug 15, 2024 N/A pytest :pypi:`pytest-mark-no-py3` pytest plugin and bowler codemod to help migrate tests to Python 3 May 17, 2019 N/A pytest :pypi:`pytest-marks` UNKNOWN Nov 23, 2012 3 - Alpha N/A - :pypi:`pytest-matcher` Easy way to match captured \`pytest\` output against expectations stored in files Mar 15, 2024 5 - Production/Stable pytest + :pypi:`pytest-mask-secrets` Pytest plugin to hide sensitive data in test reports Jan 28, 2025 N/A N/A + :pypi:`pytest-matcher` Easy way to match captured \`pytest\` output against expectations stored in files Aug 07, 2025 5 - Production/Stable pytest + :pypi:`pytest-matchers` Matchers for pytest Feb 11, 2025 N/A pytest<9.0,>=7.0 :pypi:`pytest-match-skip` Skip matching marks. Matches partial marks using wildcards. May 15, 2019 4 - Beta pytest (>=4.4.1) :pypi:`pytest-mat-report` this is report Jan 20, 2021 N/A N/A :pypi:`pytest-matrix` Provide tools for generating tests from combinations of fixtures. Jun 24, 2020 5 - Production/Stable pytest (>=5.4.3,<6.0.0) :pypi:`pytest-maxcov` Compute the maximum coverage available through pytest with the minimum execution time cost Sep 24, 2023 N/A pytest (>=7.4.0,<8.0.0) + :pypi:`pytest-max-warnings` A Pytest plugin to exit non-zero exit code when the configured maximum warnings has been exceeded. Oct 23, 2024 4 - Beta pytest>=8.3.3 :pypi:`pytest-maybe-context` Simplify tests with warning and exception cases. Apr 16, 2023 N/A pytest (>=7,<8) :pypi:`pytest-maybe-raises` Pytest fixture for optional exception testing. May 27, 2022 N/A pytest ; extra == 'dev' :pypi:`pytest-mccabe` pytest plugin to run the mccabe code complexity checker. Jul 22, 2020 3 - Alpha pytest (>=5.4.0) + :pypi:`pytest-mcp` Pytest-style framework for evaluating Model Context Protocol (MCP) servers. Jul 07, 2025 N/A pytest>=8.4.0 :pypi:`pytest-md` Plugin for generating Markdown reports for pytest results Jul 11, 2019 3 - Alpha pytest (>=4.2.1) - :pypi:`pytest-md-report` A pytest plugin to generate test outcomes reports with markdown table format. May 18, 2024 4 - Beta pytest!=6.0.0,<9,>=3.3.2 - :pypi:`pytest-meilisearch` Pytest helpers for testing projects using Meilisearch Feb 15, 2024 N/A pytest (>=7.4.3) + :pypi:`pytest-md-report` A pytest plugin to generate test outcomes reports with markdown table format. May 02, 2025 4 - Beta pytest!=6.0.0,<9,>=3.3.2 + :pypi:`pytest-meilisearch` Pytest helpers for testing projects using Meilisearch Oct 08, 2024 N/A pytest>=7.4.3 :pypi:`pytest-memlog` Log memory usage during tests May 03, 2023 N/A pytest (>=7.3.0,<8.0.0) :pypi:`pytest-memprof` Estimates memory consumption of test functions Mar 29, 2019 4 - Beta N/A - :pypi:`pytest-memray` A simple plugin to use with pytest Apr 18, 2024 N/A pytest>=7.2 + :pypi:`pytest-memray` A simple plugin to use with pytest Aug 18, 2025 N/A pytest>=7.2 :pypi:`pytest-menu` A pytest plugin for console based interactive test selection just after the collection phase Oct 04, 2017 3 - Alpha pytest (>=2.4.2) :pypi:`pytest-mercurial` pytest plugin to write integration tests for projects using Mercurial Python internals Nov 21, 2020 1 - Planning N/A + :pypi:`pytest-mergify` Pytest plugin for Mergify Oct 23, 2025 N/A pytest>=6.0.0 :pypi:`pytest-mesh` pytest_mesh插件 Aug 05, 2022 N/A pytest (==7.1.2) :pypi:`pytest-message` Pytest plugin for sending report message of marked tests execution Aug 04, 2022 N/A pytest (>=6.2.5) :pypi:`pytest-messenger` Pytest to Slack reporting plugin Nov 24, 2022 5 - Production/Stable N/A :pypi:`pytest-metadata` pytest plugin for test session metadata Feb 12, 2024 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-metaexport` Pytest plugin for exporting custom test metadata to JSON. Jun 24, 2025 N/A pytest>=7.1.0 :pypi:`pytest-metrics` Custom metrics report for pytest Apr 04, 2020 N/A pytest - :pypi:`pytest-mh` Pytest multihost plugin Jul 02, 2024 N/A pytest + :pypi:`pytest-mfd-config` Pytest Plugin that handles test and topology configs and all their belongings like helper fixtures. Jul 11, 2025 N/A pytest<9,>=7.2.1 + :pypi:`pytest-mfd-logging` Module for handling PyTest logging. Jul 09, 2025 N/A pytest<9,>=7.2.1 + :pypi:`pytest-mh` Pytest multihost plugin Oct 16, 2025 N/A pytest :pypi:`pytest-mimesis` Mimesis integration with the pytest test runner Mar 21, 2020 5 - Production/Stable pytest (>=4.2) + :pypi:`pytest-mimic` Easily record function calls while testing Apr 24, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-minecraft` A pytest plugin for running tests against Minecraft releases Apr 06, 2022 N/A pytest (>=6.0.1) :pypi:`pytest-mini` A plugin to test mp Feb 06, 2023 N/A pytest (>=7.2.0,<8.0.0) - :pypi:`pytest-minio-mock` A pytest plugin for mocking Minio S3 interactions May 26, 2024 N/A pytest>=5.0.0 + :pypi:`pytest-minio-mock` A pytest plugin for mocking Minio S3 interactions Aug 06, 2025 N/A pytest>=5.0.0 + :pypi:`pytest-mirror` A pluggy-based pytest plugin and CLI tool for ensuring your test suite mirrors your source code structure Jul 30, 2025 4 - Beta N/A :pypi:`pytest-missing-fixtures` Pytest plugin that creates missing fixtures Oct 14, 2020 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-mitmproxy` pytest plugin for mitmproxy tests May 28, 2024 N/A pytest>=7.0 + :pypi:`pytest-missing-modules` Pytest plugin to easily fake missing modules Sep 03, 2024 N/A pytest>=8.3.2 + :pypi:`pytest-mitmproxy` pytest plugin for mitmproxy tests Nov 13, 2024 N/A pytest>=7.0 + :pypi:`pytest-mitmproxy-plugin` Use MITM Proxy in autotests with full control from code Apr 10, 2025 4 - Beta pytest>=7.2.0 :pypi:`pytest-ml` Test your machine learning! May 04, 2019 4 - Beta N/A :pypi:`pytest-mocha` pytest plugin to display test execution output like a mochajs Apr 02, 2020 4 - Beta pytest (>=5.4.0) - :pypi:`pytest-mock` Thin-wrapper around the mock package for easier use with pytest Mar 21, 2024 5 - Production/Stable pytest>=6.2.5 + :pypi:`pytest-mock` Thin-wrapper around the mock package for easier use with pytest Sep 16, 2025 5 - Production/Stable pytest>=6.2.5 :pypi:`pytest-mock-api` A mock API server with configurable routes and responses available as a fixture. Feb 13, 2019 1 - Planning pytest (>=4.0.0) :pypi:`pytest-mock-generator` A pytest fixture wrapper for https://pypi.org/project/mock-generator May 16, 2022 5 - Production/Stable N/A :pypi:`pytest-mock-helper` Help you mock HTTP call and generate mock code Jan 24, 2018 N/A pytest :pypi:`pytest-mockito` Base fixtures for mockito Jul 11, 2018 4 - Beta N/A :pypi:`pytest-mockredis` An in-memory mock of a Redis server that runs in a separate thread. This is to be used for unit-tests that require a Redis database. Jan 02, 2018 2 - Pre-Alpha N/A - :pypi:`pytest-mock-resources` A pytest plugin for easily instantiating reproducible mock resources. Jun 20, 2024 N/A pytest>=1.0 + :pypi:`pytest-mock-resources` A pytest plugin for easily instantiating reproducible mock resources. Sep 17, 2025 N/A pytest>=1.0 :pypi:`pytest-mock-server` Mock server plugin for pytest Jan 09, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-mockservers` A set of fixtures to test your requests to HTTP/UDP servers Mar 31, 2020 N/A pytest (>=4.3.0) :pypi:`pytest-mocktcp` A pytest plugin for testing TCP clients Oct 11, 2022 N/A pytest :pypi:`pytest-modalt` Massively distributed pytest runs using modal.com Feb 27, 2024 4 - Beta pytest >=6.2.0 + :pypi:`pytest-modern` A more modern pytest Aug 19, 2025 4 - Beta pytest>=8 :pypi:`pytest-modified-env` Pytest plugin to fail a test if it leaves modified \`os.environ\` afterwards. Jan 29, 2022 4 - Beta N/A :pypi:`pytest-modifyjunit` Utility for adding additional properties to junit xml for IDM QE Jan 10, 2019 N/A N/A :pypi:`pytest-molecule` PyTest Molecule Plugin :: discover and run molecule tests Mar 29, 2022 5 - Production/Stable pytest (>=7.0.0) :pypi:`pytest-molecule-JC` PyTest Molecule Plugin :: discover and run molecule tests Jul 18, 2023 5 - Production/Stable pytest (>=7.0.0) - :pypi:`pytest-mongo` MongoDB process and client fixtures plugin for Pytest. Mar 13, 2024 5 - Production/Stable pytest >=6.2 + :pypi:`pytest-mongo` MongoDB process and client fixtures plugin for Pytest. Aug 01, 2025 5 - Production/Stable pytest>=6.2 :pypi:`pytest-mongodb` pytest plugin for MongoDB fixtures May 16, 2023 5 - Production/Stable N/A + :pypi:`pytest-mongodb-nono` pytest plugin for MongoDB Jan 07, 2025 N/A N/A + :pypi:`pytest-mongodb-ry` pytest plugin for MongoDB Sep 25, 2025 N/A N/A :pypi:`pytest-monitor` Pytest plugin for analyzing resource usage. Jun 25, 2023 5 - Production/Stable pytest :pypi:`pytest-monkeyplus` pytest's monkeypatch subclass with extra functionalities Sep 18, 2012 5 - Production/Stable N/A :pypi:`pytest-monkeytype` pytest-monkeytype: Generate Monkeytype annotations from your pytest tests. Jul 29, 2020 4 - Beta N/A :pypi:`pytest-moto` Fixtures for integration tests of AWS services,uses moto mocking library. Aug 28, 2015 1 - Planning N/A + :pypi:`pytest-moto-fixtures` Fixtures for testing code that interacts with AWS Feb 04, 2025 1 - Planning pytest<9,>=8.3; extra == "pytest" :pypi:`pytest-motor` A pytest plugin for motor, the non-blocking MongoDB driver. Jul 21, 2021 3 - Alpha pytest :pypi:`pytest-mp` A test batcher for multiprocessed Pytest runs May 23, 2018 4 - Beta pytest :pypi:`pytest-mpi` pytest plugin to collect information from tests Jan 08, 2022 3 - Alpha pytest - :pypi:`pytest-mpiexec` pytest plugin for running individual tests with mpiexec Apr 13, 2023 3 - Alpha pytest + :pypi:`pytest-mpiexec` pytest plugin for running individual tests with mpiexec Jul 29, 2024 3 - Alpha pytest :pypi:`pytest-mpl` pytest plugin to help with testing figures output from Matplotlib Feb 14, 2024 4 - Beta pytest :pypi:`pytest-mproc` low-startup-overhead, scalable, distributed-testing pytest plugin Nov 15, 2022 4 - Beta pytest (>=6) - :pypi:`pytest-mqtt` pytest-mqtt supports testing systems based on MQTT May 08, 2024 4 - Beta pytest<9; extra == "test" + :pypi:`pytest-mqtt` pytest-mqtt supports testing systems based on MQTT Sep 10, 2025 5 - Production/Stable pytest<9; extra == "test" :pypi:`pytest-multihost` Utility for writing multi-host tests for pytest Apr 07, 2020 4 - Beta N/A - :pypi:`pytest-multilog` Multi-process logs handling and other helpers for pytest Jan 17, 2023 N/A pytest - :pypi:`pytest-multithreading` a pytest plugin for th and concurrent testing Dec 07, 2022 N/A N/A + :pypi:`pytest-multilog` Multi-process logs handling and other helpers for pytest Sep 21, 2025 N/A pytest + :pypi:`pytest-multithreading` a pytest plugin for th and concurrent testing Aug 05, 2024 N/A N/A :pypi:`pytest-multithreading-allure` pytest_multithreading_allure Nov 25, 2022 N/A N/A :pypi:`pytest-mutagen` Add the mutation testing feature to pytest Jul 24, 2020 N/A pytest (>=5.4) :pypi:`pytest-my-cool-lib` Nov 02, 2023 N/A pytest (>=7.1.3,<8.0.0) - :pypi:`pytest-mypy` Mypy static type checker plugin for Pytest Dec 18, 2022 4 - Beta pytest (>=6.2) ; python_version >= "3.10" + :pypi:`pytest-my-plugin` A pytest plugin that does awesome things Jan 27, 2025 N/A pytest>=6.0 + :pypi:`pytest-mypy` A Pytest Plugin for Mypy Apr 02, 2025 5 - Production/Stable pytest>=7.0 :pypi:`pytest-mypyd` Mypy static type checker plugin for Pytest Aug 20, 2019 4 - Beta pytest (<4.7,>=2.8) ; python_version < "3.5" - :pypi:`pytest-mypy-plugins` pytest plugin for writing tests for mypy plugins Mar 31, 2024 4 - Beta pytest>=7.0.0 - :pypi:`pytest-mypy-plugins-shim` Substitute for "pytest-mypy-plugins" for Python implementations which aren't supported by mypy. Apr 12, 2021 N/A pytest>=6.0.0 + :pypi:`pytest-mypy-plugins` pytest plugin for writing tests for mypy plugins Dec 21, 2024 4 - Beta pytest>=7.0.0 + :pypi:`pytest-mypy-plugins-shim` Substitute for "pytest-mypy-plugins" for Python implementations which aren't supported by mypy. Feb 14, 2025 N/A pytest>=6.0.0 :pypi:`pytest-mypy-runner` Run the mypy static type checker as a pytest test case Apr 23, 2024 N/A pytest>=8.0 :pypi:`pytest-mypy-testing` Pytest plugin to check mypy output. Mar 04, 2024 N/A pytest>=7,<9 - :pypi:`pytest-mysql` MySQL process and client fixtures for pytest May 23, 2024 5 - Production/Stable pytest>=6.2 + :pypi:`pytest-mysql` MySQL process and client fixtures for pytest Dec 10, 2024 5 - Production/Stable pytest>=6.2 + :pypi:`pytest-nb` Seedable Jupyter Notebook testing tool Jul 26, 2025 N/A pytest==8.4.1 :pypi:`pytest-ndb` pytest notebook debugger Apr 28, 2024 N/A pytest :pypi:`pytest-needle` pytest plugin for visual testing websites using selenium Dec 10, 2018 4 - Beta pytest (<5.0.0,>=3.0.0) :pypi:`pytest-neo` pytest-neo is a plugin for pytest that shows tests like screen of Matrix. Jan 08, 2022 3 - Alpha pytest (>=6.2.0) - :pypi:`pytest-neos` Pytest plugin for neos Jun 11, 2024 1 - Planning N/A - :pypi:`pytest-netdut` "Automated software testing for switches using pytest" Jul 05, 2024 N/A pytest<7.3,>=3.5.0 + :pypi:`pytest-neos` Pytest plugin for neos Sep 10, 2024 5 - Production/Stable pytest<8.0,>=7.2; extra == "dev" + :pypi:`pytest-netconf` A pytest plugin that provides a mock NETCONF (RFC6241/RFC6242) server for local testing. Jan 06, 2025 N/A N/A + :pypi:`pytest-netdut` "Automated software testing for switches using pytest" Oct 09, 2025 N/A pytest>=3.5.0 :pypi:`pytest-network` A simple plugin to disable network on socket level. May 07, 2020 N/A N/A :pypi:`pytest-network-endpoints` Network endpoints plugin for pytest Mar 06, 2022 N/A pytest :pypi:`pytest-never-sleep` pytest plugin helps to avoid adding tests without mock \`time.sleep\` May 05, 2021 3 - Alpha pytest (>=3.5.1) - :pypi:`pytest-nginx` nginx fixture for pytest Aug 12, 2017 5 - Production/Stable N/A + :pypi:`pytest-nginx` nginx fixture for pytest May 03, 2025 5 - Production/Stable pytest>=3.0.0 :pypi:`pytest-nginx-iplweb` nginx fixture for pytest - iplweb temporary fork Mar 01, 2019 5 - Production/Stable N/A :pypi:`pytest-ngrok` Jan 20, 2022 3 - Alpha pytest :pypi:`pytest-ngsfixtures` pytest ngs fixtures Sep 06, 2019 2 - Pre-Alpha pytest (>=5.0.0) - :pypi:`pytest-nhsd-apim` Pytest plugin accessing NHSDigital's APIM proxies Jul 01, 2024 N/A pytest<9.0.0,>=8.2.0 + :pypi:`pytest-nhsd-apim` Pytest plugin accessing NHSDigital's APIM proxies Oct 29, 2025 N/A pytest<9.0.0,>=8.2.0 :pypi:`pytest-nice` A pytest plugin that alerts user of failed test cases with screen notifications May 04, 2019 4 - Beta pytest :pypi:`pytest-nice-parametrize` A small snippet for nicer PyTest's Parametrize Apr 17, 2021 5 - Production/Stable N/A - :pypi:`pytest_nlcov` Pytest plugin to get the coverage of the new lines (based on git diff) only Apr 11, 2024 N/A N/A - :pypi:`pytest-nocustom` Run all tests without custom markers Apr 11, 2024 5 - Production/Stable N/A + :pypi:`pytest_nlcov` Pytest plugin to get the coverage of the new lines (based on git diff) only Aug 05, 2024 N/A N/A + :pypi:`pytest-nocustom` Run all tests without custom markers Aug 05, 2024 5 - Production/Stable N/A :pypi:`pytest-node-dependency` pytest plugin for controlling execution flow Apr 10, 2024 5 - Production/Stable N/A :pypi:`pytest-nodev` Test-driven source code search for Python. Jul 21, 2016 4 - Beta pytest (>=2.8.1) - :pypi:`pytest-nogarbage` Ensure a test produces no garbage Aug 29, 2021 5 - Production/Stable pytest (>=4.6.0) + :pypi:`pytest-nogarbage` Ensure a test produces no garbage Feb 24, 2025 5 - Production/Stable pytest>=4.6.0 + :pypi:`pytest-no-problem` Pytest plugin to tell you when there's no problem Oct 18, 2025 N/A pytest>=7.0 :pypi:`pytest-nose-attrib` pytest plugin to use nose @attrib marks decorators and pick tests based on attributes and partially uses nose-attrib plugin approach Aug 13, 2023 N/A N/A :pypi:`pytest_notebook` A pytest plugin for testing Jupyter Notebooks. Nov 28, 2023 4 - Beta pytest>=3.5.0 :pypi:`pytest-notice` Send pytest execution result email Nov 05, 2020 N/A N/A @@ -909,46 +1069,55 @@ This list contains 1487 plugins. :pypi:`pytest-notimplemented` Pytest markers for not implemented features and tests. Aug 27, 2019 N/A pytest (>=5.1,<6.0) :pypi:`pytest-notion` A PyTest Reporter to send test runs to Notion.so Aug 07, 2019 N/A N/A :pypi:`pytest-nunit` A pytest plugin for generating NUnit3 test result XML output Feb 26, 2024 5 - Production/Stable N/A - :pypi:`pytest-oar` PyTest plugin for the OAR testing framework May 02, 2023 N/A pytest>=6.0.1 + :pypi:`pytest-oar` PyTest plugin for the OAR testing framework May 12, 2025 N/A pytest>=6.0.1 + :pypi:`pytest-oarepo` Oct 23, 2025 N/A pytest>=7.1.2; extra == "dev" :pypi:`pytest-object-getter` Import any object from a 3rd party module while mocking its namespace on demand. Jul 31, 2022 5 - Production/Stable pytest :pypi:`pytest-ochrus` pytest results data-base and HTML reporter Feb 21, 2018 4 - Beta N/A :pypi:`pytest-odc` A pytest plugin for simplifying ODC database tests Aug 04, 2023 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-odoo` py.test plugin to run Odoo tests Jul 06, 2023 4 - Beta pytest (>=7.2.0) + :pypi:`pytest-odoo` py.test plugin to run Odoo tests May 20, 2025 5 - Production/Stable pytest>=8 :pypi:`pytest-odoo-fixtures` Project description Jun 25, 2019 N/A N/A + :pypi:`pytest-oduit` py.test plugin to run Odoo tests Oct 06, 2025 5 - Production/Stable pytest>=8 :pypi:`pytest-oerp` pytest plugin to test OpenERP modules Feb 28, 2012 3 - Alpha N/A :pypi:`pytest-offline` Mar 09, 2023 1 - Planning pytest (>=7.0.0,<8.0.0) :pypi:`pytest-ogsm-plugin` 针对特定项目定制化插件,优化了pytest报告展示方式,并添加了项目所需特定参数 May 16, 2023 N/A N/A :pypi:`pytest-ok` The ultimate pytest output plugin Apr 01, 2019 4 - Beta N/A + :pypi:`pytest-once` xdist-safe 'run once' fixture decorator for pytest (setup/teardown across workers) Oct 10, 2025 3 - Alpha pytest>=8.4.0 :pypi:`pytest-only` Use @pytest.mark.only to run a single test May 27, 2024 5 - Production/Stable pytest<9,>=3.6.0 :pypi:`pytest-oof` A Pytest plugin providing structured, programmatic access to a test run's results Dec 11, 2023 4 - Beta N/A :pypi:`pytest-oot` Run object-oriented tests in a simple format Sep 18, 2016 4 - Beta N/A :pypi:`pytest-openfiles` Pytest plugin for detecting inadvertent open file handles Jun 05, 2024 3 - Alpha pytest>=4.6 - :pypi:`pytest-opentelemetry` A pytest plugin for instrumenting test runs via OpenTelemetry Oct 01, 2023 N/A pytest - :pypi:`pytest-opentmi` pytest plugin for publish results to opentmi Jun 02, 2022 5 - Production/Stable pytest (>=5.0) - :pypi:`pytest-operator` Fixtures for Operators Sep 28, 2022 N/A pytest + :pypi:`pytest-open-html` Auto-open HTML reports after pytest runs Mar 31, 2025 N/A pytest>=6.0 + :pypi:`pytest-opentelemetry` A pytest plugin for instrumenting test runs via OpenTelemetry Apr 25, 2025 N/A pytest + :pypi:`pytest-opentmi` pytest plugin for publish results to opentmi Mar 22, 2025 5 - Production/Stable pytest>=5.0 + :pypi:`pytest-operator` Fixtures for Charmed Operators Sep 28, 2022 N/A pytest :pypi:`pytest-optional` include/exclude values of fixtures in pytest Oct 07, 2015 N/A N/A - :pypi:`pytest-optional-tests` Easy declaration of optional tests (i.e., that are not run by default) Jul 09, 2019 4 - Beta pytest (>=4.5.0) + :pypi:`pytest-optional-tests` Easy declaration of optional tests (i.e., that are not run by default) Jul 21, 2025 4 - Beta pytest; extra == "dev" :pypi:`pytest-orchestration` A pytest plugin for orchestrating tests Jul 18, 2019 N/A N/A - :pypi:`pytest-order` pytest plugin to run your tests in a specific order Apr 02, 2024 4 - Beta pytest>=5.0; python_version < "3.10" + :pypi:`pytest-order` pytest plugin to run your tests in a specific order Aug 22, 2024 5 - Production/Stable pytest>=5.0; python_version < "3.10" + :pypi:`pytest-ordered` Declare the order in which tests should run in your pytest.ini Oct 07, 2024 N/A pytest>=6.2.0 :pypi:`pytest-ordering` pytest plugin to run your tests in a specific order Nov 14, 2018 4 - Beta pytest :pypi:`pytest-order-modify` 新增run_marker 来自定义用例的执行顺序 Nov 04, 2022 N/A N/A :pypi:`pytest-osxnotify` OS X notifications for py.test results. May 15, 2015 N/A N/A :pypi:`pytest-ot` A pytest plugin for instrumenting test runs via OpenTelemetry Mar 21, 2024 N/A pytest; extra == "dev" - :pypi:`pytest-otel` OpenTelemetry plugin for Pytest Mar 18, 2024 N/A pytest==8.1.1 + :pypi:`pytest-otel` OpenTelemetry plugin for Pytest Apr 24, 2025 N/A pytest==8.3.5 + :pypi:`pytest-otelmark` Pytest plugin for otelmark. Sep 14, 2025 3 - Alpha pytest>=8.3.5 :pypi:`pytest-override-env-var` Pytest mark to override a value of an environment variable. Feb 25, 2023 N/A N/A - :pypi:`pytest-owner` Add owner mark for tests Apr 25, 2022 N/A N/A + :pypi:`pytest-owner` Add owner mark for tests Aug 19, 2024 N/A pytest :pypi:`pytest-pact` A simple plugin to use with pytest Jan 07, 2019 4 - Beta N/A + :pypi:`pytest-pagerduty` Pytest plugin for PagerDuty integration via automation testing. Mar 22, 2025 N/A pytest<9.0.0,>=7.4.0 :pypi:`pytest-pahrametahrize` Parametrize your tests with a Boston accent. Nov 24, 2021 4 - Beta pytest (>=6.0,<7.0) :pypi:`pytest-parallel` a pytest plugin for parallel and concurrent testing Oct 10, 2021 3 - Alpha pytest (>=3.0.0) :pypi:`pytest-parallel-39` a pytest plugin for parallel and concurrent testing Jul 12, 2021 3 - Alpha pytest (>=3.0.0) :pypi:`pytest-parallelize-tests` pytest plugin that parallelizes test execution across multiple hosts Jan 27, 2023 4 - Beta N/A :pypi:`pytest-param` pytest plugin to test all, first, last or random params Sep 11, 2016 4 - Beta pytest (>=2.6.0) - :pypi:`pytest-paramark` Configure pytest fixtures using a combination of"parametrize" and markers Jan 10, 2020 4 - Beta pytest (>=4.5.0) :pypi:`pytest-parametrization` Simpler PyTest parametrization May 22, 2022 5 - Production/Stable N/A + :pypi:`pytest-parametrization-annotation` A pytest library for parametrizing tests using type hints. Dec 10, 2024 5 - Production/Stable pytest>=7 + :pypi:`pytest-parametrize` pytest decorator for parametrizing test cases in a dict-way Sep 25, 2025 5 - Production/Stable pytest<9.0.0,>=8.3.0 :pypi:`pytest-parametrize-cases` A more user-friendly way to write parametrized tests. Mar 13, 2022 N/A pytest (>=6.1.2) - :pypi:`pytest-parametrized` Pytest decorator for parametrizing tests with default iterables. Nov 03, 2023 5 - Production/Stable pytest + :pypi:`pytest-parametrized` Pytest decorator for parametrizing tests with default iterables. Dec 21, 2024 5 - Production/Stable pytest :pypi:`pytest-parametrize-suite` A simple pytest extension for creating a named test suite. Jan 19, 2023 5 - Production/Stable pytest :pypi:`pytest_param_files` Create pytest parametrize decorators from external files. Jul 29, 2023 N/A pytest + :pypi:`pytest-params` Simplified pytest test case parameters. Apr 27, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-param-scope` pytest parametrize scope fixture workaround Oct 18, 2023 N/A pytest :pypi:`pytest-parawtf` Finally spell paramete?ri[sz]e correctly Dec 03, 2018 4 - Beta pytest (>=3.6.0) :pypi:`pytest-pass` Check out https://github.com/elilutsky/pytest-pass Dec 04, 2019 N/A N/A @@ -956,7 +1125,7 @@ This list contains 1487 plugins. :pypi:`pytest-paste-config` Allow setting the path to a paste config file Sep 18, 2013 3 - Alpha N/A :pypi:`pytest-patch` An automagic \`patch\` fixture that can patch objects directly or by name. Apr 29, 2023 3 - Alpha pytest (>=7.0.0) :pypi:`pytest-patches` A contextmanager pytest fixture for handling multiple mock patches Aug 30, 2021 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-patterns` pytest plugin to make testing complicated long string output easy to write and easy to debug Jun 14, 2024 4 - Beta N/A + :pypi:`pytest-patterns` pytest plugin to make testing complicated long string output easy to write and easy to debug Oct 22, 2024 4 - Beta pytest>=6 :pypi:`pytest-pdb` pytest plugin which adds pdb helper commands related to pytest. Jul 31, 2018 N/A N/A :pypi:`pytest-peach` pytest plugin for fuzzing with Peach API Security Apr 12, 2019 4 - Beta pytest (>=2.8.7) :pypi:`pytest-pep257` py.test plugin for pep257 Jul 09, 2016 N/A N/A @@ -965,13 +1134,15 @@ This list contains 1487 plugins. :pypi:`pytest-percents` Mar 16, 2024 N/A N/A :pypi:`pytest-perf` Run performance tests against the mainline code. May 20, 2024 5 - Production/Stable pytest!=8.1.*,>=6; extra == "testing" :pypi:`pytest-performance` A simple plugin to ensure the execution of critical sections of code has not been impacted Sep 11, 2020 5 - Production/Stable pytest (>=3.7.0) - :pypi:`pytest-performancetotal` A performance plugin for pytest Mar 19, 2024 4 - Beta N/A - :pypi:`pytest-persistence` Pytest tool for persistent objects May 23, 2024 N/A N/A - :pypi:`pytest-pexpect` Pytest pexpect plugin. Mar 27, 2024 4 - Beta pytest>=6.2.0 - :pypi:`pytest-pg` A tiny plugin for pytest which runs PostgreSQL in Docker May 21, 2024 5 - Production/Stable pytest>=6.0.0 + :pypi:`pytest-performancetotal` A performance plugin for pytest Aug 05, 2025 5 - Production/Stable N/A + :pypi:`pytest-persistence` Pytest tool for persistent objects Aug 21, 2024 N/A N/A + :pypi:`pytest-pexpect` Pytest pexpect plugin. Sep 10, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-pg` A tiny plugin for pytest which runs PostgreSQL in Docker May 18, 2025 5 - Production/Stable pytest>=7.4 :pypi:`pytest-pgsql` Pytest plugins and helpers for tests using a Postgres database. May 13, 2020 5 - Production/Stable pytest (>=3.0.0) :pypi:`pytest-phmdoctest` pytest plugin to test Python examples in Markdown using phmdoctest. Apr 15, 2022 4 - Beta pytest (>=5.4.3) - :pypi:`pytest-picked` Run the tests related to the changed files Jul 27, 2023 N/A pytest (>=3.7.0) + :pypi:`pytest-phoenix-interface` Pytest extension tool for phoenix projects. Mar 19, 2025 N/A N/A + :pypi:`pytest-picked` Run the tests related to the changed files Nov 06, 2024 N/A pytest>=3.7.0 + :pypi:`pytest-pickle-cache` A pytest plugin for caching test results using pickle. Feb 17, 2025 N/A pytest>=7 :pypi:`pytest-pigeonhole` Jun 25, 2018 5 - Production/Stable pytest (>=3.4) :pypi:`pytest-pikachu` Show surprise when tests are passing Aug 05, 2021 5 - Production/Stable pytest :pypi:`pytest-pilot` Slice in your test base thanks to powerful markers. Oct 09, 2020 5 - Production/Stable N/A @@ -981,55 +1152,63 @@ This list contains 1487 plugins. :pypi:`pytest-pinpoint` A pytest plugin which runs SBFL algorithms to detect faults. Sep 25, 2020 N/A pytest (>=4.4.0) :pypi:`pytest-pipeline` Pytest plugin for functional testing of data analysispipelines Jan 24, 2017 3 - Alpha N/A :pypi:`pytest-pitch` runs tests in an order such that coverage increases as fast as possible Nov 02, 2023 4 - Beta pytest >=7.3.1 + :pypi:`pytest-platform-adapter` Pytest集成自动化平台插件 Feb 18, 2025 5 - Production/Stable pytest>=6.2.5 :pypi:`pytest-platform-markers` Markers for pytest to skip tests on specific platforms Sep 09, 2019 4 - Beta pytest (>=3.6.0) :pypi:`pytest-play` pytest plugin that let you automate actions and assertions with test metrics reporting executing plain YAML files Jun 12, 2019 5 - Production/Stable N/A :pypi:`pytest-playbook` Pytest plugin for reading playbooks. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) - :pypi:`pytest-playwright` A pytest wrapper with fixtures for Playwright to automate web browsers Jul 03, 2024 N/A N/A - :pypi:`pytest_playwright_async` ASYNC Pytest plugin for Playwright May 24, 2024 N/A N/A - :pypi:`pytest-playwright-asyncio` Aug 29, 2023 N/A N/A + :pypi:`pytest-playwright` A pytest wrapper with fixtures for Playwright to automate web browsers Sep 08, 2025 N/A pytest<9.0.0,>=6.2.4 + :pypi:`pytest_playwright_async` ASYNC Pytest plugin for Playwright Sep 28, 2024 N/A N/A + :pypi:`pytest-playwright-asyncio` A pytest wrapper with async fixtures for Playwright to automate web browsers Sep 08, 2025 N/A pytest<9.0.0,>=6.2.4 + :pypi:`pytest-playwright-axe` An axe-core integration for accessibility testing using Playwright Python. Nov 01, 2025 5 - Production/Stable N/A :pypi:`pytest-playwright-enhanced` A pytest plugin for playwright python Mar 24, 2024 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-playwrights` A pytest wrapper with fixtures for Playwright to automate web browsers Dec 02, 2021 N/A N/A :pypi:`pytest-playwright-snapshot` A pytest wrapper for snapshot testing with playwright Aug 19, 2021 N/A N/A :pypi:`pytest-playwright-visual` A pytest fixture for visual testing with Playwright Apr 28, 2022 N/A N/A - :pypi:`pytest-plone` Pytest plugin to test Plone addons May 15, 2024 3 - Alpha pytest<8.0.0 + :pypi:`pytest-playwright-visual-snapshot` Easy pytest visual regression testing using playwright Jul 02, 2025 N/A N/A + :pypi:`pytest-pl-grader` A pytest plugin for autograding Python code. Designed for use with the PrairieLearn platform. Nov 01, 2025 3 - Alpha pytest + :pypi:`pytest-plone` Pytest plugin to test Plone addons Jun 11, 2025 3 - Alpha pytest<8.0.0 :pypi:`pytest-plt` Fixtures for quickly making Matplotlib plots in tests Jan 17, 2024 5 - Production/Stable pytest :pypi:`pytest-plugin-helpers` A plugin to help developing and testing other plugins Nov 23, 2019 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-plus` PyTest Plus Plugin :: extends pytest functionality Mar 26, 2024 5 - Production/Stable pytest>=7.4.2 + :pypi:`pytest-plugins` A Python package for managing pytest plugins. Oct 23, 2025 N/A pytest + :pypi:`pytest-plus` PyTest Plus Plugin :: extends pytest functionality Feb 02, 2025 5 - Production/Stable pytest>=7.4.2 :pypi:`pytest-pmisc` Mar 21, 2019 5 - Production/Stable N/A - :pypi:`pytest-pogo` Pytest plugin for pogo-migrate May 22, 2024 1 - Planning pytest<9,>=7 + :pypi:`pytest-pogo` Pytest plugin for pogo-migrate May 05, 2025 4 - Beta pytest<9,>=7 :pypi:`pytest-pointers` Pytest plugin to define functions you test with special marks for better navigation and reports Dec 26, 2022 N/A N/A :pypi:`pytest-pokie` Pokie plugin for pytest Oct 19, 2023 5 - Production/Stable N/A :pypi:`pytest-polarion-cfme` pytest plugin for collecting test cases and recording test results Nov 13, 2017 3 - Alpha N/A :pypi:`pytest-polarion-collect` pytest plugin for collecting polarion test cases data Jun 18, 2020 3 - Alpha pytest :pypi:`pytest-polecat` Provides Polecat pytest fixtures Aug 12, 2019 4 - Beta N/A + :pypi:`pytest-polymeric-report` A polymeric test report plugin for pytest Oct 20, 2025 N/A N/A :pypi:`pytest-ponyorm` PonyORM in Pytest Oct 31, 2018 N/A pytest (>=3.1.1) :pypi:`pytest-poo` Visualize your crappy tests Mar 25, 2021 5 - Production/Stable pytest (>=2.3.4) :pypi:`pytest-poo-fail` Visualize your failed tests with poo Feb 12, 2015 5 - Production/Stable N/A :pypi:`pytest-pook` Pytest plugin for pook Feb 15, 2024 4 - Beta pytest :pypi:`pytest-pop` A pytest plugin to help with testing pop projects May 09, 2023 5 - Production/Stable pytest - :pypi:`pytest-porringer` Jan 18, 2024 N/A pytest>=7.4.4 + :pypi:`pytest-porcochu` Show surprise when tests are passing Nov 28, 2024 5 - Production/Stable N/A :pypi:`pytest-portion` Select a portion of the collected tests Jan 28, 2021 4 - Beta pytest (>=3.5.0) :pypi:`pytest-postgres` Run PostgreSQL in Docker container in Pytest. Mar 22, 2020 N/A pytest - :pypi:`pytest-postgresql` Postgresql fixtures and fixture factories for Pytest. Mar 11, 2024 5 - Production/Stable pytest >=6.2 + :pypi:`pytest-postgresql` Postgresql fixtures and fixture factories for Pytest. May 17, 2025 5 - Production/Stable pytest>=7.2 :pypi:`pytest-power` pytest plugin with powerful fixtures Dec 31, 2020 N/A pytest (>=5.4) - :pypi:`pytest-powerpack` Mar 17, 2024 N/A pytest (>=8.1.1,<9.0.0) + :pypi:`pytest-powerpack` A plugin containing extra batteries for pytest Jan 04, 2025 N/A pytest<9.0.0,>=8.1.1 :pypi:`pytest-prefer-nested-dup-tests` A Pytest plugin to drop duplicated tests during collection, but will prefer keeping nested packages. Apr 27, 2022 4 - Beta pytest (>=7.1.1,<8.0.0) - :pypi:`pytest-pretty` pytest plugin for printing summary data as I want it Apr 05, 2023 5 - Production/Stable pytest>=7 + :pypi:`pytest-pretty` pytest plugin for printing summary data as I want it Jun 04, 2025 5 - Production/Stable pytest>=7 :pypi:`pytest-pretty-terminal` pytest plugin for generating prettier terminal output Jan 31, 2022 N/A pytest (>=3.4.1) :pypi:`pytest-pride` Minitest-style test colors Apr 02, 2016 3 - Alpha N/A - :pypi:`pytest-print` pytest-print adds the printer fixture you can use to print messages to the user (directly to the pytest runner, not stdout) Aug 25, 2023 5 - Production/Stable pytest>=7.4 - :pypi:`pytest-priority` pytest plugin for add priority for tests Jul 23, 2023 N/A N/A - :pypi:`pytest-proceed` Apr 10, 2024 N/A pytest + :pypi:`pytest-print` pytest-print adds the printer fixture you can use to print messages to the user (directly to the pytest runner, not stdout) Oct 09, 2025 5 - Production/Stable pytest>=8.4.2 + :pypi:`pytest-priority` pytest plugin for add priority for tests Aug 19, 2024 N/A pytest + :pypi:`pytest-proceed` Oct 01, 2024 N/A pytest :pypi:`pytest-profiles` pytest plugin for configuration profiles Dec 09, 2021 4 - Beta pytest (>=3.7.0) - :pypi:`pytest-profiling` Profiling plugin for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-profiling` Profiling plugin for py.test Nov 29, 2024 5 - Production/Stable pytest :pypi:`pytest-progress` pytest plugin for instant test progress status Jun 18, 2024 5 - Production/Stable pytest>=2.7 :pypi:`pytest-prometheus` Report test pass / failures to a Prometheus PushGateway Oct 03, 2017 N/A N/A :pypi:`pytest-prometheus-pushgateway` Pytest report plugin for Zulip Sep 27, 2022 5 - Production/Stable pytest + :pypi:`pytest-prometheus-pushgw` Pytest plugin to export test metrics to Prometheus Pushgateway May 19, 2025 N/A pytest>=6.0.0 + :pypi:`pytest-proofy` Pytest plugin for Proofy test reporting Oct 17, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-prosper` Test helpers for Prosper projects Sep 24, 2018 N/A N/A - :pypi:`pytest-prysk` Pytest plugin for prysk Mar 12, 2024 4 - Beta pytest (>=7.3.2) + :pypi:`pytest-prysk` Pytest plugin for prysk Dec 10, 2024 4 - Beta pytest>=7.3.2 :pypi:`pytest-pspec` A rspec format reporter for Python ptest Jun 02, 2020 4 - Beta pytest (>=3.0.0) :pypi:`pytest-psqlgraph` pytest plugin for testing applications that use psqlgraph Oct 19, 2021 4 - Beta pytest (>=6.0) - :pypi:`pytest-pt` pytest plugin to use \*.pt files as tests May 15, 2024 4 - Beta pytest + :pypi:`pytest-pt` pytest plugin to use \*.pt files as tests Sep 22, 2024 5 - Production/Stable pytest :pypi:`pytest-ptera` Use ptera probes in tests Mar 01, 2022 N/A pytest (>=6.2.4,<7.0.0) :pypi:`pytest-publish` Jun 04, 2024 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-pudb` Pytest PuDB debugger integration Oct 25, 2018 3 - Alpha pytest (>=2.0) @@ -1038,42 +1217,48 @@ This list contains 1487 plugins. :pypi:`pytest-pusher` pytest plugin for push report to minio Jan 06, 2023 5 - Production/Stable pytest (>=3.6) :pypi:`pytest-py125` Dec 03, 2022 N/A N/A :pypi:`pytest-pycharm` Plugin for py.test to enter PyCharm debugger on uncaught exceptions Aug 13, 2020 5 - Production/Stable pytest (>=2.3) - :pypi:`pytest-pycodestyle` pytest plugin to run pycodestyle Oct 28, 2022 3 - Alpha N/A + :pypi:`pytest-pycodestyle` pytest plugin to run pycodestyle Jul 20, 2025 3 - Alpha pytest>=7.0 + :pypi:`pytest-pydantic-schema-sync` Pytest plugin to synchronise Pydantic model schemas with JSONSchema files Aug 29, 2024 N/A pytest>=6 :pypi:`pytest-pydev` py.test plugin to connect to a remote debug server with PyDev or PyCharm. Nov 15, 2017 3 - Alpha N/A - :pypi:`pytest-pydocstyle` pytest plugin to run pydocstyle Jan 05, 2023 3 - Alpha N/A + :pypi:`pytest-pydocstyle` pytest plugin to run pydocstyle Oct 09, 2024 3 - Alpha pytest>=7.0 + :pypi:`pytest-pylembic` This package provides pytest plugin for validating Alembic migrations using the pylembic package. Jul 22, 2025 3 - Alpha N/A :pypi:`pytest-pylint` pytest plugin to check source code with pylint Oct 06, 2023 5 - Production/Stable pytest >=7.0 + :pypi:`pytest-pylyzer` A pytest plugin for pylyzer Feb 15, 2025 4 - Beta N/A :pypi:`pytest-pymysql-autorecord` Record PyMySQL queries and mock with the stored data. Sep 02, 2022 N/A N/A - :pypi:`pytest-pyodide` Pytest plugin for testing applications that use Pyodide Jun 12, 2024 N/A pytest + :pypi:`pytest-pyodide` Pytest plugin for testing applications that use Pyodide Oct 24, 2025 N/A pytest :pypi:`pytest-pypi` Easily test your HTTP library against a local copy of pypi Mar 04, 2018 3 - Alpha N/A :pypi:`pytest-pypom-navigation` Core engine for cookiecutter-qa and pytest-play packages Feb 18, 2019 4 - Beta pytest (>=3.0.7) :pypi:`pytest-pyppeteer` A plugin to run pyppeteer in pytest Apr 28, 2022 N/A pytest (>=6.2.5,<7.0.0) :pypi:`pytest-pyq` Pytest fixture "q" for pyq Mar 10, 2020 5 - Production/Stable N/A - :pypi:`pytest-pyramid` pytest_pyramid - provides fixtures for testing pyramid applications with pytest test suite Oct 11, 2023 5 - Production/Stable pytest - :pypi:`pytest-pyramid-server` Pyramid server fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-pyramid` pytest_pyramid - provides fixtures for testing pyramid applications with pytest test suite Sep 30, 2025 5 - Production/Stable pytest + :pypi:`pytest-pyramid-server` Pyramid server fixture for py.test Oct 17, 2024 5 - Production/Stable pytest :pypi:`pytest-pyreport` PyReport is a lightweight reporting plugin for Pytest that provides concise HTML report May 05, 2024 N/A pytest :pypi:`pytest-pyright` Pytest plugin for type checking code with Pyright Jan 26, 2024 4 - Beta pytest >=7.0.0 - :pypi:`pytest-pyspec` A plugin that transforms the pytest output into a result similar to the RSpec. It enables the use of docstrings to display results and also enables the use of the prefixes "describe", "with" and "it". Jan 02, 2024 N/A pytest (>=7.2.1,<8.0.0) - :pypi:`pytest-pystack` Plugin to run pystack after a timeout for a test suite. Jan 04, 2024 N/A pytest >=3.5.0 + :pypi:`pytest-pyspark-plugin` Pytest pyspark plugin (p3) Jul 28, 2025 4 - Beta pytest>=8.0.0 + :pypi:`pytest-pyspec` A plugin that transforms the pytest output into a result similar to the RSpec. It enables the use of docstrings to display results and also enables the use of the prefixes "describe", "with" and "it". Aug 17, 2024 N/A pytest<9.0.0,>=8.3.2 + :pypi:`pytest-pystack` Plugin to run pystack after a timeout for a test suite. Nov 16, 2024 N/A pytest>=3.5.0 + :pypi:`pytest-pytestdb` Add your description here Sep 14, 2025 N/A N/A :pypi:`pytest-pytestrail` Pytest plugin for interaction with TestRail Aug 27, 2020 4 - Beta pytest (>=3.8.0) - :pypi:`pytest-pythonhashseed` Pytest plugin to set PYTHONHASHSEED env var. Feb 25, 2024 4 - Beta pytest>=3.0.0 + :pypi:`pytest-pytestrail-internal` Pytest plugin for interaction with TestRail, Pytest plugin for TestRail (internal fork from: https://github.com/tolstislon/pytest-pytestrail with PR #25 fix) Jun 12, 2025 4 - Beta pytest>=3.8.0 + :pypi:`pytest-pythonhashseed` Pytest plugin to set PYTHONHASHSEED env var. Sep 28, 2025 4 - Beta pytest>=3.0.0 :pypi:`pytest-pythonpath` pytest plugin for adding to the PYTHONPATH from command line or configs. Feb 10, 2022 5 - Production/Stable pytest (<7,>=2.5.2) :pypi:`pytest-python-test-engineer-sort` Sort plugin for Pytest May 13, 2024 N/A pytest>=6.2.0 :pypi:`pytest-pytorch` pytest plugin for a better developer experience when working with the PyTorch test suite May 25, 2021 4 - Beta pytest :pypi:`pytest-pyvenv` A package for create venv in tests Feb 27, 2024 N/A pytest ; extra == 'test' - :pypi:`pytest-pyvista` Pytest-pyvista package Sep 29, 2023 4 - Beta pytest>=3.5.0 - :pypi:`pytest-qanova` A pytest plugin to collect test information May 26, 2024 3 - Alpha pytest - :pypi:`pytest-qaseio` Pytest plugin for Qase.io integration May 30, 2024 4 - Beta pytest<9.0.0,>=7.2.2 + :pypi:`pytest-pyvista` Pytest-pyvista package. Oct 06, 2025 4 - Beta pytest>=6.2.0 + :pypi:`pytest-qanova` A pytest plugin to collect test information Sep 05, 2024 3 - Alpha pytest + :pypi:`pytest-qaseio` Pytest plugin for Qase.io integration Oct 01, 2025 5 - Production/Stable pytest<9.0.0,>=7.2.2 :pypi:`pytest-qasync` Pytest support for qasync. Jul 12, 2021 4 - Beta pytest (>=5.4.0) :pypi:`pytest-qatouch` Pytest plugin for uploading test results to your QA Touch Testrun. Feb 14, 2023 4 - Beta pytest (>=6.2.0) :pypi:`pytest-qgis` A pytest plugin for testing QGIS python plugins Jun 14, 2024 5 - Production/Stable pytest>=6.0 :pypi:`pytest-qml` Run QML Tests with pytest Dec 02, 2020 4 - Beta pytest (>=6.0.0) :pypi:`pytest-qr` pytest plugin to generate test result QR codes Nov 25, 2021 4 - Beta N/A - :pypi:`pytest-qt` pytest support for PyQt and PySide applications Feb 07, 2024 5 - Production/Stable pytest - :pypi:`pytest-qt-app` QT app fixture for py.test Dec 23, 2015 5 - Production/Stable N/A + :pypi:`pytest-qt` pytest support for PyQt and PySide applications Jul 01, 2025 5 - Production/Stable pytest + :pypi:`pytest-qt-app` QT app fixture for py.test Oct 17, 2024 5 - Production/Stable pytest :pypi:`pytest-quarantine` A plugin for pytest to manage expected test failures Nov 24, 2019 5 - Production/Stable pytest (>=4.6) :pypi:`pytest-quickcheck` pytest plugin to generate random data inspired by QuickCheck Nov 05, 2022 4 - Beta pytest (>=4.0) :pypi:`pytest_quickify` Run test suites with pytest-quickify. Jun 14, 2019 N/A pytest - :pypi:`pytest-rabbitmq` RabbitMQ process and client fixtures for pytest May 08, 2024 5 - Production/Stable pytest>=6.2 + :pypi:`pytest-rabbitmq` RabbitMQ process and client fixtures for pytest Oct 15, 2024 5 - Production/Stable pytest>=6.2 :pypi:`pytest-race` Race conditions tester for pytest Jun 07, 2022 4 - Beta N/A :pypi:`pytest-rage` pytest plugin to implement PEP712 Oct 21, 2011 3 - Alpha N/A :pypi:`pytest-rail` pytest plugin for creating TestRail runs and adding results May 02, 2022 N/A pytest (>=3.6) @@ -1082,152 +1267,180 @@ This list contains 1487 plugins. :pypi:`pytest-raisesregexp` Simple pytest plugin to look for regex in Exceptions Dec 18, 2015 N/A N/A :pypi:`pytest-raisin` Plugin enabling the use of exception instances with pytest.raises Feb 06, 2022 N/A pytest :pypi:`pytest-random` py.test plugin to randomize tests Apr 28, 2013 3 - Alpha N/A - :pypi:`pytest-randomly` Pytest plugin to randomly order tests and control random.seed. Aug 15, 2023 5 - Production/Stable pytest + :pypi:`pytest-randomly` Pytest plugin to randomly order tests and control random.seed. Sep 12, 2025 5 - Production/Stable pytest :pypi:`pytest-randomness` Pytest plugin about random seed management May 30, 2019 3 - Alpha N/A :pypi:`pytest-random-num` Randomise the order in which pytest tests are run with some control over the randomness Oct 19, 2020 5 - Production/Stable N/A - :pypi:`pytest-random-order` Randomise the order in which pytest tests are run with some control over the randomness Jan 20, 2024 5 - Production/Stable pytest >=3.0.0 - :pypi:`pytest-ranking` A Pytest plugin for automatically prioritizing/ranking tests to speed up failure detection Jun 07, 2024 4 - Beta pytest>=7.4.3 - :pypi:`pytest-readme` Test your README.md file Sep 02, 2022 5 - Production/Stable N/A - :pypi:`pytest-reana` Pytest fixtures for REANA. Mar 14, 2024 3 - Alpha N/A - :pypi:`pytest-recorder` Pytest plugin, meant to facilitate unit tests writing for tools consumming Web APIs. Jun 27, 2024 N/A N/A - :pypi:`pytest-recording` A pytest plugin that allows you recording of network interactions via VCR.py Jul 09, 2024 4 - Beta pytest>=3.5.0 + :pypi:`pytest-random-order` Randomise the order in which pytest tests are run with some control over the randomness Jun 22, 2025 5 - Production/Stable pytest + :pypi:`pytest-ranking` A Pytest plugin for faster fault detection via regression test prioritization Apr 08, 2025 4 - Beta pytest>=7.4.3 + :pypi:`pytest-rca-report` Interactive RCA report generator for pytest runs, with AI-based analysis and visual dashboard Aug 04, 2025 N/A N/A + :pypi:`pytest-readme` Test your README.md file Aug 01, 2025 5 - Production/Stable pytest + :pypi:`pytest-reana` Pytest fixtures for REANA. Oct 10, 2025 3 - Alpha N/A + :pypi:`pytest-recap` Capture your test sessions. Recap the results. Jun 16, 2025 N/A pytest>=6.2.0 + :pypi:`pytest-recorder` Pytest plugin, meant to facilitate unit tests writing for tools consumming Web APIs. Oct 28, 2025 N/A pytest>=8.4.1 + :pypi:`pytest-recording` A pytest plugin powered by VCR.py to record and replay HTTP traffic May 08, 2025 4 - Beta pytest>=3.5.0 :pypi:`pytest-recordings` Provides pytest plugins for reporting request/response traffic, screenshots, and more to ReportPortal Aug 13, 2020 N/A N/A - :pypi:`pytest-redis` Redis fixtures and fixture factories for Pytest. Jun 19, 2024 5 - Production/Stable pytest>=6.2 + :pypi:`pytest-record-video` 用例执行过程中录制视频 Oct 31, 2024 N/A N/A + :pypi:`pytest-redis` Redis fixtures and fixture factories for Pytest. Nov 27, 2024 5 - Production/Stable pytest>=6.2 :pypi:`pytest-redislite` Pytest plugin for testing code using Redis Apr 05, 2022 4 - Beta pytest :pypi:`pytest-redmine` Pytest plugin for redmine Mar 19, 2018 1 - Planning N/A :pypi:`pytest-ref` A plugin to store reference files to ease regression testing Nov 23, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-reference-formatter` Conveniently run pytest with a dot-formatted test reference. Oct 01, 2019 4 - Beta N/A :pypi:`pytest-regex` Select pytest tests with regular expressions May 29, 2023 4 - Beta pytest (>=3.5.0) :pypi:`pytest-regex-dependency` Management of Pytest dependencies via regex patterns Jun 12, 2022 N/A pytest - :pypi:`pytest-regressions` Easy to use fixtures to write regression tests. Aug 31, 2023 5 - Production/Stable pytest >=6.2.0 - :pypi:`pytest-regtest` pytest plugin for snapshot regression testing Feb 26, 2024 N/A pytest>7.2 + :pypi:`pytest-regressions` Easy to use fixtures to write regression tests. Sep 05, 2025 5 - Production/Stable pytest>=6.2.0 + :pypi:`pytest-regtest` pytest plugin for snapshot regression testing Oct 11, 2025 N/A pytest>7.2 :pypi:`pytest-relative-order` a pytest plugin that sorts tests using "before" and "after" markers May 17, 2021 4 - Beta N/A + :pypi:`pytest-relative-path` Handle relative path in pytest options or ini configs Aug 30, 2024 N/A pytest :pypi:`pytest-relaxed` Relaxed test discovery/organization for pytest Mar 29, 2024 5 - Production/Stable pytest>=7 :pypi:`pytest-remfiles` Pytest plugin to create a temporary directory with remote files Jul 01, 2019 5 - Production/Stable N/A :pypi:`pytest-remotedata` Pytest plugin for controlling remote data access. Sep 26, 2023 5 - Production/Stable pytest >=4.6 :pypi:`pytest-remote-response` Pytest plugin for capturing and mocking connection requests. Apr 26, 2023 5 - Production/Stable pytest (>=4.6) :pypi:`pytest-remove-stale-bytecode` py.test plugin to remove stale byte code files. Jul 07, 2023 4 - Beta pytest :pypi:`pytest-reorder` Reorder tests depending on their paths and names. May 31, 2018 4 - Beta pytest - :pypi:`pytest-repeat` pytest plugin for repeating tests Oct 09, 2023 5 - Production/Stable pytest + :pypi:`pytest-repeat` pytest plugin for repeating tests Apr 07, 2025 5 - Production/Stable pytest :pypi:`pytest_repeater` py.test plugin for repeating single test multiple times. Feb 09, 2018 1 - Planning N/A - :pypi:`pytest-replay` Saves previous test runs and allow re-execute previous pytest runs to reproduce crashes or flaky tests Jan 11, 2024 5 - Production/Stable pytest - :pypi:`pytest-repo-health` A pytest plugin to report on repository standards conformance Apr 17, 2023 3 - Alpha pytest + :pypi:`pytest-replay` Saves previous test runs and allow re-execute previous pytest runs to reproduce crashes or flaky tests Feb 05, 2025 5 - Production/Stable pytest + :pypi:`pytest-repo-health` A pytest plugin to report on repository standards conformance May 05, 2025 3 - Alpha pytest :pypi:`pytest-report` Creates json report that is compatible with atom.io's linter message format May 11, 2016 4 - Beta N/A :pypi:`pytest-reporter` Generate Pytest reports with templates Feb 28, 2024 4 - Beta pytest - :pypi:`pytest-reporter-html1` A basic HTML report template for Pytest Jun 28, 2024 4 - Beta N/A - :pypi:`pytest-reporter-html-dots` A basic HTML report for pytest using Jinja2 template engine. Jan 22, 2023 N/A N/A + :pypi:`pytest-reporter-html1` A basic HTML report template for Pytest Oct 10, 2025 4 - Beta N/A + :pypi:`pytest-reporter-html-dots` A basic HTML report for pytest using Jinja2 template engine. Apr 26, 2025 N/A N/A + :pypi:`pytest-reporter-plus` Lightweight enhanced HTML reporter for Pytest Jul 16, 2025 N/A N/A + :pypi:`pytest-report-extras` Pytest plugin to enhance pytest-html and allure reports by adding comments, screenshots, webpage sources and attachments. Aug 08, 2025 N/A pytest>=8.4.0 :pypi:`pytest-reportinfra` Pytest plugin for reportinfra Aug 11, 2019 3 - Alpha N/A :pypi:`pytest-reporting` A plugin to report summarized results in a table format Oct 25, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-reportlog` Replacement for the --resultlog option, focused in simplicity and extensibility May 22, 2023 3 - Alpha pytest :pypi:`pytest-report-me` A pytest plugin to generate report. Dec 31, 2020 N/A pytest :pypi:`pytest-report-parameters` pytest plugin for adding tests' parameters to junit report Jun 18, 2020 3 - Alpha pytest (>=2.4.2) - :pypi:`pytest-reportportal` Agent for Reporting results of tests to the Report Portal Mar 27, 2024 N/A pytest>=3.8.0 + :pypi:`pytest-reportportal` Agent for Reporting results of tests to the Report Portal Jul 08, 2025 N/A pytest>=4.6.10 :pypi:`pytest-report-stream` A pytest plugin which allows to stream test reports at runtime Oct 22, 2023 4 - Beta N/A :pypi:`pytest-repo-structure` Pytest Repo Structure Mar 18, 2024 1 - Planning N/A + :pypi:`pytest-req` pytest requests plugin Sep 08, 2025 5 - Production/Stable pytest>=8.4.2 + :pypi:`pytest-reqcov` A pytest plugin for requirement coverage tracking Jul 04, 2025 3 - Alpha pytest>=6.0 :pypi:`pytest-reqs` pytest plugin to check pinned requirements May 12, 2019 N/A pytest (>=2.4.2) :pypi:`pytest-requests` A simple plugin to use with pytest Jun 24, 2019 4 - Beta pytest (>=3.5.0) :pypi:`pytest-requestselapsed` collect and show http requests elapsed time Aug 14, 2022 N/A N/A :pypi:`pytest-requests-futures` Pytest Plugin to Mock Requests Futures Jul 06, 2022 5 - Production/Stable pytest + :pypi:`pytest-requirements` pytest plugin for using custom markers to relate tests to requirements and usecases Feb 28, 2025 N/A pytest :pypi:`pytest-requires` A pytest plugin to elegantly skip tests with optional requirements Dec 21, 2021 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-reqyaml` This is a plugin where generate requests test cases from yaml. Aug 16, 2025 N/A pytest>=8.4.1 :pypi:`pytest-reraise` Make multi-threaded pytest test cases fail when they should Sep 20, 2022 5 - Production/Stable pytest (>=4.6) :pypi:`pytest-rerun` Re-run only changed files in specified branch Jul 08, 2019 N/A pytest (>=3.6) - :pypi:`pytest-rerun-all` Rerun testsuite for a certain time or iterations Nov 16, 2023 3 - Alpha pytest (>=7.0.0) + :pypi:`pytest-rerun-all` Rerun testsuite for a certain time or iterations Jul 30, 2025 3 - Alpha pytest>=7.0.0 :pypi:`pytest-rerunclassfailures` pytest rerun class failures plugin Apr 24, 2024 5 - Production/Stable pytest>=7.2 - :pypi:`pytest-rerunfailures` pytest plugin to re-run tests to eliminate flaky failures Mar 13, 2024 5 - Production/Stable pytest >=7.2 + :pypi:`pytest-rerunfailures` pytest plugin to re-run tests to eliminate flaky failures Oct 10, 2025 5 - Production/Stable pytest!=8.2.2,>=7.4 :pypi:`pytest-rerunfailures-all-logs` pytest plugin to re-run tests to eliminate flaky failures Mar 07, 2022 5 - Production/Stable N/A - :pypi:`pytest-reserial` Pytest fixture for recording and replaying serial port traffic. May 23, 2024 4 - Beta pytest - :pypi:`pytest-resilient-circuits` Resilient Circuits fixtures for PyTest May 17, 2024 N/A pytest~=4.6; python_version == "2.7" + :pypi:`pytest-reserial` Pytest fixture for recording and replaying serial port traffic. Dec 22, 2024 4 - Beta pytest + :pypi:`pytest-resilient-circuits` Resilient Circuits fixtures for PyTest Jul 29, 2025 N/A pytest~=7.0 :pypi:`pytest-resource` Load resource fixture plugin to use with pytest Nov 14, 2018 4 - Beta N/A - :pypi:`pytest-resource-path` Provides path for uniform access to test resources in isolated directory May 01, 2021 5 - Production/Stable pytest (>=3.5.0) + :pypi:`pytest-resource-path` Provides path for uniform access to test resources in isolated directory Sep 18, 2025 5 - Production/Stable pytest>=3.5.0 :pypi:`pytest-resource-usage` Pytest plugin for reporting running time and peak memory usage Nov 06, 2022 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-respect` Pytest plugin to load resource files relative to test code and to expect values to match them. Oct 21, 2025 5 - Production/Stable pytest>=8.0.0 :pypi:`pytest-responsemock` Simplified requests calls mocking for pytest Mar 10, 2022 5 - Production/Stable N/A :pypi:`pytest-responses` py.test integration for responses Oct 11, 2022 N/A pytest (>=2.5) :pypi:`pytest-rest-api` Aug 08, 2022 N/A pytest (>=7.1.2,<8.0.0) - :pypi:`pytest-restrict` Pytest plugin to restrict the test types allowed Jul 10, 2023 5 - Production/Stable pytest + :pypi:`pytest-restrict` Pytest plugin to restrict the test types allowed Sep 09, 2025 5 - Production/Stable pytest :pypi:`pytest-result-log` A pytest plugin that records the start, end, and result information of each use case in a log file Jan 10, 2024 N/A pytest>=7.2.0 + :pypi:`pytest-result-notify` Default template for PDM package Apr 27, 2025 N/A pytest>=8.3.5 + :pypi:`pytest-results` Easily spot regressions in your tests. Oct 08, 2025 4 - Beta pytest :pypi:`pytest-result-sender` Apr 20, 2023 N/A pytest>=7.3.1 + :pypi:`pytest-result-sender-jms` Default template for PDM package May 22, 2025 N/A pytest>=8.3.5 + :pypi:`pytest-result-sender-lj` Default template for PDM package Dec 17, 2024 N/A pytest>=8.3.4 + :pypi:`pytest-result-sender-lyt` Default template for PDM package Mar 14, 2025 N/A pytest>=8.3.5 + :pypi:`pytest-result-sender-misszhang` Default template for PDM package Mar 21, 2025 N/A pytest>=8.3.5 :pypi:`pytest-resume` A Pytest plugin to resuming from the last run test Apr 22, 2023 4 - Beta pytest (>=7.0) :pypi:`pytest-rethinkdb` A RethinkDB plugin for pytest. Jul 24, 2016 4 - Beta N/A - :pypi:`pytest-retry` Adds the ability to retry flaky tests in CI environments May 14, 2024 N/A pytest>=7.0.0 - :pypi:`pytest-retry-class` A pytest plugin to rerun entire class on failure Mar 25, 2023 N/A pytest (>=5.3) + :pypi:`pytest-retry` Adds the ability to retry flaky tests in CI environments Jan 19, 2025 N/A pytest>=7.0.0 + :pypi:`pytest-retry-class` A pytest plugin to rerun entire class on failure Nov 24, 2024 N/A pytest>=5.3 :pypi:`pytest-reusable-testcases` Apr 28, 2023 N/A N/A - :pypi:`pytest-reverse` Pytest plugin to reverse test order. Jul 10, 2023 5 - Production/Stable pytest - :pypi:`pytest-rich` Leverage rich for richer test session output Mar 03, 2022 4 - Beta pytest (>=7.0) + :pypi:`pytest-revealtype-injector` Pytest plugin for replacing reveal_type() calls inside test functions with static and runtime type checking result comparison, for confirming type annotation validity. Oct 23, 2025 4 - Beta pytest<9,>=7.0 + :pypi:`pytest-reverse` Pytest plugin to reverse test order. Sep 09, 2025 5 - Production/Stable pytest + :pypi:`pytest-rich` Leverage rich for richer test session output Dec 12, 2024 4 - Beta pytest>=7.0 :pypi:`pytest-richer` Pytest plugin providing a Rich based reporter. Oct 27, 2023 3 - Alpha pytest :pypi:`pytest-rich-reporter` A pytest plugin using Rich for beautiful test result formatting. Feb 17, 2022 1 - Planning pytest (>=5.0.0) :pypi:`pytest-richtrace` A pytest plugin that displays the names and information of the pytest hook functions as they are executed. Jun 20, 2023 N/A N/A :pypi:`pytest-ringo` pytest plugin to test webapplications using the Ringo webframework Sep 27, 2017 3 - Alpha N/A :pypi:`pytest-rmsis` Sycronise pytest results to Jira RMsis Aug 10, 2022 N/A pytest (>=5.3.5) + :pypi:`pytest-rmysql` This is a plugin which is able to connet MySQL easyly. Aug 17, 2025 N/A pytest>=8.4.1 :pypi:`pytest-rng` Fixtures for seeding tests and making randomness reproducible Aug 08, 2019 5 - Production/Stable pytest :pypi:`pytest-roast` pytest plugin for ROAST configuration override and fixtures Nov 09, 2022 5 - Production/Stable pytest - :pypi:`pytest_robotframework` a pytest plugin that can run both python and robotframework tests while generating robot reports for them Jul 01, 2024 N/A pytest<9,>=7 + :pypi:`pytest-robotframework` a pytest plugin that can run both python and robotframework tests while generating robot reports for them Oct 06, 2025 N/A pytest<9,>=7 :pypi:`pytest-rocketchat` Pytest to Rocket.Chat reporting plugin Apr 18, 2021 5 - Production/Stable N/A :pypi:`pytest-rotest` Pytest integration with rotest Sep 08, 2019 N/A pytest (>=3.5.0) :pypi:`pytest-rpc` Extend py.test for RPC OpenStack testing. Feb 22, 2019 4 - Beta pytest (~=3.6) :pypi:`pytest-rst` Test code from RST documents with pytest Jan 26, 2023 N/A N/A :pypi:`pytest-rt` pytest data collector plugin for Testgr May 05, 2022 N/A N/A :pypi:`pytest-rts` Coverage-based regression test selection (RTS) plugin for pytest May 17, 2021 N/A pytest - :pypi:`pytest-ruff` pytest plugin to check ruff requirements. Jul 09, 2024 4 - Beta pytest>=5 + :pypi:`pytest-ruff` pytest plugin to check ruff requirements. Jun 19, 2025 4 - Beta pytest>=5 :pypi:`pytest-run-changed` Pytest plugin that runs changed tests only Apr 02, 2021 3 - Alpha pytest :pypi:`pytest-runfailed` implement a --failed option for pytest Mar 24, 2016 N/A N/A + :pypi:`pytest-run-parallel` A simple pytest plugin to run tests concurrently Oct 23, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-run-subprocess` Pytest Plugin for running and testing subprocesses. Nov 12, 2022 5 - Production/Stable pytest :pypi:`pytest-runtime-types` Checks type annotations on runtime while running tests. Feb 09, 2023 N/A pytest - :pypi:`pytest-runtime-xfail` Call runtime_xfail() to mark running test as xfail. Aug 26, 2021 N/A pytest>=5.0.0 + :pypi:`pytest-runtime-xfail` Call runtime_xfail() to mark running test as xfail. Oct 10, 2025 5 - Production/Stable pytest>=5.0.0 :pypi:`pytest-runtime-yoyo` run case mark timeout Jun 12, 2023 N/A pytest (>=7.2.0) :pypi:`pytest-saccharin` pytest-saccharin is a updated fork of pytest-sugar, a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). Oct 31, 2022 3 - Alpha N/A :pypi:`pytest-salt` Pytest Salt Plugin Jan 27, 2020 4 - Beta N/A :pypi:`pytest-salt-containers` A Pytest plugin that builds and creates docker containers Nov 09, 2016 4 - Beta N/A - :pypi:`pytest-salt-factories` Pytest Salt Plugin Mar 22, 2024 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-salt-factories` Pytest Salt Plugin Jul 08, 2025 5 - Production/Stable pytest>=7.4.0 :pypi:`pytest-salt-from-filenames` Simple PyTest Plugin For Salt's Test Suite Specifically Jan 29, 2019 4 - Beta pytest (>=4.1) :pypi:`pytest-salt-runtests-bridge` Simple PyTest Plugin For Salt's Test Suite Specifically Dec 05, 2019 4 - Beta pytest (>=4.1) :pypi:`pytest-sample-argvalues` A utility function to help choose a random sample from your argvalues in pytest. May 07, 2024 N/A pytest :pypi:`pytest-sanic` a pytest plugin for Sanic Oct 25, 2021 N/A pytest (>=5.2) + :pypi:`pytest-sanitizer` A pytest plugin to sanitize output for LLMs (personal tool, no warranty or liability) Mar 16, 2025 3 - Alpha pytest>=6.0.0 :pypi:`pytest-sanity` Dec 07, 2020 N/A N/A :pypi:`pytest-sa-pg` May 14, 2019 N/A N/A :pypi:`pytest_sauce` pytest_sauce provides sane and helpful methods worked out in clearcode to run py.test tests with selenium/saucelabs Jul 14, 2014 3 - Alpha N/A - :pypi:`pytest-sbase` A complete web automation framework for end-to-end testing. Jul 08, 2024 5 - Production/Stable N/A + :pypi:`pytest-sbase` A complete web automation framework for end-to-end testing. Nov 01, 2025 5 - Production/Stable N/A :pypi:`pytest-scenario` pytest plugin for test scenarios Feb 06, 2017 3 - Alpha N/A - :pypi:`pytest-scenario-files` A pytest plugin that generates unit test scenarios from data files. May 19, 2024 5 - Production/Stable pytest>=7.2.0 - :pypi:`pytest-schedule` The job of test scheduling for humans. Jan 07, 2023 5 - Production/Stable N/A + :pypi:`pytest-scenario-files` A pytest plugin that generates unit test scenarios from data files. Sep 03, 2025 5 - Production/Stable pytest<9,>=7.4 + :pypi:`pytest-scenarios` Add your description here Oct 29, 2025 N/A N/A + :pypi:`pytest-schedule` Automate and customize test scheduling effortlessly on local machines. Oct 31, 2024 N/A N/A :pypi:`pytest-schema` 👍 Validate return values against a schema-like object in testing Feb 16, 2024 5 - Production/Stable pytest >=3.5.0 + :pypi:`pytest-scim2-server` SCIM2 server fixture for Pytest May 14, 2025 4 - Beta pytest>=8.3.4 :pypi:`pytest-screenshot-on-failure` Saves a screenshot when a test case from a pytest execution fails Jul 21, 2023 4 - Beta N/A + :pypi:`pytest-scrutinize` Scrutinize your pytest test suites for slow fixtures, tests and more. Aug 19, 2024 4 - Beta pytest>=6 :pypi:`pytest-securestore` An encrypted password store for use within pytest cases Nov 08, 2021 4 - Beta N/A :pypi:`pytest-select` A pytest plugin which allows to (de-)select tests from a file. Jan 18, 2019 3 - Alpha pytest (>=3.0) :pypi:`pytest-selenium` pytest plugin for Selenium Feb 01, 2024 5 - Production/Stable pytest>=6.0.0 :pypi:`pytest-selenium-auto` pytest plugin to automatically capture screenshots upon selenium webdriver events Nov 07, 2023 N/A pytest >= 7.0.0 - :pypi:`pytest-seleniumbase` A complete web automation framework for end-to-end testing. Jul 08, 2024 5 - Production/Stable N/A + :pypi:`pytest-seleniumbase` A complete web automation framework for end-to-end testing. Nov 01, 2025 5 - Production/Stable N/A :pypi:`pytest-selenium-enhancer` pytest plugin for Selenium Apr 29, 2022 5 - Production/Stable N/A :pypi:`pytest-selenium-pdiff` A pytest package implementing perceptualdiff for Selenium tests. Apr 06, 2017 2 - Pre-Alpha N/A - :pypi:`pytest-selfie` A pytest plugin for selfie snapshot testing. Apr 05, 2024 N/A pytest<9.0.0,>=8.0.0 - :pypi:`pytest-send-email` Send pytest execution result email Dec 04, 2019 N/A N/A - :pypi:`pytest-sentry` A pytest plugin to send testrun information to Sentry.io Apr 25, 2024 N/A pytest + :pypi:`pytest-selfie` A pytest plugin for selfie snapshot testing. Dec 16, 2024 N/A pytest>=8.0.0 + :pypi:`pytest-send-email` Send pytest execution result email Sep 02, 2024 N/A pytest + :pypi:`pytest-sentry` A pytest plugin to send testrun information to Sentry.io Jul 01, 2025 N/A pytest :pypi:`pytest-sequence-markers` Pytest plugin for sequencing markers for execution of tests May 23, 2023 5 - Production/Stable N/A - :pypi:`pytest-server` test server exec cmd Jun 24, 2024 N/A N/A - :pypi:`pytest-server-fixtures` Extensible server fixures for py.test Dec 19, 2023 5 - Production/Stable pytest + :pypi:`pytest-server` test server exec cmd Sep 09, 2024 N/A N/A + :pypi:`pytest-server-fixtures` Extensible server fixtures for py.test Nov 29, 2024 5 - Production/Stable pytest :pypi:`pytest-serverless` Automatically mocks resources from serverless.yml in pytest using moto. May 09, 2022 4 - Beta N/A - :pypi:`pytest-servers` pytest servers Jun 17, 2024 3 - Alpha pytest>=6.2 - :pypi:`pytest-service` May 11, 2024 5 - Production/Stable pytest>=6.0.0 - :pypi:`pytest-services` Services plugin for pytest testing framework Oct 30, 2020 6 - Mature N/A + :pypi:`pytest-servers` pytest servers Aug 04, 2025 3 - Alpha pytest>=6.2 + :pypi:`pytest-service` Aug 06, 2024 5 - Production/Stable pytest>=6.0.0 + :pypi:`pytest-services` Services plugin for pytest testing framework Jul 16, 2025 6 - Mature pytest :pypi:`pytest-session2file` pytest-session2file (aka: pytest-session_to_file for v0.1.0 - v0.1.2) is a py.test plugin for capturing and saving to file the stdout of py.test. Jan 26, 2021 3 - Alpha pytest :pypi:`pytest-session-fixture-globalize` py.test plugin to make session fixtures behave as if written in conftest, even if it is written in some modules May 15, 2018 4 - Beta N/A :pypi:`pytest-session_to_file` pytest-session_to_file is a py.test plugin for capturing and saving to file the stdout of py.test. Oct 01, 2015 3 - Alpha N/A :pypi:`pytest-setupinfo` Displaying setup info during pytest command run Jan 23, 2023 N/A N/A :pypi:`pytest-sftpserver` py.test plugin to locally test sftp server connections. Sep 16, 2019 4 - Beta N/A :pypi:`pytest-shard` Dec 11, 2020 4 - Beta pytest + :pypi:`pytest-shard-fork` Shard tests to support parallelism across multiple machines Jun 13, 2025 4 - Beta pytest + :pypi:`pytest-shared-session-scope` Pytest session-scoped fixture that works with xdist Oct 31, 2025 N/A pytest>=7.0.0 :pypi:`pytest-share-hdf` Plugin to save test data in HDF files and retrieve them for comparison Sep 21, 2022 4 - Beta pytest (>=3.5.0) :pypi:`pytest-sharkreport` this is pytest report plugin. Jul 11, 2022 N/A pytest (>=3.5) :pypi:`pytest-shell` A pytest plugin to help with testing shell scripts / black box commands Mar 27, 2022 N/A N/A - :pypi:`pytest-shell-utilities` Pytest plugin to simplify running shell commands against the system Feb 23, 2024 5 - Production/Stable pytest >=7.4.0 + :pypi:`pytest-shell-utilities` Pytest plugin to simplify running shell commands against the system Oct 22, 2024 5 - Production/Stable pytest>=7.4.0 :pypi:`pytest-sheraf` Versatile ZODB abstraction layer - pytest fixtures Feb 11, 2020 N/A pytest :pypi:`pytest-sherlock` pytest plugin help to find coupled tests Aug 14, 2023 5 - Production/Stable pytest >=3.5.1 :pypi:`pytest-shortcuts` Expand command-line shortcuts listed in pytest configuration Oct 29, 2020 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-shutil` A goodie-bag of unix shell and environment tools for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-shutil` A goodie-bag of unix shell and environment tools for py.test Nov 29, 2024 5 - Production/Stable pytest + :pypi:`pytest-sigil` Proper fixture resource cleanup by handling signals Oct 21, 2025 N/A pytest<9.0.0,>=7.0.0 :pypi:`pytest-simbind` Pytest plugin to operate with objects generated by Simbind tool. Mar 28, 2024 N/A pytest>=7.0.0 :pypi:`pytest-simplehttpserver` Simple pytest fixture to spin up an HTTP server Jun 24, 2021 4 - Beta N/A :pypi:`pytest-simple-plugin` Simple pytest plugin Nov 27, 2019 N/A N/A :pypi:`pytest-simple-settings` simple-settings plugin for pytest Nov 17, 2020 4 - Beta pytest :pypi:`pytest-single-file-logging` Allow for multiple processes to log to a single file May 05, 2016 4 - Beta pytest (>=2.8.1) - :pypi:`pytest-skip-markers` Pytest Salt Plugin Jan 04, 2024 5 - Production/Stable pytest >=7.1.0 + :pypi:`pytest-skip` A pytest plugin which allows to (de-)select or skip tests from a file. Sep 12, 2025 3 - Alpha pytest + :pypi:`pytest-skip-markers` Pytest Salt Plugin Aug 09, 2024 5 - Production/Stable pytest>=7.1.0 :pypi:`pytest-skipper` A plugin that selects only tests with changes in execution path Mar 26, 2017 3 - Alpha pytest (>=3.0.6) :pypi:`pytest-skippy` Automatically skip tests that don't need to run! Jan 27, 2018 3 - Alpha pytest (>=2.3.4) :pypi:`pytest-skip-slow` A pytest plugin to skip \`@pytest.mark.slow\` tests by default. Feb 09, 2023 N/A pytest>=6.2.0 @@ -1236,114 +1449,133 @@ This list contains 1487 plugins. :pypi:`pytest-slow` A pytest plugin to skip \`@pytest.mark.slow\` tests by default. Sep 28, 2021 N/A N/A :pypi:`pytest-slowest-first` Sort tests by their last duration, slowest first Dec 11, 2022 4 - Beta N/A :pypi:`pytest-slow-first` Prioritize running the slowest tests first. Jan 30, 2024 4 - Beta pytest >=3.5.0 - :pypi:`pytest-slow-last` Run tests in order of execution time (faster tests first) Dec 10, 2022 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-slow-last` Run tests in order of execution time (faster tests first) Mar 16, 2025 4 - Beta pytest>=3.5.0 :pypi:`pytest-smartcollect` A plugin for collecting tests that touch changed code Oct 04, 2018 N/A pytest (>=3.5.0) :pypi:`pytest-smartcov` Smart coverage plugin for pytest. Sep 30, 2017 3 - Alpha N/A + :pypi:`pytest-smart-debugger-backend` Backend server for Pytest Smart Debugger Sep 17, 2025 N/A N/A + :pypi:`pytest-smart-rerun` A Pytest plugin for intelligent retrying of flaky tests. Oct 12, 2025 3 - Alpha N/A :pypi:`pytest-smell` Automated bad smell detection tool for Pytest Jun 26, 2022 N/A N/A + :pypi:`pytest-smoke` Pytest plugin for smoke testing Oct 08, 2025 4 - Beta pytest<9,>=7.0.0 :pypi:`pytest-smtp` Send email with pytest execution result Feb 20, 2021 N/A pytest :pypi:`pytest-smtp4dev` Plugin for smtp4dev API Jun 27, 2023 5 - Production/Stable N/A :pypi:`pytest-smtpd` An SMTP server for testing built on aiosmtpd May 15, 2023 N/A pytest :pypi:`pytest-smtp-test-server` pytest plugin for using \`smtp-test-server\` as a fixture Dec 03, 2023 2 - Pre-Alpha pytest (>=7.4.3,<8.0.0) :pypi:`pytest-snail` Plugin for adding a marker to slow running tests. 🐌 Nov 04, 2019 3 - Alpha pytest (>=5.0.1) + :pypi:`pytest-snap` A text-based snapshot testing library implemented as a pytest plugin Aug 25, 2025 N/A pytest>=8.0.0 + :pypi:`pytest-snapcheck` Minimal deterministic test-run snapshot capture for pytest. Sep 07, 2025 N/A pytest>=8.0 :pypi:`pytest-snapci` py.test plugin for Snap-CI Nov 12, 2015 N/A N/A + :pypi:`pytest-snapmock` Snapshots for your mocks. Nov 15, 2024 N/A N/A :pypi:`pytest-snapshot` A plugin for snapshot testing with pytest. Apr 23, 2022 4 - Beta pytest (>=3.0.0) :pypi:`pytest-snapshot-with-message-generator` A plugin for snapshot testing with pytest. Jul 25, 2023 4 - Beta pytest (>=3.0.0) :pypi:`pytest-snmpserver` May 12, 2021 N/A N/A + :pypi:`pytest-snob` A pytest plugin that only selects meaningful python tests to run. Jan 12, 2025 N/A pytest :pypi:`pytest-snowflake-bdd` Setup test data and run tests on snowflake in BDD style! Jan 05, 2022 4 - Beta pytest (>=6.2.0) :pypi:`pytest-socket` Pytest Plugin to disable socket calls during tests Jan 28, 2024 4 - Beta pytest (>=6.2.5) :pypi:`pytest-sofaepione` Test the installation of SOFA and the SofaEpione plugin. Aug 17, 2022 N/A N/A :pypi:`pytest-soft-assertions` May 05, 2020 3 - Alpha pytest :pypi:`pytest-solidity` A PyTest library plugin for Solidity language. Jan 15, 2022 1 - Planning pytest (<7,>=6.0.1) ; extra == 'tests' :pypi:`pytest-solr` Solr process and client fixtures for py.test. May 11, 2020 3 - Alpha pytest (>=3.0.0) - :pypi:`pytest-sort` Tools for sorting test cases Jan 07, 2024 N/A pytest >=7.4.0 + :pypi:`pytest-sort` Tools for sorting test cases Mar 22, 2025 N/A pytest>=7.4.0 :pypi:`pytest-sorter` A simple plugin to first execute tests that historically failed more Apr 20, 2021 4 - Beta pytest (>=3.1.1) :pypi:`pytest-sosu` Unofficial PyTest plugin for Sauce Labs Aug 04, 2023 2 - Pre-Alpha pytest :pypi:`pytest-sourceorder` Test-ordering plugin for pytest Sep 01, 2021 4 - Beta pytest - :pypi:`pytest-spark` pytest plugin to run the tests with support of pyspark. Feb 23, 2020 4 - Beta pytest + :pypi:`pytest-spark` pytest plugin to run the tests with support of pyspark. May 21, 2025 4 - Beta pytest :pypi:`pytest-spawner` py.test plugin to spawn process and communicate with them. Jul 31, 2015 4 - Beta N/A - :pypi:`pytest-spec` Library pytest-spec is a pytest plugin to display test execution output like a SPECIFICATION. May 04, 2021 N/A N/A + :pypi:`pytest-spec` Library pytest-spec is a pytest plugin to display test execution output like a SPECIFICATION. Oct 08, 2025 N/A pytest; extra == "test" :pypi:`pytest-spec2md` Library pytest-spec2md is a pytest plugin to create a markdown specification while running pytest. Apr 10, 2024 N/A pytest>7.0 :pypi:`pytest-speed` Modern benchmarking library for python with pytest integration. Jan 22, 2023 3 - Alpha pytest>=7 :pypi:`pytest-sphinx` Doctest plugin for pytest with support for Sphinx-specific doctest-directives Apr 13, 2024 4 - Beta pytest>=8.1.1 :pypi:`pytest-spiratest` Exports unit tests as test runs in Spira (SpiraTest/Team/Plan) Jan 01, 2024 N/A N/A :pypi:`pytest-splinter` Splinter plugin for pytest testing framework Sep 09, 2022 6 - Mature pytest (>=3.0.0) :pypi:`pytest-splinter4` Pytest plugin for the splinter automation library Feb 01, 2024 6 - Mature pytest >=8.0.0 - :pypi:`pytest-split` Pytest plugin which splits the test suite to equally sized sub suites based on test execution time. Jun 19, 2024 4 - Beta pytest<9,>=5 + :pypi:`pytest-split` Pytest plugin which splits the test suite to equally sized sub suites based on test execution time. Oct 16, 2024 4 - Beta pytest<9,>=5 :pypi:`pytest-split-ext` Pytest plugin which splits the test suite to equally sized sub suites based on test execution time. Sep 23, 2023 4 - Beta pytest (>=5,<8) :pypi:`pytest-splitio` Split.io SDK integration for e2e tests Sep 22, 2020 N/A pytest (<7,>=5.0) :pypi:`pytest-split-tests` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Forked from Mark Adams' original project pytest-test-groups. Jul 30, 2021 5 - Production/Stable pytest (>=2.5) :pypi:`pytest-split-tests-tresorit` Feb 22, 2021 1 - Planning N/A - :pypi:`pytest-splunk-addon` A Dynamic test tool for Splunk Apps and Add-ons Jul 11, 2024 N/A pytest<8,>5.4.0 - :pypi:`pytest-splunk-addon-ui-smartx` Library to support testing Splunk Add-on UX Jul 10, 2024 N/A N/A + :pypi:`pytest-splunk-addon` A Dynamic test tool for Splunk Apps and Add-ons Aug 19, 2025 N/A pytest<8,>5.4.0 + :pypi:`pytest-splunk-addon-ui-smartx` Library to support testing Splunk Add-on UX Aug 28, 2025 N/A N/A :pypi:`pytest-splunk-env` pytest fixtures for interaction with Splunk Enterprise and Splunk Cloud Oct 22, 2020 N/A pytest (>=6.1.1,<7.0.0) :pypi:`pytest-sqitch` sqitch for pytest Apr 06, 2020 4 - Beta N/A - :pypi:`pytest-sqlalchemy` pytest plugin with sqlalchemy related fixtures Mar 13, 2018 3 - Alpha N/A - :pypi:`pytest-sqlalchemy-mock` pytest sqlalchemy plugin for mock May 21, 2024 3 - Alpha pytest>=7.0.0 + :pypi:`pytest-sqlalchemy` pytest plugin with sqlalchemy related fixtures Apr 19, 2025 3 - Alpha pytest>=8.0 + :pypi:`pytest-sqlalchemy-mock` pytest sqlalchemy plugin for mock Aug 10, 2024 3 - Alpha pytest>=7.0.0 :pypi:`pytest-sqlalchemy-session` A pytest plugin for preserving test isolation that use SQLAlchemy. May 19, 2023 4 - Beta pytest (>=7.0) :pypi:`pytest-sql-bigquery` Yet another SQL-testing framework for BigQuery provided by pytest plugin Dec 19, 2019 N/A pytest :pypi:`pytest-sqlfluff` A pytest plugin to use sqlfluff to enable format checking of sql files. Dec 21, 2022 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-sqlguard` Pytest fixture to record and check SQL Queries made by SQLAlchemy Jun 06, 2025 4 - Beta pytest>=7 :pypi:`pytest-squadcast` Pytest report plugin for Squadcast Feb 22, 2022 5 - Production/Stable pytest :pypi:`pytest-srcpaths` Add paths to sys.path Oct 15, 2021 N/A pytest>=6.2.0 :pypi:`pytest-ssh` pytest plugin for ssh command run May 27, 2019 N/A pytest :pypi:`pytest-start-from` Start pytest run from a given point Apr 11, 2016 N/A N/A - :pypi:`pytest-star-track-issue` A package to prevent Dependency Confusion attacks against Yandex. Feb 20, 2024 N/A N/A - :pypi:`pytest-static` pytest-static Jun 20, 2024 1 - Planning pytest<8.0.0,>=7.4.3 - :pypi:`pytest-stats` Collects tests metadata for future analysis, easy to extend for any data store Jul 03, 2024 N/A pytest>=8.0.0 + :pypi:`pytest-static` pytest-static May 25, 2025 3 - Alpha pytest<8.0.0,>=7.4.3 + :pypi:`pytest-stats` Collects tests metadata for future analysis, easy to extend for any data store Jul 18, 2024 N/A pytest>=8.0.0 :pypi:`pytest-statsd` pytest plugin for reporting to graphite Nov 30, 2018 5 - Production/Stable pytest (>=3.0.0) + :pypi:`pytest-status` Add status mark for tests Aug 22, 2024 N/A pytest + :pypi:`pytest-stderr-db` Add your description here Sep 14, 2025 N/A N/A + :pypi:`pytest-stdout-db` Add your description here Sep 14, 2025 N/A N/A :pypi:`pytest-stepfunctions` A small description May 08, 2021 4 - Beta pytest :pypi:`pytest-steps` Create step-wise / incremental tests in pytest. Sep 23, 2021 5 - Production/Stable N/A + :pypi:`pytest-stepthrough` Pause and wait for Enter after each test with --step Aug 14, 2025 N/A N/A :pypi:`pytest-stepwise` Run a test suite one failing test at a time. Dec 01, 2015 4 - Beta N/A - :pypi:`pytest-stf` pytest plugin for openSTF Mar 25, 2024 N/A pytest>=5.0 + :pypi:`pytest-stf` pytest plugin for openSTF Sep 23, 2025 N/A pytest>=5.0 + :pypi:`pytest-stochastics` pytest plugin that allows selectively running tests several times and accepting \*some\* failures. Dec 01, 2024 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-stoq` A plugin to pytest stoq Feb 09, 2021 4 - Beta N/A - :pypi:`pytest-store` Pytest plugin to store values from test runs Nov 16, 2023 3 - Alpha pytest (>=7.0.0) + :pypi:`pytest-storage` Pytest plugin to store test artifacts Sep 12, 2025 3 - Alpha pytest>=8.4.2 + :pypi:`pytest-store` Pytest plugin to store values from test runs Jul 30, 2025 3 - Alpha pytest>=7.0.0 + :pypi:`pytest-streaming` Plugin for testing pubsub, pulsar, and kafka systems with pytest locally and in ci/cd May 28, 2025 5 - Production/Stable pytest>=8.3.5 :pypi:`pytest-stress` A Pytest plugin that allows you to loop tests for a user defined amount of time. Dec 07, 2019 4 - Beta pytest (>=3.6.0) - :pypi:`pytest-structlog` Structured logging assertions Jun 09, 2024 N/A pytest + :pypi:`pytest-structlog` Structured logging assertions Sep 10, 2025 N/A pytest :pypi:`pytest-structmpd` provide structured temporary directory Oct 17, 2018 N/A N/A :pypi:`pytest-stub` Stub packages, modules and attributes. Apr 28, 2020 5 - Production/Stable N/A :pypi:`pytest-stubprocess` Provide stub implementations for subprocesses in Python tests Sep 17, 2018 3 - Alpha pytest (>=3.5.0) :pypi:`pytest-study` A pytest plugin to organize long run tests (named studies) without interfering the regular tests Sep 26, 2017 3 - Alpha pytest (>=2.0) :pypi:`pytest-subinterpreter` Run pytest in a subinterpreter Nov 25, 2023 N/A pytest>=7.0.0 - :pypi:`pytest-subprocess` A plugin to fake subprocess for pytest Jan 28, 2023 5 - Production/Stable pytest (>=4.0.0) + :pypi:`pytest-subket` Pytest Plugin to disable socket calls during tests Jul 31, 2025 4 - Beta N/A + :pypi:`pytest-subprocess` A plugin to fake subprocess for pytest Jan 04, 2025 5 - Production/Stable pytest>=4.0.0 :pypi:`pytest-subtesthack` A hack to explicitly set up and tear down fixtures. Jul 16, 2022 N/A N/A - :pypi:`pytest-subtests` unittest subTest() support and subtests fixture Jul 07, 2024 4 - Beta pytest>=7.0 + :pypi:`pytest-subtests` unittest subTest() support and subtests fixture Oct 20, 2025 4 - Beta pytest>=7.4 :pypi:`pytest-subunit` pytest-subunit is a plugin for py.test which outputs testsresult in subunit format. Sep 17, 2023 N/A pytest (>=2.3) - :pypi:`pytest-sugar` pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). Feb 01, 2024 4 - Beta pytest >=6.2.0 + :pypi:`pytest-sugar` pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). Aug 23, 2025 4 - Beta pytest>=6.2.0 :pypi:`pytest-suitemanager` A simple plugin to use with pytest Apr 28, 2023 4 - Beta N/A :pypi:`pytest-suite-timeout` A pytest plugin for ensuring max suite time Jan 26, 2024 N/A pytest>=7.0.0 :pypi:`pytest-supercov` Pytest plugin for measuring explicit test-file to source-file coverage Jul 02, 2023 N/A N/A - :pypi:`pytest-svn` SVN repository fixture for py.test May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-svn` SVN repository fixture for py.test Oct 17, 2024 5 - Production/Stable pytest :pypi:`pytest-symbols` pytest-symbols is a pytest plugin that adds support for passing test environment symbols into pytest tests. Nov 20, 2017 3 - Alpha N/A - :pypi:`pytest-synodic` Synodic Pytest utilities Mar 09, 2024 N/A pytest>=8.0.2 :pypi:`pytest-system-statistics` Pytest plugin to track and report system usage statistics Feb 16, 2022 5 - Production/Stable pytest (>=6.0.0) :pypi:`pytest-system-test-plugin` Pyst - Pytest System-Test Plugin Feb 03, 2022 N/A N/A - :pypi:`pytest_tagging` a pytest plugin to tag tests Apr 08, 2024 N/A pytest<8.0.0,>=7.1.3 - :pypi:`pytest-takeltest` Fixtures for ansible, testinfra and molecule Feb 15, 2023 N/A N/A + :pypi:`pytest_tagging` a pytest plugin to tag tests Nov 08, 2024 N/A pytest>=7.1.3 + :pypi:`pytest-takeltest` Fixtures for ansible, testinfra and molecule Sep 07, 2024 N/A N/A :pypi:`pytest-talisker` Nov 28, 2021 N/A N/A :pypi:`pytest-tally` A Pytest plugin to generate realtime summary stats, and display them in-console using a text-based dashboard. May 22, 2023 4 - Beta pytest (>=6.2.5) - :pypi:`pytest-tap` Test Anything Protocol (TAP) reporting plugin for pytest Jul 15, 2023 5 - Production/Stable pytest (>=3.0) + :pypi:`pytest-tap` Test Anything Protocol (TAP) reporting plugin for pytest Jan 30, 2025 5 - Production/Stable pytest>=3.0 :pypi:`pytest-tape` easy assertion with expected results saved to yaml files Mar 17, 2021 4 - Beta N/A :pypi:`pytest-target` Pytest plugin for remote target orchestration. Jan 21, 2021 3 - Alpha pytest (>=6.1.2,<7.0.0) + :pypi:`pytest-taskgraph` Add your description here Apr 09, 2025 N/A pytest :pypi:`pytest-tblineinfo` tblineinfo is a py.test plugin that insert the node id in the final py.test report when --tb=line option is used Dec 01, 2015 3 - Alpha pytest (>=2.0) :pypi:`pytest-tcpclient` A pytest plugin for testing TCP clients Nov 16, 2022 N/A pytest (<8,>=7.1.3) :pypi:`pytest-tdd` run pytest on a python module Aug 18, 2023 4 - Beta N/A :pypi:`pytest-teamcity-logblock` py.test plugin to introduce block structure in teamcity build log, if output is not captured May 15, 2018 4 - Beta N/A + :pypi:`pytest-teardown` Apr 15, 2025 N/A pytest<9.0.0,>=7.4.1 :pypi:`pytest-telegram` Pytest to Telegram reporting plugin Apr 25, 2024 5 - Production/Stable N/A :pypi:`pytest-telegram-notifier` Telegram notification plugin for Pytest Jun 27, 2023 5 - Production/Stable N/A :pypi:`pytest-tempdir` Predictable and repeatable tempdir support. Oct 11, 2019 4 - Beta pytest (>=2.8.1) :pypi:`pytest-terra-fixt` Terraform and Terragrunt fixtures for pytest Sep 15, 2022 N/A pytest (==6.2.5) :pypi:`pytest-terraform` A pytest plugin for using terraform fixtures May 21, 2024 N/A pytest>=6.0 :pypi:`pytest-terraform-fixture` generate terraform resources to use with pytest Nov 14, 2018 4 - Beta N/A + :pypi:`pytest-test-analyzer` A powerful tool for analyzing pytest test files and generating detailed reports Jun 14, 2025 4 - Beta N/A :pypi:`pytest-testbook` A plugin to run tests written in Jupyter notebook Dec 11, 2016 3 - Alpha N/A :pypi:`pytest-testconfig` Test configuration plugin for pytest. Jan 11, 2020 4 - Beta pytest (>=3.5.0) + :pypi:`pytest-testdata` Get and load testdata in pytest projects Aug 30, 2024 N/A pytest :pypi:`pytest-testdirectory` A py.test plugin providing temporary directories in unit tests. May 02, 2023 5 - Production/Stable pytest :pypi:`pytest-testdox` A testdox format reporter for pytest Jul 22, 2023 5 - Production/Stable pytest (>=4.6.0) :pypi:`pytest-test-grouping` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Feb 01, 2023 5 - Production/Stable pytest (>=2.5) - :pypi:`pytest-test-groups` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. Oct 25, 2016 5 - Production/Stable N/A - :pypi:`pytest-testinfra` Test infrastructures May 26, 2024 5 - Production/Stable pytest>=6 + :pypi:`pytest-test-groups` A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. May 08, 2025 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-testinfra` Test infrastructures Mar 30, 2025 5 - Production/Stable pytest>=6 :pypi:`pytest-testinfra-jpic` Test infrastructures Sep 21, 2023 5 - Production/Stable N/A :pypi:`pytest-testinfra-winrm-transport` Test infrastructures Sep 21, 2023 5 - Production/Stable N/A + :pypi:`pytest-testit-parametrize` A pytest plugin for uploading parameterized tests parameters into TMS TestIT Dec 04, 2024 4 - Beta pytest>=8.3.3 :pypi:`pytest-testlink-adaptor` pytest reporting plugin for testlink Dec 20, 2018 4 - Beta pytest (>=2.6) - :pypi:`pytest-testmon` selects tests affected by changed files and methods Feb 27, 2024 4 - Beta pytest <9,>=5 + :pypi:`pytest-testmon` selects tests affected by changed files and methods Dec 22, 2024 4 - Beta pytest<9,>=5 :pypi:`pytest-testmon-dev` selects tests affected by changed files and methods Mar 30, 2023 4 - Beta pytest (<8,>=5) :pypi:`pytest-testmon-oc` nOly selects tests affected by changed files and methods Jun 01, 2022 4 - Beta pytest (<8,>=5) :pypi:`pytest-testmon-skip-libraries` selects tests affected by changed files and methods Mar 03, 2023 4 - Beta pytest (<8,>=5) @@ -1351,6 +1583,7 @@ This list contains 1487 plugins. :pypi:`pytest-testpluggy` set your encoding Jan 07, 2022 N/A pytest :pypi:`pytest-testrail` pytest plugin for creating TestRail runs and adding results Aug 27, 2020 N/A pytest (>=3.6) :pypi:`pytest-testrail2` A pytest plugin to upload results to TestRail. Feb 10, 2023 N/A pytest (<8.0,>=7.2.0) + :pypi:`pytest-testrail-api` TestRail Api Python Client Mar 17, 2025 N/A pytest :pypi:`pytest-testrail-api-client` TestRail Api Python Client Dec 14, 2021 N/A pytest :pypi:`pytest-testrail-appetize` pytest plugin for creating TestRail runs and adding results Sep 29, 2021 N/A N/A :pypi:`pytest-testrail-client` pytest plugin for Testrail Sep 29, 2020 5 - Production/Stable N/A @@ -1365,10 +1598,10 @@ This list contains 1487 plugins. :pypi:`pytest-testslide` TestSlide fixture for pytest Jan 07, 2021 5 - Production/Stable pytest (~=6.2) :pypi:`pytest-test-this` Plugin for py.test to run relevant tests, based on naively checking if a test contains a reference to the symbol you supply Sep 15, 2019 2 - Pre-Alpha pytest (>=2.3) :pypi:`pytest-test-tracer-for-pytest` A plugin that allows coll test data for use on Test Tracer Jun 28, 2024 4 - Beta pytest>=6.2.0 - :pypi:`pytest-test-tracer-for-pytest-bdd` A plugin that allows coll test data for use on Test Tracer Jul 01, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-test-tracer-for-pytest-bdd` A plugin that allows coll test data for use on Test Tracer Aug 20, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-test-utils` Feb 08, 2024 N/A pytest >=3.9 - :pypi:`pytest-tesults` Tesults plugin for pytest Feb 15, 2024 5 - Production/Stable pytest >=3.5.0 - :pypi:`pytest-textual-snapshot` Snapshot testing for Textual apps Aug 23, 2023 4 - Beta pytest (>=7.0.0) + :pypi:`pytest-tesults` Tesults plugin for pytest Nov 12, 2024 5 - Production/Stable pytest>=3.5.0 + :pypi:`pytest-textual-snapshot` Snapshot testing for Textual apps Jan 23, 2025 5 - Production/Stable pytest>=8.0.0 :pypi:`pytest-tezos` pytest-ligo Jan 16, 2020 4 - Beta N/A :pypi:`pytest-tf` Test your OpenTofu and Terraform config using a PyTest plugin May 29, 2024 N/A pytest<9.0.0,>=8.2.1 :pypi:`pytest-th2-bdd` pytest_th2_bdd May 13, 2022 N/A N/A @@ -1376,16 +1609,17 @@ This list contains 1487 plugins. :pypi:`pytest-thread` Jul 07, 2023 N/A N/A :pypi:`pytest-threadleak` Detects thread leaks Jul 03, 2022 4 - Beta pytest (>=3.1.1) :pypi:`pytest-tick` Ticking on tests Aug 31, 2021 5 - Production/Stable pytest (>=6.2.5,<7.0.0) - :pypi:`pytest-time` Jun 24, 2023 3 - Alpha pytest + :pypi:`pytest-time` Jan 20, 2025 3 - Alpha pytest :pypi:`pytest-timeassert-ethan` execution duration Dec 25, 2023 N/A pytest :pypi:`pytest-timeit` A pytest plugin to time test function runs Oct 13, 2016 4 - Beta N/A - :pypi:`pytest-timeout` pytest plugin to abort hanging tests Mar 07, 2024 5 - Production/Stable pytest >=7.0.0 + :pypi:`pytest-timeout` pytest plugin to abort hanging tests May 05, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-timeouts` Linux-only Pytest plugin to control durations of various test case execution phases Sep 21, 2019 5 - Production/Stable N/A :pypi:`pytest-timer` A timer plugin for pytest Dec 26, 2023 N/A pytest :pypi:`pytest-timestamper` Pytest plugin to add a timestamp prefix to the pytest output Mar 27, 2024 N/A N/A :pypi:`pytest-timestamps` A simple plugin to view timestamps for each test Sep 11, 2023 N/A pytest (>=7.3,<8.0) + :pypi:`pytest-timing-plugin` pytest插件开发demo Jul 21, 2025 N/A N/A :pypi:`pytest-tiny-api-client` The companion pytest plugin for tiny-api-client Jan 04, 2024 5 - Production/Stable pytest - :pypi:`pytest-tinybird` A pytest plugin to report test results to tinybird Jun 26, 2023 4 - Beta pytest (>=3.8.0) + :pypi:`pytest-tinybird` A pytest plugin to report test results to tinybird May 07, 2025 4 - Beta pytest>=3.8.0 :pypi:`pytest-tipsi-django` Better fixtures for django Feb 05, 2024 5 - Production/Stable pytest>=6.0.0 :pypi:`pytest-tipsi-testing` Better fixtures management. Various helpers Feb 04, 2024 5 - Production/Stable pytest>=3.3.0 :pypi:`pytest-tldr` A pytest plugin that limits the output to just the things you need. Oct 26, 2022 4 - Beta pytest (>=3.5.0) @@ -1394,7 +1628,7 @@ This list contains 1487 plugins. :pypi:`pytest-tmp-files` Utilities to create temporary file hierarchies in pytest. Dec 08, 2023 N/A pytest :pypi:`pytest-tmpfs` A pytest plugin that helps you on using a temporary filesystem for testing. Aug 29, 2022 N/A pytest :pypi:`pytest-tmreport` this is a vue-element ui report for pytest Aug 12, 2022 N/A N/A - :pypi:`pytest-tmux` A pytest plugin that enables tmux driven tests Apr 22, 2023 4 - Beta N/A + :pypi:`pytest-tmux` A pytest plugin that enables tmux driven tests Sep 01, 2025 4 - Beta N/A :pypi:`pytest-todo` A small plugin for the pytest testing framework, marking TODO comments as failure May 23, 2019 4 - Beta pytest :pypi:`pytest-tomato` Mar 01, 2019 5 - Production/Stable N/A :pypi:`pytest-toolbelt` This is just a collection of utilities for pytest, but don't really belong in pytest proper. Aug 12, 2019 3 - Alpha N/A @@ -1411,7 +1645,7 @@ This list contains 1487 plugins. :pypi:`pytest-translations` Test your translation files. Sep 11, 2023 5 - Production/Stable pytest (>=7) :pypi:`pytest-travis-fold` Folds captured output sections in Travis CI build log Nov 29, 2017 4 - Beta pytest (>=2.6.0) :pypi:`pytest-trello` Plugin for py.test that integrates trello using markers Nov 20, 2015 5 - Production/Stable N/A - :pypi:`pytest-trepan` Pytest plugin for trepan debugger. Jul 28, 2018 5 - Production/Stable N/A + :pypi:`pytest-trepan` Pytest plugin for trepan debugger. Sep 11, 2025 5 - Production/Stable pytest>=4.0.0 :pypi:`pytest-trialtemp` py.test plugin for using the same _trial_temp working directory as trial Jun 08, 2015 N/A N/A :pypi:`pytest-trio` Pytest plugin for trio Nov 01, 2022 N/A pytest (>=7.2.0) :pypi:`pytest-trytond` Pytest plugin for the Tryton server framework Nov 04, 2022 4 - Beta pytest (>=5) @@ -1419,27 +1653,36 @@ This list contains 1487 plugins. :pypi:`pytest-tst` Customize pytest options, output and exit code to make it compatible with tst Apr 27, 2022 N/A pytest (>=5.0.0) :pypi:`pytest-tstcls` Test Class Base Mar 23, 2020 5 - Production/Stable N/A :pypi:`pytest-tui` Text User Interface (TUI) and HTML report for Pytest test runs Dec 08, 2023 4 - Beta N/A + :pypi:`pytest-tui-runner` Textual-based terminal UI for running pytest tests Oct 23, 2025 N/A pytest>=8.3.5 + :pypi:`pytest-tuitest` pytest plugin for testing TUI and regular command-line applications. Apr 11, 2025 N/A pytest>=7.4.0 :pypi:`pytest-tutorials` Mar 11, 2023 N/A N/A :pypi:`pytest-twilio-conversations-client-mock` Aug 02, 2022 N/A N/A - :pypi:`pytest-twisted` A twisted plugin for pytest. Jul 10, 2024 5 - Production/Stable pytest>=2.3 + :pypi:`pytest-twisted` A twisted plugin for pytest. Sep 10, 2024 5 - Production/Stable pytest>=2.3 + :pypi:`pytest-ty` A pytest plugin to run the ty type checker Oct 10, 2025 3 - Alpha pytest>=7.0.0 :pypi:`pytest-typechecker` Run type checkers on specified test files Feb 04, 2022 N/A pytest (>=6.2.5,<7.0.0) + :pypi:`pytest-typed-schema-shot` Pytest plugin for automatic JSON Schema generation and validation from examples Jun 14, 2025 N/A pytest :pypi:`pytest-typhoon-config` A Typhoon HIL plugin that facilitates test parameter configuration at runtime Apr 07, 2022 5 - Production/Stable N/A :pypi:`pytest-typhoon-polarion` Typhoontest plugin for Siemens Polarion Feb 01, 2024 4 - Beta N/A :pypi:`pytest-typhoon-xray` Typhoon HIL plugin for pytest Aug 15, 2023 4 - Beta N/A + :pypi:`pytest-typing-runner` Pytest plugin to make it easier to run and check python code against static type checkers May 31, 2025 N/A N/A :pypi:`pytest-tytest` Typhoon HIL plugin for pytest May 25, 2020 4 - Beta pytest (>=5.4.2) + :pypi:`pytest-tzshift` A Pytest plugin that transparently re-runs tests under a matrix of timezones and locales. Jun 25, 2025 4 - Beta pytest>=7.0 :pypi:`pytest-ubersmith` Easily mock calls to ubersmith at the \`requests\` level. Apr 13, 2015 N/A N/A :pypi:`pytest-ui` Text User Interface for running python tests Jul 05, 2021 4 - Beta pytest :pypi:`pytest-ui-failed-screenshot` UI自动测试失败时自动截图,并将截图加入到测试报告中 Dec 06, 2022 N/A N/A :pypi:`pytest-ui-failed-screenshot-allure` UI自动测试失败时自动截图,并将截图加入到Allure测试报告中 Dec 06, 2022 N/A N/A - :pypi:`pytest-uncollect-if` A plugin to uncollect pytests tests rather than using skipif Mar 24, 2024 4 - Beta pytest>=6.2.0 + :pypi:`pytest-uncollect-if` A plugin to uncollect pytests tests rather than using skipif Dec 26, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-unflakable` Unflakable plugin for PyTest Apr 30, 2024 4 - Beta pytest>=6.2.0 :pypi:`pytest-unhandled-exception-exit-code` Plugin for py.test set a different exit code on uncaught exceptions Jun 22, 2020 5 - Production/Stable pytest (>=2.3) - :pypi:`pytest-unique` Pytest fixture to generate unique values. Sep 15, 2023 N/A pytest (>=7.4.2,<8.0.0) + :pypi:`pytest-unique` Pytest fixture to generate unique values. Jun 10, 2025 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-unittest-filter` A pytest plugin for filtering unittest-based test classes Jan 12, 2019 4 - Beta pytest (>=3.1.0) + :pypi:`pytest-unittest-id-runner` A pytest plugin to run tests using unittest-style test IDs Feb 09, 2025 N/A pytest>=6.0.0 + :pypi:`pytest-unmagic` Pytest fixtures with conventional import semantics Jul 14, 2025 5 - Production/Stable pytest :pypi:`pytest-unmarked` Run only unmarked tests Aug 27, 2019 5 - Production/Stable N/A - :pypi:`pytest-unordered` Test equality of unordered collections in pytest Jul 05, 2024 4 - Beta pytest>=7.0.0 + :pypi:`pytest-unordered` Test equality of unordered collections in pytest Jun 03, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-unstable` Set a test as unstable to return 0 even if it failed Sep 27, 2022 4 - Beta N/A - :pypi:`pytest-unused-fixtures` A pytest plugin to list unused fixtures after a test run. Apr 08, 2024 4 - Beta pytest>7.3.2 + :pypi:`pytest-unused-fixtures` A pytest plugin to list unused fixtures after a test run. Mar 15, 2025 4 - Beta pytest>7.3.2 + :pypi:`pytest-unused-port` pytest fixture finding an unused local port Oct 22, 2025 N/A pytest :pypi:`pytest-upload-report` pytest-upload-report is a plugin for pytest that upload your test report for test results. Jun 18, 2021 5 - Production/Stable N/A :pypi:`pytest-utils` Some helpers for pytest. Feb 02, 2023 4 - Beta pytest (>=7.0.0,<8.0.0) :pypi:`pytest-vagrant` A py.test plugin providing access to vagrant. Sep 07, 2021 5 - Production/Stable pytest @@ -1451,64 +1694,76 @@ This list contains 1487 plugins. :pypi:`pytest-vcrpandas` Test from HTTP interactions to dataframe processed. Jan 12, 2019 4 - Beta pytest :pypi:`pytest-vcs` Sep 22, 2022 4 - Beta N/A :pypi:`pytest-venv` py.test fixture for creating a virtual environment Nov 23, 2023 4 - Beta pytest - :pypi:`pytest-verbose-parametrize` More descriptive output for parametrized py.test tests May 28, 2019 5 - Production/Stable pytest + :pypi:`pytest-verbose-parametrize` More descriptive output for parametrized py.test tests Nov 29, 2024 5 - Production/Stable pytest + :pypi:`pytest-verify` A pytest plugin for snapshot verification with optional visual diff viewer. Oct 25, 2025 5 - Production/Stable N/A :pypi:`pytest-vimqf` A simple pytest plugin that will shrink pytest output when specified, to fit vim quickfix window. Feb 08, 2021 4 - Beta pytest (>=6.2.2,<7.0.0) - :pypi:`pytest-virtualenv` Virtualenv fixture for py.test May 28, 2019 5 - Production/Stable pytest - :pypi:`pytest-visual` Nov 01, 2023 3 - Alpha pytest >=7.0.0 + :pypi:`pytest-virtualenv` Virtualenv fixture for py.test Nov 29, 2024 5 - Production/Stable pytest + :pypi:`pytest-visual` Nov 28, 2024 4 - Beta pytest>=7.0.0 :pypi:`pytest-vnc` VNC client for Pytest Nov 06, 2023 N/A pytest :pypi:`pytest-voluptuous` Pytest plugin for asserting data against voluptuous schema. Jun 09, 2020 N/A pytest :pypi:`pytest-vscodedebug` A pytest plugin to easily enable debugging tests within Visual Studio Code Dec 04, 2020 4 - Beta N/A :pypi:`pytest-vscode-pycharm-cls` A PyTest helper to enable start remote debugger on test start or failure or when pytest.set_trace is used. Feb 01, 2023 N/A pytest + :pypi:`pytest-vtestify` A pytest plugin for visual assertion using SSIM and image comparison. Feb 04, 2025 N/A pytest :pypi:`pytest-vts` pytest plugin for automatic recording of http stubbed tests Jun 05, 2019 N/A pytest (>=2.3) - :pypi:`pytest-vulture` A pytest plugin to checks dead code with vulture Jun 01, 2023 N/A pytest (>=7.0.0) + :pypi:`pytest-vulture` A pytest plugin to checks dead code with vulture Nov 25, 2024 N/A pytest>=7.0.0 :pypi:`pytest-vw` pytest-vw makes your failing test cases succeed under CI tools scrutiny Oct 07, 2015 4 - Beta N/A :pypi:`pytest-vyper` Plugin for the vyper smart contract language. May 28, 2020 2 - Pre-Alpha N/A :pypi:`pytest-wa-e2e-plugin` Pytest plugin for testing whatsapp bots with end to end tests Feb 18, 2020 4 - Beta pytest (>=3.5.0) - :pypi:`pytest-wake` Mar 20, 2024 N/A pytest + :pypi:`pytest-wake` Nov 19, 2024 N/A pytest :pypi:`pytest-watch` Local continuous test runner with pytest and watchdog. May 20, 2018 N/A N/A - :pypi:`pytest-watcher` Automatically rerun your tests on file modifications Apr 01, 2024 4 - Beta N/A + :pypi:`pytest-watcher` Automatically rerun your tests on file modifications Aug 28, 2024 4 - Beta N/A + :pypi:`pytest-watch-plugin` Placeholder for internal package Sep 12, 2024 N/A N/A :pypi:`pytest_wdb` Trace pytest tests with wdb to halt on error with --wdb. Jul 04, 2016 N/A N/A :pypi:`pytest-wdl` Pytest plugin for testing WDL workflows. Nov 17, 2020 5 - Production/Stable N/A :pypi:`pytest-web3-data` A pytest plugin to fetch test data from IPFS HTTP gateways during pytest execution. Oct 04, 2023 4 - Beta pytest - :pypi:`pytest-webdriver` Selenium webdriver fixture for py.test May 28, 2019 5 - Production/Stable pytest - :pypi:`pytest-webtest-extras` Pytest plugin to enhance pytest-html and allure reports of webtest projects by adding screenshots, comments and webpage sources. Jun 08, 2024 N/A pytest>=7.0.0 + :pypi:`pytest-webdriver` Selenium webdriver fixture for py.test Oct 17, 2024 5 - Production/Stable pytest + :pypi:`pytest-webstage` Test web apps with pytest Sep 20, 2024 N/A pytest<9.0,>=7.0 :pypi:`pytest-wetest` Welian API Automation test framework pytest plugin Nov 10, 2018 4 - Beta N/A - :pypi:`pytest-when` Utility which makes mocking more readable and controllable May 28, 2024 N/A pytest>=7.3.1 + :pypi:`pytest-when` Utility which makes mocking more readable and controllable Sep 25, 2025 N/A pytest>=7.3.1 :pypi:`pytest-whirlwind` Testing Tornado. Jun 12, 2020 N/A N/A :pypi:`pytest-wholenodeid` pytest addon for displaying the whole node id for failures Aug 26, 2015 4 - Beta pytest (>=2.0) :pypi:`pytest-win32consoletitle` Pytest progress in console title (Win32 only) Aug 08, 2021 N/A N/A :pypi:`pytest-winnotify` Windows tray notifications for py.test results. Apr 22, 2016 N/A N/A :pypi:`pytest-wiremock` A pytest plugin for programmatically using wiremock in integration tests Mar 27, 2022 N/A pytest (>=7.1.1,<8.0.0) + :pypi:`pytest-wiretap` \`pytest\` plugin for recording call stacks Mar 18, 2025 N/A pytest :pypi:`pytest-with-docker` pytest with docker helpers. Nov 09, 2021 N/A pytest + :pypi:`pytest-workaround-12888` forces an import of readline early in the process to work around pytest bug #12888 Jan 15, 2025 N/A N/A :pypi:`pytest-workflow` A pytest plugin for configuring workflow/pipeline tests using YAML files Mar 18, 2024 5 - Production/Stable pytest >=7.0.0 - :pypi:`pytest-xdist` pytest xdist plugin for distributed testing, most importantly across multiple CPUs Apr 28, 2024 5 - Production/Stable pytest>=7.0.0 + :pypi:`pytest-xdist` pytest xdist plugin for distributed testing, most importantly across multiple CPUs Jul 01, 2025 5 - Production/Stable pytest>=7.0.0 :pypi:`pytest-xdist-debug-for-graingert` pytest xdist plugin for distributed testing and loop-on-failing modes Jul 24, 2019 5 - Production/Stable pytest (>=4.4.0) :pypi:`pytest-xdist-forked` forked from pytest-xdist Feb 10, 2020 5 - Production/Stable pytest (>=4.4.0) + :pypi:`pytest-xdist-gnumake` A small example package Jun 22, 2025 N/A pytest :pypi:`pytest-xdist-tracker` pytest plugin helps to reproduce failures for particular xdist node Nov 18, 2021 3 - Alpha pytest (>=3.5.1) - :pypi:`pytest-xdist-worker-stats` A pytest plugin to list worker statistics after a xdist run. Apr 16, 2024 4 - Beta pytest>=7.0.0 + :pypi:`pytest-xdist-worker-stats` A pytest plugin to list worker statistics after a xdist run. Mar 15, 2025 4 - Beta pytest>=7.0.0 + :pypi:`pytest-xdocker` Pytest fixture to run docker across test runs. Jun 10, 2025 N/A pytest<9.0.0,>=8.0.0 :pypi:`pytest-xfaillist` Maintain a xfaillist in an additional file to avoid merge-conflicts. Sep 17, 2021 N/A pytest (>=6.2.2,<7.0.0) :pypi:`pytest-xfiles` Pytest fixtures providing data read from function, module or package related (x)files. Feb 27, 2018 N/A N/A + :pypi:`pytest-xflaky` A simple plugin to use with pytest Oct 14, 2024 4 - Beta pytest>=8.2.1 + :pypi:`pytest-xhtml` pytest plugin for generating HTML reports Oct 18, 2025 5 - Production/Stable pytest>=7 :pypi:`pytest-xiuyu` This is a pytest plugin Jul 25, 2023 5 - Production/Stable N/A :pypi:`pytest-xlog` Extended logging for test and decorators May 31, 2020 4 - Beta N/A - :pypi:`pytest-xlsx` pytest plugin for generating test cases by xlsx(excel) Apr 23, 2024 N/A pytest~=7.0 - :pypi:`pytest-xpara` An extended parametrizing plugin of pytest. Oct 30, 2017 3 - Alpha pytest + :pypi:`pytest-xlsx` pytest plugin for generating test cases by xlsx(excel) Aug 07, 2024 N/A pytest~=8.2.2 + :pypi:`pytest-xml` Create simple XML results for parsing Nov 14, 2024 4 - Beta pytest>=8.0.0 + :pypi:`pytest-xpara` An extended parametrizing plugin of pytest. Aug 07, 2024 3 - Alpha pytest :pypi:`pytest-xprocess` A pytest plugin for managing processes across test runs. May 19, 2024 4 - Beta pytest>=2.8 :pypi:`pytest-xray` May 30, 2019 3 - Alpha N/A :pypi:`pytest-xrayjira` Mar 17, 2020 3 - Alpha pytest (==4.3.1) + :pypi:`pytest-xray-reporter` Pytest plugin for generating Xray JSON reports May 21, 2025 4 - Beta pytest>=7.0.0 :pypi:`pytest-xray-server` May 03, 2022 3 - Alpha pytest (>=5.3.1) - :pypi:`pytest-xskynet` A package to prevent Dependency Confusion attacks against Yandex. Feb 20, 2024 N/A N/A :pypi:`pytest-xstress` Jun 01, 2024 N/A pytest<9.0.0,>=8.0.0 - :pypi:`pytest-xvfb` A pytest plugin to run Xvfb (or Xephyr/Xvnc) for tests. May 29, 2023 4 - Beta pytest (>=2.8.1) - :pypi:`pytest-xvirt` A pytest plugin to virtualize test. For example to transparently running them on a remote box. Jul 03, 2024 4 - Beta pytest>=7.2.2 + :pypi:`pytest-xtime` pytest plugin for recording execution time Jun 05, 2025 4 - Beta pytest + :pypi:`pytest-xvfb` A pytest plugin to run Xvfb (or Xephyr/Xvnc) for tests. Mar 12, 2025 4 - Beta pytest>=2.8.1 + :pypi:`pytest-xvirt` A pytest plugin to virtualize test. For example to transparently running them on a remote box. Dec 15, 2024 4 - Beta pytest>=7.2.2 :pypi:`pytest-yaml` This plugin is used to load yaml output to your test using pytest framework. Oct 05, 2018 N/A pytest - :pypi:`pytest-yaml-sanmu` pytest plugin for generating test cases by yaml Jul 12, 2024 N/A pytest>=7.4.0 + :pypi:`pytest-yaml-fei` a pytest yaml allure package Aug 03, 2025 N/A pytest + :pypi:`pytest-yaml-sanmu` Pytest plugin for generating test cases with YAML. In test cases, you can use markers, fixtures, variables, and even call Python functions. Sep 16, 2025 N/A pytest>=8.2.2 :pypi:`pytest-yamltree` Create or check file/directory trees described by YAML Mar 02, 2020 4 - Beta pytest (>=3.1.1) :pypi:`pytest-yamlwsgi` Run tests against wsgi apps defined in yaml May 11, 2010 N/A N/A :pypi:`pytest-yaml-yoyo` http/https API run by yaml Jun 19, 2023 N/A pytest (>=7.2.0) :pypi:`pytest-yapf` Run yapf Jul 06, 2017 4 - Beta pytest (>=3.1.1) :pypi:`pytest-yapf3` Validate your Python file format with yapf Mar 29, 2023 5 - Production/Stable pytest (>=7) :pypi:`pytest-yield` PyTest plugin to run tests concurrently, each \`yield\` switch context to other one Jan 23, 2019 N/A N/A - :pypi:`pytest-yls` Pytest plugin to test the YLS as a whole. Mar 30, 2024 N/A pytest<8.0.0,>=7.2.2 + :pypi:`pytest-yls` Pytest plugin to test the YLS as a whole. Apr 09, 2025 N/A pytest<9.0.0,>=8.3.3 :pypi:`pytest-youqu-playwright` pytest-youqu-playwright Jun 12, 2024 N/A pytest :pypi:`pytest-yuk` Display tests you are uneasy with, using 🤢/🤮 for pass/fail of tests marked with yuk. Mar 26, 2021 N/A pytest>=5.0.0 :pypi:`pytest-zafira` A Zafira plugin for pytest Sep 18, 2019 5 - Production/Stable pytest (==4.1.1) @@ -1516,32 +1771,42 @@ This list contains 1487 plugins. :pypi:`pytest-zcc` eee Jun 02, 2024 N/A N/A :pypi:`pytest-zebrunner` Pytest connector for Zebrunner reporting Jul 04, 2024 5 - Production/Stable pytest>=4.5.0 :pypi:`pytest-zeebe` Pytest fixtures for testing Camunda 8 processes using a Zeebe test engine. Feb 01, 2024 N/A pytest (>=7.4.2,<8.0.0) + :pypi:`pytest-zephyr-scale-integration` A library for integrating Jira Zephyr Scale (Adaptavist\TM4J) with pytest Jun 26, 2025 N/A pytest + :pypi:`pytest-zephyr-telegram` Плагин для отправки данных автотестов в Телеграм и Зефир Sep 30, 2024 N/A pytest==8.3.2 :pypi:`pytest-zest` Zesty additions to pytest. Nov 17, 2022 N/A N/A :pypi:`pytest-zhongwen-wendang` PyTest 中文文档 Mar 04, 2024 4 - Beta N/A :pypi:`pytest-zigzag` Extend py.test for RPC OpenStack testing. Feb 27, 2019 4 - Beta pytest (~=3.6) :pypi:`pytest-zulip` Pytest report plugin for Zulip May 07, 2022 5 - Production/Stable pytest :pypi:`pytest-zy` 接口自动化测试框架 Mar 24, 2024 N/A pytest~=7.2.0 + :pypi:`tursu` 🎬 A pytest plugin that transpiles Gherkin feature files to Python using AST, enforcing typing for ease of use and debugging. May 05, 2025 4 - Beta pytest>=8.3.5 =============================================== ====================================================================================================================================================================================================================================================================================================================================================================================== ============== ===================== ================================================ .. only:: latex + :pypi:`databricks-labs-pytester` + *last release*: Oct 17, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3 + + Python Testing for Databricks + :pypi:`logassert` - *last release*: May 20, 2022, + *last release*: Aug 14, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest; extra == "dev" - Simple but powerful assertion and verification of logged lines. + Simple but powerful assertion and verification of logged lines :pypi:`logot` - *last release*: Mar 23, 2024, + *last release*: Jul 28, 2025, *status*: 5 - Production/Stable, - *requires*: pytest<9,>=7; extra == "pytest" + *requires*: pytest; extra == "pytest" Test whether your code is logging correctly 🪵 :pypi:`nuts` - *last release*: May 28, 2024, + *last release*: May 10, 2025, *status*: N/A, *requires*: pytest<8,>=7 @@ -1562,11 +1827,11 @@ This list contains 1487 plugins. A contextmanager pytest fixture for handling multiple mock abstracts :pypi:`pytest-accept` - *last release*: Feb 10, 2024, + *last release*: Aug 19, 2025, *status*: N/A, - *requires*: pytest (>=6) + *requires*: pytest>=7 + - A pytest-plugin for updating doctest outputs :pypi:`pytest-adaptavist` *last release*: Oct 13, 2022, @@ -1576,9 +1841,9 @@ This list contains 1487 plugins. pytest plugin for generating test execution results within Jira Test Management (tm4j) :pypi:`pytest-adaptavist-fixed` - *last release*: Nov 08, 2023, + *last release*: Jan 17, 2025, *status*: N/A, - *requires*: pytest >=5.4.0 + *requires*: pytest>=5.4.0 pytest plugin for generating test execution results within Jira Test Management (tm4j) @@ -1631,6 +1896,13 @@ This list contains 1487 plugins. pytest plugin for pytest-repeat that generate aggregate report of the same test cases with additional statistics details. + :pypi:`pytest-ai` + *last release*: Jan 22, 2025, + *status*: N/A, + *requires*: N/A + + A Python package to generate regular, edge-case, and security HTTP tests. + :pypi:`pytest-ai1899` *last release*: Mar 13, 2024, *status*: 5 - Production/Stable, @@ -1639,12 +1911,19 @@ This list contains 1487 plugins. pytest plugin for connecting to ai1899 smart system stack :pypi:`pytest-aio` - *last release*: Apr 08, 2024, + *last release*: Jul 31, 2024, *status*: 5 - Production/Stable, *requires*: pytest Pytest plugin for testing async python code + :pypi:`pytest-aioboto3` + *last release*: Jan 17, 2025, + *status*: N/A, + *requires*: N/A + + Aioboto3 Pytest with Moto + :pypi:`pytest-aiofiles` *last release*: May 14, 2017, *status*: 5 - Production/Stable, @@ -1660,9 +1939,9 @@ This list contains 1487 plugins. :pypi:`pytest-aiohttp` - *last release*: Sep 06, 2023, + *last release*: Jan 23, 2025, *status*: 4 - Beta, - *requires*: pytest >=6.1.0 + *requires*: pytest>=6.1.0 Pytest plugin for aiohttp support @@ -1673,6 +1952,13 @@ This list contains 1487 plugins. Pytest \`client\` fixture for the Aiohttp + :pypi:`pytest-aiohttp-mock` + *last release*: Sep 13, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=8 + + Send responses to aiohttp. + :pypi:`pytest-aiomoto` *last release*: Jun 24, 2023, *status*: N/A, @@ -1681,16 +1967,16 @@ This list contains 1487 plugins. pytest-aiomoto :pypi:`pytest-aioresponses` - *last release*: Jul 29, 2021, + *last release*: Jan 02, 2025, *status*: 4 - Beta, - *requires*: pytest (>=3.5.0) + *requires*: pytest>=3.5.0 py.test integration for aioresponses :pypi:`pytest-aioworkers` - *last release*: May 01, 2023, + *last release*: Dec 26, 2024, *status*: 5 - Production/Stable, - *requires*: pytest>=6.1.0 + *requires*: pytest>=8.3.4 A plugin to test aioworkers project with pytest @@ -1709,12 +1995,19 @@ This list contains 1487 plugins. :pypi:`pytest-alembic` - *last release*: Mar 04, 2024, + *last release*: May 27, 2025, *status*: N/A, - *requires*: pytest (>=6.0) + *requires*: pytest>=7.0 A pytest plugin for verifying alembic migrations. + :pypi:`pytest-alerts` + *last release*: Feb 21, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.4.0 + + A pytest plugin for sending test results to Slack and Telegram + :pypi:`pytest-allclose` *last release*: Jul 30, 2019, *status*: 5 - Production/Stable, @@ -1750,6 +2043,13 @@ This list contains 1487 plugins. pytest plugin to test case doc string dls instructions + :pypi:`pytest-allure-host` + *last release*: Oct 21, 2025, + *status*: 3 - Alpha, + *requires*: N/A + + Publish Allure static reports to private S3 behind CloudFront with history preservation + :pypi:`pytest-allure-id2history` *last release*: May 14, 2024, *status*: 4 - Beta, @@ -1771,6 +2071,13 @@ This list contains 1487 plugins. The pytest plugin aimed to display test coverage of the specs(requirements) in Allure + :pypi:`pytest-allure-step` + *last release*: Jul 13, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=6.0.0 + + Enhanced logging integration with Allure reports for pytest + :pypi:`pytest-alphamoon` *last release*: Dec 30, 2021, *status*: 5 - Production/Stable, @@ -1778,6 +2085,13 @@ This list contains 1487 plugins. Static code checks used at Alphamoon + :pypi:`pytest-amaranth-sim` + *last release*: Sep 21, 2024, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + Fixture to automate running Amaranth simulations + :pypi:`pytest-analyzer` *last release*: Feb 21, 2024, *status*: N/A, @@ -1806,8 +2120,15 @@ This list contains 1487 plugins. pytest-annotate: Generate PyAnnotate annotations from your pytest tests. + :pypi:`pytest-annotated` + *last release*: Sep 30, 2024, + *status*: N/A, + *requires*: pytest>=8.3.3 + + Pytest plugin to allow use of Annotated in tests to resolve fixtures + :pypi:`pytest-ansible` - *last release*: Jul 10, 2024, + *last release*: Aug 21, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=6 @@ -1835,9 +2156,9 @@ This list contains 1487 plugins. A pytest plugin for running unit tests within an ansible collection :pypi:`pytest-antilru` - *last release*: Jul 05, 2022, + *last release*: Jul 28, 2024, *status*: 5 - Production/Stable, - *requires*: pytest + *requires*: pytest>=7; python_version >= "3.10" Bust functools.lru_cache when running pytest to avoid test pollution @@ -1876,6 +2197,27 @@ This list contains 1487 plugins. An ASGI middleware to populate OpenAPI Specification examples from pytest functions + :pypi:`pytest-api-cov` + *last release*: Oct 28, 2025, + *status*: N/A, + *requires*: pytest>=6.0.0 + + Pytest Plugin to provide API Coverage statistics for Python Web Frameworks + + :pypi:`pytest-api-framework` + *last release*: Jun 22, 2025, + *status*: N/A, + *requires*: pytest==7.2.2 + + pytest framework + + :pypi:`pytest-api-framework-alpha` + *last release*: Oct 29, 2025, + *status*: N/A, + *requires*: pytest==7.2.2 + + + :pypi:`pytest-api-soup` *last release*: Aug 27, 2022, *status*: N/A, @@ -1911,6 +2253,13 @@ This list contains 1487 plugins. Pytest plugin for appium + :pypi:`pytest-approval` + *last release*: Oct 27, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + A simple approval test library utilizing external diff programs such as PyCharm and Visual Studio Code to compare approved and received output. + :pypi:`pytest-approvaltests` *last release*: May 08, 2022, *status*: 4 - Beta, @@ -1919,16 +2268,16 @@ This list contains 1487 plugins. A plugin to use approvaltests with pytest :pypi:`pytest-approvaltests-geo` - *last release*: Feb 05, 2024, + *last release*: Jul 14, 2025, *status*: 5 - Production/Stable, *requires*: pytest Extension for ApprovalTests.Python specific to geo data verification :pypi:`pytest-archon` - *last release*: Dec 18, 2023, + *last release*: Sep 19, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=7.2 + *requires*: pytest>=7.2 Rule your architecture like a real developer @@ -1939,6 +2288,20 @@ This list contains 1487 plugins. pyest results colection plugin + :pypi:`pytest-argus-reporter` + *last release*: Sep 17, 2025, + *status*: 4 - Beta, + *requires*: pytest>=3.0; extra == "dev" + + A simple plugin to report results of test into argus + + :pypi:`pytest-argus-server` + *last release*: Mar 24, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A plugin that provides a running Argus API server for tests + :pypi:`pytest-arraydiff` *last release*: Nov 27, 2023, *status*: 4 - Beta, @@ -1946,6 +2309,13 @@ This list contains 1487 plugins. pytest plugin to help with comparing array output from tests + :pypi:`pytest-asdf-plugin` + *last release*: Aug 18, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7 + + Pytest plugin for testing ASDF schemas + :pypi:`pytest-asgi-server` *last release*: Dec 12, 2020, *status*: N/A, @@ -1981,6 +2351,13 @@ This list contains 1487 plugins. Pytest Assertions + :pypi:`pytest-assert-type` + *last release*: Oct 26, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=6.2.0 + + Use typing.assert_type() to test runtime behavior + :pypi:`pytest-assertutil` *last release*: May 10, 2019, *status*: N/A, @@ -1996,11 +2373,11 @@ This list contains 1487 plugins. Useful assertion utilities for use with pytest :pypi:`pytest-assist` - *last release*: Jun 24, 2024, - *status*: N/A, + *last release*: Oct 29, 2025, + *status*: 4 - Beta, *requires*: pytest - load testing library + pytest plugin library :pypi:`pytest-assume` *last release*: Jun 24, 2021, @@ -2058,6 +2435,13 @@ This list contains 1487 plugins. pytest-async - Run your coroutine in event loop without decorator + :pypi:`pytest-async-benchmark` + *last release*: May 28, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + pytest-async-benchmark: Modern pytest benchmarking for async code. 🚀 + :pypi:`pytest-async-generators` *last release*: Jul 05, 2023, *status*: N/A, @@ -2066,14 +2450,21 @@ This list contains 1487 plugins. Pytest fixtures for async generators :pypi:`pytest-asyncio` - *last release*: May 19, 2024, - *status*: 4 - Beta, - *requires*: pytest<9,>=7.0.0 + *last release*: Sep 12, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest<9,>=8.2 Pytest support for asyncio + :pypi:`pytest-asyncio-concurrent` + *last release*: May 17, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + Pytest plugin to execute python async tests concurrently. + :pypi:`pytest-asyncio-cooperative` - *last release*: Jul 04, 2024, + *last release*: Jun 24, 2025, *status*: N/A, *requires*: N/A @@ -2114,6 +2505,13 @@ This list contains 1487 plugins. Skip rest of tests if previous test failed. + :pypi:`pytest-atstack` + *last release*: Jan 02, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A simple plugin to use with pytest + :pypi:`pytest-attrib` *last release*: May 24, 2016, *status*: 4 - Beta, @@ -2149,6 +2547,13 @@ This list contains 1487 plugins. automatically check condition and log all the checks + :pypi:`pytest-autofixture` + *last release*: Aug 01, 2024, + *status*: N/A, + *requires*: pytest>=8 + + simplify pytest fixtures + :pypi:`pytest-automation` *last release*: Apr 24, 2024, *status*: N/A, @@ -2170,6 +2575,13 @@ This list contains 1487 plugins. pytest plugin: avoid repeating arguments in parametrize + :pypi:`pytest-autoprofile` + *last release*: Aug 06, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0 + + \`line_profiler.autoprofile\`-ing your \`pytest\` test suite + :pypi:`pytest-autotest` *last release*: Aug 25, 2021, *status*: N/A, @@ -2177,13 +2589,6 @@ This list contains 1487 plugins. This fixture provides a configured "driver" for Android Automated Testing, using uiautomator2. - :pypi:`pytest-aux` - *last release*: Jul 05, 2024, - *status*: N/A, - *requires*: N/A - - templates/examples and aux for pytest - :pypi:`pytest-aviator` *last release*: Nov 04, 2022, *status*: 4 - Beta, @@ -2198,6 +2603,13 @@ This list contains 1487 plugins. Makes pytest skip tests that don not need rerunning + :pypi:`pytest-awaiting-fix` + *last release*: Aug 09, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A simple plugin to use with pytest for traceability across Jira and disabled automated tests + :pypi:`pytest-aws` *last release*: Oct 04, 2017, *status*: 4 - Beta, @@ -2220,9 +2632,9 @@ This list contains 1487 plugins. Protect your AWS credentials in unit tests :pypi:`pytest-aws-fixtures` - *last release*: Feb 02, 2024, + *last release*: Apr 06, 2025, *status*: N/A, - *requires*: pytest (>=8.0.0,<9.0.0) + *requires*: pytest<9.0.0,>=8.0.0 A series of fixtures to use in integration tests involving actual AWS services. @@ -2248,9 +2660,9 @@ This list contains 1487 plugins. Pytest utilities and mocks for Azure :pypi:`pytest-azure-devops` - *last release*: Jun 20, 2022, + *last release*: Jul 16, 2025, *status*: 4 - Beta, - *requires*: pytest (>=3.5.0) + *requires*: pytest>=3.5.0 Simplifies using azure devops parallel strategy (https://docs.microsoft.com/en-us/azure/devops/pipelines/test/parallel-testing-any-test-runner) with pytest. @@ -2282,6 +2694,13 @@ This list contains 1487 plugins. pytest plugin for URL based testing + :pypi:`pytest-bashdoctest` + *last release*: Oct 03, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + A pytest plugin for testing bash command examples in markdown documentation + :pypi:`pytest-batch-regression` *last release*: May 08, 2024, *status*: N/A, @@ -2290,16 +2709,16 @@ This list contains 1487 plugins. A pytest plugin to repeat the entire test suite in batches. :pypi:`pytest-bazel` - *last release*: Jul 12, 2024, + *last release*: Oct 31, 2025, *status*: 4 - Beta, *requires*: pytest A pytest runner with bazel support :pypi:`pytest-bdd` - *last release*: Jun 04, 2024, + *last release*: Dec 05, 2024, *status*: 6 - Mature, - *requires*: pytest>=6.2.0 + *requires*: pytest>=7.0.0 BDD for pytest @@ -2311,19 +2730,26 @@ This list contains 1487 plugins. pytest plugin to display BDD info in HTML test report :pypi:`pytest-bdd-ng` - *last release*: Dec 31, 2023, + *last release*: Nov 26, 2024, *status*: 4 - Beta, - *requires*: pytest >=5.0 + *requires*: pytest>=5.2 BDD for pytest :pypi:`pytest-bdd-report` - *last release*: May 20, 2024, + *last release*: Aug 19, 2025, *status*: N/A, - *requires*: pytest >=7.1.3 + *requires*: pytest>=7.1.3 A pytest-bdd plugin for generating useful and informative BDD test reports + :pypi:`pytest-bdd-reporter` + *last release*: Oct 14, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=6.0.0 + + Enterprise-grade BDD test reporting with interactive dashboards, suite management, and comprehensive email integration + :pypi:`pytest-bdd-splinter` *last release*: Aug 12, 2019, *status*: 5 - Production/Stable, @@ -2353,14 +2779,14 @@ This list contains 1487 plugins. A pytest plugin that reports test results to the BeakerLib framework :pypi:`pytest-beartype` - *last release*: Jan 25, 2024, + *last release*: Oct 31, 2024, *status*: N/A, *requires*: pytest Pytest plugin to run your tests with beartype checking enabled. :pypi:`pytest-bec-e2e` - *last release*: Jul 08, 2024, + *last release*: Oct 31, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -2388,9 +2814,9 @@ This list contains 1487 plugins. Benchmark utility that plugs into pytest. :pypi:`pytest-benchmark` - *last release*: Oct 25, 2022, + *last release*: Oct 30, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=3.8) + *requires*: pytest>=8.1 A \`\`pytest\`\` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer. @@ -2437,9 +2863,9 @@ This list contains 1487 plugins. Find tests leaking state and affecting other :pypi:`pytest-black` - *last release*: Oct 05, 2020, + *last release*: Dec 15, 2024, *status*: 4 - Beta, - *requires*: N/A + *requires*: pytest>=7.0.0 A pytest plugin to enable format checking with black @@ -2465,9 +2891,9 @@ This list contains 1487 plugins. A pytest plugin helps developers to debug by providing useful commits history. :pypi:`pytest-blender` - *last release*: Aug 10, 2023, + *last release*: Jun 25, 2025, *status*: N/A, - *requires*: pytest ; extra == 'dev' + *requires*: pytest Blender Pytest plugin. @@ -2492,6 +2918,13 @@ This list contains 1487 plugins. pytest plugin to mark a test as blocker and skip all other tests + :pypi:`pytest-b-logger` + *last release*: Oct 28, 2025, + *status*: N/A, + *requires*: pytest + + BLogger is a Pytest plugin for enhanced test logging and generating convenient and lightweight reports. + :pypi:`pytest-blue` *last release*: Sep 05, 2022, *status*: N/A, @@ -2506,6 +2939,27 @@ This list contains 1487 plugins. Local continuous test runner with pytest and watchdog. + :pypi:`pytest-boardfarm3` + *last release*: Sep 15, 2025, + *status*: N/A, + *requires*: pytest + + Integrate boardfarm as a pytest plugin. + + :pypi:`pytest-boilerplate` + *last release*: Sep 12, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest>=4.0.0 + + The pytest plugin for your Django Boilerplate. + + :pypi:`pytest-bonsai` + *last release*: Apr 08, 2025, + *status*: N/A, + *requires*: pytest>=6 + + + :pypi:`pytest-boost-xml` *last release*: Nov 30, 2022, *status*: 4 - Beta, @@ -2521,7 +2975,7 @@ This list contains 1487 plugins. :pypi:`pytest-boto-mock` - *last release*: Jun 05, 2024, + *last release*: Jul 16, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=8.2.0 @@ -2569,8 +3023,15 @@ This list contains 1487 plugins. A pytest plugin for running tests on a Briefcase project. + :pypi:`pytest-brightest` + *last release*: Jul 15, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=8.4.1 + + Bright ideas for improving your pytest experience + :pypi:`pytest-broadcaster` - *last release*: Apr 06, 2024, + *last release*: Mar 02, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -2612,9 +3073,9 @@ This list contains 1487 plugins. Budo Systems is a martial arts school management system. This module is the Budo Systems Pytest Plugin. :pypi:`pytest-bug` - *last release*: Jun 05, 2024, + *last release*: Jun 17, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=8.0.0 + *requires*: pytest>=8.4.0 Pytest plugin for marking tests as a bug @@ -2709,6 +3170,13 @@ This list contains 1487 plugins. A plugin which allows to compare results with canonical results, based on previous runs + :pypi:`pytest-canvas` + *last release*: Jul 22, 2025, + *status*: N/A, + *requires*: pytest<9,>=8.4 + + A minimal pytest plugin that streamlines testing for projects using the Canvas SDK. + :pypi:`pytest-caprng` *last release*: May 02, 2018, *status*: 4 - Beta, @@ -2716,6 +3184,13 @@ This list contains 1487 plugins. A plugin that replays pRNG state on failure. + :pypi:`pytest-capsqlalchemy` + *last release*: Mar 19, 2025, + *status*: 4 - Beta, + *requires*: N/A + + Pytest plugin to allow capturing SQLAlchemy queries. + :pypi:`pytest-capture-deprecatedwarnings` *last release*: Apr 30, 2019, *status*: N/A, @@ -2730,13 +3205,41 @@ This list contains 1487 plugins. pytest plugin to capture all warnings and put them in one file of your choice + :pypi:`pytest-case` + *last release*: Nov 25, 2024, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.3.3 + + A clean, modern, wrapper for pytest.mark.parametrize + + :pypi:`pytest-case-provider` + *last release*: Oct 26, 2025, + *status*: 3 - Alpha, + *requires*: pytest<9,>=8 + + Advanced pytest parametrization plugin that generates test case instances from sync or async factories. + :pypi:`pytest-cases` - *last release*: Apr 04, 2024, + *last release*: Jun 09, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest Separate test code from test cases in pytest. + :pypi:`pytest-case-start-from` + *last release*: Oct 28, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.0.0 + + A pytest plugin to start test execution from a specific test case + + :pypi:`pytest-casewise-package-install` + *last release*: Oct 31, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=6.0.0 + + A pytest plugin for test case-level dynamic dependency management + :pypi:`pytest-cassandra` *last release*: Nov 04, 2017, *status*: 1 - Planning, @@ -2758,13 +3261,27 @@ This list contains 1487 plugins. Pytest plugin with server for catching HTTP requests. + :pypi:`pytest-cdist` + *last release*: Jan 30, 2025, + *status*: N/A, + *requires*: pytest>=7 + + A pytest plugin to split your test suite into multiple parts + :pypi:`pytest-celery` - *last release*: Apr 11, 2024, - *status*: 4 - Beta, + *last release*: Jul 30, 2025, + *status*: 5 - Production/Stable, *requires*: N/A Pytest plugin for Celery + :pypi:`pytest-celery-py37` + *last release*: May 23, 2025, + *status*: 5 - Production/Stable, + *requires*: N/A + + Pytest plugin for Celery (compatible with python 3.7) + :pypi:`pytest-cfg-fetcher` *last release*: Feb 26, 2024, *status*: N/A, @@ -2822,8 +3339,8 @@ This list contains 1487 plugins. A pytest fixture for changing current working directory :pypi:`pytest-check` - *last release*: Jan 18, 2024, - *status*: N/A, + *last release*: Oct 07, 2025, + *status*: 5 - Production/Stable, *requires*: pytest>=7.0.0 A pytest plugin that allows multiple failures per test. @@ -2864,7 +3381,7 @@ This list contains 1487 plugins. Check links in files :pypi:`pytest-checklist` - *last release*: Jun 10, 2024, + *last release*: May 23, 2025, *status*: N/A, *requires*: N/A @@ -2877,12 +3394,12 @@ This list contains 1487 plugins. pytest plugin to test Check_MK checks - :pypi:`pytest-check-requirements` - *last release*: Feb 20, 2024, + :pypi:`pytest-checkpoint` + *last release*: Oct 04, 2025, *status*: N/A, - *requires*: N/A + *requires*: pytest>=8.0.0 - A package to prevent Dependency Confusion attacks against Yandex. + Restore a checkpoint in pytest :pypi:`pytest-ch-framework` *last release*: Apr 17, 2024, @@ -2892,11 +3409,18 @@ This list contains 1487 plugins. My pytest framework :pypi:`pytest-chic-report` - *last release*: Jan 31, 2023, - *status*: 5 - Production/Stable, - *requires*: N/A + *last release*: Nov 01, 2024, + *status*: N/A, + *requires*: pytest>=6.0 + + Simple pytest plugin for generating and sending report to messengers. + + :pypi:`pytest-chinesereport` + *last release*: Apr 16, 2025, + *status*: 4 - Beta, + *requires*: pytest>=3.5.0 + - A pytest plugin to send a report and printing summary of tests. :pypi:`pytest-choose` *last release*: Feb 04, 2024, @@ -2905,6 +3429,13 @@ This list contains 1487 plugins. Provide the pytest with the ability to collect use cases based on rules in text files + :pypi:`pytest-chronicle` + *last release*: Oct 30, 2025, + *status*: N/A, + *requires*: pytest>=8.0; extra == "dev" + + Reusable pytest results ingestion tooling with database export and CLI helpers. + :pypi:`pytest-chunks` *last release*: Jul 05, 2022, *status*: N/A, @@ -2954,6 +3485,13 @@ This list contains 1487 plugins. A plugin providing an alternative, colourful diff output for failing assertions. + :pypi:`pytest-class-fixtures` + *last release*: Nov 15, 2024, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.3.3 + + Class as PyTest fixtures (and BDD steps) + :pypi:`pytest-cldf` *last release*: Nov 07, 2022, *status*: N/A, @@ -2961,8 +3499,15 @@ This list contains 1487 plugins. Easy quality control for CLDF datasets using pytest + :pypi:`pytest-clean-database` + *last release*: Mar 14, 2025, + *status*: 3 - Alpha, + *requires*: pytest<9,>=7.0 + + A pytest plugin that cleans your database up after every test. + :pypi:`pytest-cleanslate` - *last release*: Jun 17, 2024, + *last release*: Apr 10, 2025, *status*: N/A, *requires*: pytest @@ -2976,19 +3521,26 @@ This list contains 1487 plugins. Automated, comprehensive and well-organised pytest test cases. :pypi:`pytest-cleanuptotal` - *last release*: Mar 19, 2024, + *last release*: Jul 22, 2025, *status*: 5 - Production/Stable, *requires*: N/A A cleanup plugin for pytest :pypi:`pytest-clerk` - *last release*: Jun 27, 2024, + *last release*: Aug 30, 2025, *status*: N/A, *requires*: pytest<9.0.0,>=8.0.0 A set of pytest fixtures to help with integration testing with Clerk. + :pypi:`pytest-cli2-ansible` + *last release*: Mar 05, 2025, + *status*: N/A, + *requires*: N/A + + + :pypi:`pytest-click` *last release*: Feb 11, 2022, *status*: 5 - Production/Stable, @@ -3004,9 +3556,9 @@ This list contains 1487 plugins. Automatically register fixtures for custom CLI arguments :pypi:`pytest-clld` - *last release*: Jul 06, 2022, + *last release*: Oct 23, 2024, *status*: N/A, - *requires*: pytest (>=3.6) + *requires*: pytest>=3.9 @@ -3032,7 +3584,7 @@ This list contains 1487 plugins. Distribute tests to cloud machines without fuss :pypi:`pytest-cmake` - *last release*: May 31, 2024, + *last release*: Aug 14, 2025, *status*: N/A, *requires*: pytest<9,>=4 @@ -3045,6 +3597,13 @@ This list contains 1487 plugins. Execute CMake Presets via pytest + :pypi:`pytest-cmdline-add-args` + *last release*: Sep 01, 2024, + *status*: N/A, + *requires*: N/A + + Pytest plugin for custom argument handling and Allure reporting. This plugin allows you to add arguments before running a test. + :pypi:`pytest-cobra` *last release*: Jun 29, 2019, *status*: 3 - Alpha, @@ -3052,6 +3611,20 @@ This list contains 1487 plugins. PyTest plugin for testing Smart Contracts for Ethereum blockchain. + :pypi:`pytest-cocotb` + *last release*: Mar 15, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest; extra == "test" + + Pytest plugin to integrate Cocotb + + :pypi:`pytest-codeblock` + *last release*: May 10, 2025, + *status*: 4 - Beta, + *requires*: pytest + + Pytest plugin to collect and test code blocks in reStructuredText and Markdown files. + :pypi:`pytest_codeblocks` *last release*: Sep 17, 2023, *status*: 5 - Production/Stable, @@ -3074,9 +3647,9 @@ This list contains 1487 plugins. pytest plugin to add source code sanity checks (pep8 and friends) :pypi:`pytest-codecov` - *last release*: Nov 29, 2022, + *last release*: Mar 25, 2025, *status*: 4 - Beta, - *requires*: pytest (>=4.6.0) + *requires*: pytest>=4.6.0 Pytest plugin for uploading pytest-cov results to codecov.io @@ -3102,7 +3675,7 @@ This list contains 1487 plugins. pytest plugin to run pycodestyle :pypi:`pytest-codspeed` - *last release*: Mar 19, 2024, + *last release*: Oct 24, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=3.8 @@ -3165,7 +3738,7 @@ This list contains 1487 plugins. An interactive GUI test runner for PyTest :pypi:`pytest-common-subject` - *last release*: Jun 12, 2024, + *last release*: Oct 22, 2025, *status*: N/A, *requires*: pytest<9,>=3.6 @@ -3185,6 +3758,13 @@ This list contains 1487 plugins. Concurrently execute test cases with multithread, multiprocess and gevent + :pypi:`pytest-conductor` + *last release*: Jul 30, 2025, + *status*: N/A, + *requires*: pytest<8.4; python_version == "3.8" + + Pytest plugin for coordinating the order in which marked tests run. + :pypi:`pytest-config` *last release*: Nov 07, 2014, *status*: 5 - Production/Stable, @@ -3214,7 +3794,7 @@ This list contains 1487 plugins. pytest plugin with fixtures for testing consul aware apps :pypi:`pytest-container` - *last release*: Apr 10, 2024, + *last release*: Jun 30, 2025, *status*: 4 - Beta, *requires*: pytest>=3.10 @@ -3249,7 +3829,7 @@ This list contains 1487 plugins. The pytest plugin for your Cookiecutter templates. 🍪 :pypi:`pytest-copie` - *last release*: Jun 26, 2024, + *last release*: Sep 29, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -3277,9 +3857,9 @@ This list contains 1487 plugins. count erros and send email :pypi:`pytest-cov` - *last release*: Mar 24, 2024, + *last release*: Sep 09, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=4.6 + *requires*: pytest>=7 Pytest plugin for measuring coverage. @@ -3305,7 +3885,7 @@ This list contains 1487 plugins. Coverage dynamic context support for PyTest, including sub-processes :pypi:`pytest-coveragemarkers` - *last release*: Jun 04, 2024, + *last release*: May 15, 2025, *status*: N/A, *requires*: pytest<8.0.0,>=7.1.2 @@ -3326,21 +3906,14 @@ This list contains 1487 plugins. Too many faillure, less tests. :pypi:`pytest-cpp` - *last release*: Nov 01, 2023, + *last release*: Sep 18, 2024, *status*: 5 - Production/Stable, - *requires*: pytest >=7.0 + *requires*: pytest Use pytest's runner to discover and execute C++ tests - :pypi:`pytest-cppython` - *last release*: Mar 14, 2024, - *status*: N/A, - *requires*: N/A - - A pytest plugin that imports CPPython testing types - - :pypi:`pytest-cqase` - *last release*: Aug 22, 2022, + :pypi:`pytest-cqase` + *last release*: Aug 22, 2022, *status*: N/A, *requires*: pytest (>=7.1.2,<8.0.0) @@ -3360,13 +3933,34 @@ This list contains 1487 plugins. Manages CrateDB instances during your integration tests - :pypi:`pytest-crayons` - *last release*: Oct 08, 2023, + :pypi:`pytest-cratedb` + *last release*: Oct 08, 2024, + *status*: 4 - Beta, + *requires*: pytest<9 + + Manage CrateDB instances for integration tests + + :pypi:`pytest-cratedb-reporter` + *last release*: Mar 11, 2025, *status*: N/A, + *requires*: pytest>=6.0.0 + + A pytest plugin for reporting test results to CrateDB + + :pypi:`pytest-crayons` + *last release*: Oct 14, 2025, + *status*: 5 - Production/Stable, *requires*: pytest A pytest plugin for colorful print statements + :pypi:`pytest-cream` + *last release*: Oct 26, 2025, + *status*: N/A, + *requires*: pytest + + The cream of test execution - smooth pytest workflows with intelligent orchestration + :pypi:`pytest-create` *last release*: Feb 15, 2023, *status*: 1 - Planning, @@ -3396,17 +3990,24 @@ This list contains 1487 plugins. CSV output for pytest. :pypi:`pytest-csv-params` - *last release*: Jul 01, 2023, + *last release*: May 29, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=7.4.0,<8.0.0) + *requires*: pytest<9,>=8.3 Pytest plugin for Test Case Parametrization with CSV files - :pypi:`pytest-curio` - *last release*: Oct 07, 2020, + :pypi:`pytest-culprit` + *last release*: May 15, 2025, *status*: N/A, *requires*: N/A + Find the last Git commit where a pytest test started failing + + :pypi:`pytest-curio` + *last release*: Oct 06, 2024, + *status*: N/A, + *requires*: pytest + Pytest support for curio. :pypi:`pytest-curl-report` @@ -3458,6 +4059,13 @@ This list contains 1487 plugins. Custom grouping for pytest-xdist, rename test cases name and test cases nodeid, support allure report + :pypi:`pytest-custom-timeout` + *last release*: Jan 08, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.0.0 + + Use custom logic when a test times out. Based on pytest-timeout. + :pypi:`pytest-cython` *last release*: Apr 05, 2024, *status*: 5 - Production/Stable, @@ -3487,7 +4095,7 @@ This list contains 1487 plugins. pytest fixtures to run dash applications. :pypi:`pytest-dashboard` - *last release*: May 30, 2024, + *last release*: Jun 02, 2025, *status*: N/A, *requires*: pytest<8.0.0,>=7.4.3 @@ -3501,7 +4109,7 @@ This list contains 1487 plugins. Useful functions for managing data for pytest fixtures :pypi:`pytest-databases` - *last release*: Jul 02, 2024, + *last release*: Oct 06, 2025, *status*: 4 - Beta, *requires*: pytest @@ -3515,9 +4123,9 @@ This list contains 1487 plugins. Pytest plugin for remote Databricks notebooks testing :pypi:`pytest-datadir` - *last release*: Oct 03, 2023, + *last release*: Jul 30, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=5.0 + *requires*: pytest>=7.0 pytest plugin for test data directories and files @@ -3564,11 +4172,11 @@ This list contains 1487 plugins. py.test plugin to create a 'tmp_path' containing predefined files/directories. :pypi:`pytest-datafixtures` - *last release*: Dec 05, 2020, + *last release*: May 15, 2025, *status*: 5 - Production/Stable, *requires*: N/A - Data fixtures for pytest made simple + Data fixtures for pytest made simple. :pypi:`pytest-data-from-files` *last release*: Oct 13, 2021, @@ -3577,6 +4185,20 @@ This list contains 1487 plugins. pytest plugin to provide data from files loaded automatically + :pypi:`pytest-dataguard` + *last release*: Oct 08, 2025, + *status*: N/A, + *requires*: pytest>=8.4.2 + + Data validation and integrity testing for your datasets using pytest. + + :pypi:`pytest-data-loader` + *last release*: Oct 29, 2025, + *status*: 4 - Beta, + *requires*: pytest<9,>=7.0.0 + + Pytest plugin for loading test data for data-driven testing (DDT) + :pypi:`pytest-dataplugin` *last release*: Sep 16, 2017, *status*: 1 - Planning, @@ -3585,7 +4207,7 @@ This list contains 1487 plugins. A pytest plugin for managing an archive of test data. :pypi:`pytest-datarecorder` - *last release*: Feb 15, 2024, + *last release*: Jul 31, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -3613,9 +4235,9 @@ This list contains 1487 plugins. A pytest plugin for test driven data-wrangling (this is the development version of datatest's pytest integration). :pypi:`pytest-db` - *last release*: Dec 04, 2019, + *last release*: Aug 22, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest Session scope fixture "db" for mysql query or change @@ -3661,10 +4283,17 @@ This list contains 1487 plugins. Pytest extension for dbt. + :pypi:`pytest-dbt-duckdb` + *last release*: Oct 28, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3.4 + + Fearless testing for dbt models, powered by DuckDB. + :pypi:`pytest-dbt-postgres` - *last release*: Jan 02, 2024, + *last release*: Sep 03, 2024, *status*: N/A, - *requires*: pytest (>=7.4.3,<8.0.0) + *requires*: pytest<9.0.0,>=8.3.2 Pytest tooling to unittest DBT & Postgres models @@ -3703,6 +4332,13 @@ This list contains 1487 plugins. Identifies duplicate unit tests + :pypi:`pytest-deepassert` + *last release*: Sep 02, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=7.0.0 + + A pytest plugin for enhanced assertion reporting with detailed diffs + :pypi:`pytest-deepcov` *last release*: Mar 30, 2021, *status*: N/A, @@ -3710,12 +4346,19 @@ This list contains 1487 plugins. deepcov - :pypi:`pytest-defer` - *last release*: Aug 24, 2021, + :pypi:`pytest_defer` + *last release*: Nov 13, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest>=8.3 + + A 'defer' fixture for pytest + :pypi:`pytest-delta` + *last release*: Oct 27, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0 + Run only tests impacted by your code changes (delta-based selection) for pytest. :pypi:`pytest-demo-plugin` *last release*: May 15, 2021, @@ -3738,6 +4381,13 @@ This list contains 1487 plugins. Tests that depend on other tests + :pypi:`pytest-depper` + *last release*: Oct 23, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + Smart test selection based on AST-level code dependency analysis + :pypi:`pytest-deprecate` *last release*: Jul 01, 2019, *status*: N/A, @@ -3745,10 +4395,17 @@ This list contains 1487 plugins. Mark tests as testing a deprecated feature with a warning note. + :pypi:`pytest-deprecator` + *last release*: Dec 02, 2024, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A simple plugin to use with pytest + :pypi:`pytest-describe` - *last release*: Feb 10, 2024, + *last release*: Oct 23, 2025, *status*: 5 - Production/Stable, - *requires*: pytest <9,>=4.6 + *requires*: pytest<9,>=6 Describe-style plugin for pytest @@ -3760,19 +4417,26 @@ This list contains 1487 plugins. plugin for rich text descriptions :pypi:`pytest-deselect-if` - *last release*: Mar 24, 2024, + *last release*: Dec 26, 2024, *status*: 4 - Beta, *requires*: pytest>=6.2.0 A plugin to deselect pytests tests rather than using skipif :pypi:`pytest-devpi-server` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest DevPI server fixture for py.test + :pypi:`pytest-dfm` + *last release*: Sep 13, 2025, + *status*: N/A, + *requires*: pytest + + pytest-dfm provides a pytest integration for DV Flow Manager, a build system for silicon design + :pypi:`pytest-dhos` *last release*: Sep 07, 2022, *status*: N/A, @@ -3808,13 +4472,6 @@ This list contains 1487 plugins. A simple plugin to use with pytest - :pypi:`pytest-diffeo` - *last release*: Feb 20, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-diff-selector` *last release*: Feb 24, 2022, *status*: 4 - Beta, @@ -3829,6 +4486,13 @@ This list contains 1487 plugins. PyTest plugin for generating Difido reports + :pypi:`pytest-directives` + *last release*: Aug 11, 2025, + *status*: 3 - Alpha, + *requires*: pytest + + Control your tests flow + :pypi:`pytest-dir-equal` *last release*: Dec 11, 2023, *status*: 4 - Beta, @@ -3837,7 +4501,7 @@ This list contains 1487 plugins. pytest-dir-equals is a pytest plugin providing helpers to assert directories equality allowing golden testing :pypi:`pytest-dirty` - *last release*: Jul 11, 2024, + *last release*: Jun 08, 2025, *status*: 3 - Alpha, *requires*: pytest>=8.2; extra == "dev" @@ -3893,9 +4557,9 @@ This list contains 1487 plugins. pytest-ditto plugin for pyarrow tables. :pypi:`pytest-django` - *last release*: Jan 30, 2024, + *last release*: Apr 03, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=7.0.0 + *requires*: pytest>=7.0.0 A Django plugin for pytest. @@ -3907,8 +4571,8 @@ This list contains 1487 plugins. A Django plugin for pytest. :pypi:`pytest-djangoapp` - *last release*: May 19, 2023, - *status*: 4 - Beta, + *last release*: Sep 28, 2025, + *status*: 5 - Production/Stable, *requires*: pytest Nice pytest plugin to help you with Django pluggable application testing. @@ -3977,7 +4641,7 @@ This list contains 1487 plugins. Cleanup your Haystack indexes between tests :pypi:`pytest-django-ifactory` - *last release*: Aug 27, 2023, + *last release*: Apr 30, 2025, *status*: 5 - Production/Stable, *requires*: N/A @@ -3991,7 +4655,7 @@ This list contains 1487 plugins. The bare minimum to integrate py.test with Django. :pypi:`pytest-django-liveserver-ssl` - *last release*: Jan 20, 2022, + *last release*: Jan 09, 2025, *status*: 3 - Alpha, *requires*: N/A @@ -4068,14 +4732,14 @@ This list contains 1487 plugins. An RST Documentation Generator for pytest-based test suites :pypi:`pytest-docker` - *last release*: Feb 02, 2024, + *last release*: Jul 04, 2025, *status*: N/A, - *requires*: pytest <9.0,>=4.0 + *requires*: pytest<9.0,>=4.0 Simple pytest fixtures for Docker and Docker Compose based tests :pypi:`pytest-docker-apache-fixtures` - *last release*: Feb 16, 2022, + *last release*: Aug 12, 2024, *status*: 4 - Beta, *requires*: pytest @@ -4103,9 +4767,9 @@ This list contains 1487 plugins. Manages Docker containers during your integration tests :pypi:`pytest-docker-compose-v2` - *last release*: Feb 28, 2024, + *last release*: Dec 11, 2024, *status*: 4 - Beta, - *requires*: pytest<8,>=7.2.2 + *requires*: pytest>=7.2.2 Manages Docker containers during your integration tests @@ -4117,21 +4781,21 @@ This list contains 1487 plugins. A plugin to use docker databases for pytests :pypi:`pytest-docker-fixtures` - *last release*: Apr 03, 2024, + *last release*: Jun 25, 2025, *status*: 3 - Alpha, - *requires*: N/A + *requires*: pytest pytest docker fixtures :pypi:`pytest-docker-git-fixtures` - *last release*: Feb 09, 2022, + *last release*: Aug 12, 2024, *status*: 4 - Beta, *requires*: pytest Pytest fixtures for testing with git scm. :pypi:`pytest-docker-haproxy-fixtures` - *last release*: Feb 09, 2022, + *last release*: Aug 12, 2024, *status*: 4 - Beta, *requires*: pytest @@ -4159,7 +4823,7 @@ This list contains 1487 plugins. Easy to use, simple to extend, pytest plugin that minimally leverages docker-py. :pypi:`pytest-docker-registry-fixtures` - *last release*: Apr 08, 2022, + *last release*: Aug 12, 2024, *status*: 4 - Beta, *requires*: pytest @@ -4173,16 +4837,16 @@ This list contains 1487 plugins. pytest plugin to start docker container :pypi:`pytest-docker-squid-fixtures` - *last release*: Feb 09, 2022, + *last release*: Aug 12, 2024, *status*: 4 - Beta, *requires*: pytest Pytest fixtures for testing with squid. :pypi:`pytest-docker-tools` - *last release*: Feb 17, 2022, + *last release*: Mar 16, 2025, *status*: 4 - Beta, - *requires*: pytest (>=6.0.1) + *requires*: pytest>=6.0.1 Docker integration tests for pytest @@ -4228,10 +4892,17 @@ This list contains 1487 plugins. Run pytest --doctest-modules with markdown docstrings in code blocks (\`\`\`) + :pypi:`pytest-doctest-only` + *last release*: Jul 30, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3.0 + + A plugin to run only doctest + :pypi:`pytest-doctestplus` - *last release*: Mar 10, 2024, + *last release*: Oct 18, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=4.6 + *requires*: pytest>=4.6 Pytest plugin with advanced doctest features. @@ -4284,6 +4955,13 @@ This list contains 1487 plugins. A py.test plugin that parses environment files before running tests + :pypi:`pytest-dotenv-modern` + *last release*: Sep 27, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.0.0 + + A modern pytest plugin that loads environment variables from dotenv files + :pypi:`pytest-dot-only-pkcopley` *last release*: Oct 27, 2023, *status*: N/A, @@ -4291,6 +4969,20 @@ This list contains 1487 plugins. A Pytest marker for only running a single test + :pypi:`pytest-dparam` + *last release*: Aug 27, 2024, + *status*: 6 - Mature, + *requires*: pytest + + A more readable alternative to @pytest.mark.parametrize. + + :pypi:`pytest-dpg` + *last release*: Aug 13, 2024, + *status*: N/A, + *requires*: N/A + + pytest-dpg is a pytest plugin for testing Dear PyGui (DPG) applications + :pypi:`pytest-draw` *last release*: Mar 21, 2023, *status*: 3 - Alpha, @@ -4305,6 +4997,13 @@ This list contains 1487 plugins. A Django REST framework plugin for pytest. + :pypi:`pytest-drill-sergeant` + *last release*: Sep 12, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + A pytest plugin that enforces test quality standards through automatic marker detection and AAA structure validation + :pypi:`pytest-drivings` *last release*: Jan 13, 2021, *status*: N/A, @@ -4319,13 +5018,41 @@ This list contains 1487 plugins. A Pytest plugin to drop duplicated tests during collection + :pypi:`pytest-dryci` + *last release*: Sep 27, 2024, + *status*: 4 - Beta, + *requires*: N/A + + Test caching plugin for pytest + :pypi:`pytest-dryrun` - *last release*: Jul 18, 2023, + *last release*: Jan 19, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=7.4.0,<8.0.0) + *requires*: pytest<9,>=7.40 A Pytest plugin to ignore tests during collection without reporting them in the test summary. + :pypi:`pytest-dsl` + *last release*: Oct 31, 2025, + *status*: N/A, + *requires*: pytest>=7.0.0 + + A DSL testing framework based on pytest + + :pypi:`pytest-dsl-ssh` + *last release*: Jul 25, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + SSH/SFTP关键字插件,为pytest-dsl提供SSH和SFTP操作能力 + + :pypi:`pytest-dsl-ui` + *last release*: Aug 21, 2025, + *status*: N/A, + *requires*: pytest>=7.0.0; extra == "dev" + + Playwright-based UI automation keywords for pytest-dsl framework + :pypi:`pytest-dummynet` *last release*: Dec 15, 2021, *status*: 5 - Production/Stable, @@ -4341,19 +5068,26 @@ This list contains 1487 plugins. A pytest plugin for dumping test results to json. :pypi:`pytest-duration-insights` - *last release*: Jun 25, 2021, + *last release*: Jul 15, 2024, *status*: N/A, *requires*: N/A :pypi:`pytest-durations` - *last release*: Apr 22, 2022, + *last release*: Aug 29, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=4.6) + *requires*: pytest>=4.6 Pytest plugin reporting fixtures and test functions execution time. + :pypi:`pytest-dynamic-parameterize` + *last release*: Oct 14, 2025, + *status*: N/A, + *requires*: pytest + + A Python package for managing pytest plugins. + :pypi:`pytest-dynamicrerun` *last release*: Aug 15, 2020, *status*: 4 - Beta, @@ -4362,7 +5096,7 @@ This list contains 1487 plugins. A pytest plugin to rerun tests dynamically based off of test outcome and output. :pypi:`pytest-dynamodb` - *last release*: Mar 12, 2024, + *last release*: Apr 04, 2025, *status*: 5 - Production/Stable, *requires*: pytest @@ -4375,13 +5109,6 @@ This list contains 1487 plugins. pytest-easy-addoption: Easy way to work with pytest addoption - :pypi:`pytest-easy-api` - *last release*: Feb 16, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-easyMPI` *last release*: Oct 21, 2020, *status*: N/A, @@ -4418,14 +5145,14 @@ This list contains 1487 plugins. Pytest execution on EC2 instance :pypi:`pytest-echo` - *last release*: Dec 05, 2023, + *last release*: Apr 27, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=2.2 + *requires*: pytest>=8.3.3 - pytest plugin with mechanisms for echoing environment variables, package version and generic attributes + pytest plugin that allows to dump environment variables, package version and generic attributes :pypi:`pytest-edit` - *last release*: Jun 09, 2024, + *last release*: Nov 17, 2024, *status*: N/A, *requires*: pytest @@ -4439,9 +5166,16 @@ This list contains 1487 plugins. Pytest plugin to select test using Ekstazi algorithm :pypi:`pytest-elasticsearch` - *last release*: Mar 15, 2024, + *last release*: Dec 03, 2024, *status*: 5 - Production/Stable, - *requires*: pytest >=7.0 + *requires*: pytest>=7.0 + + Elasticsearch fixtures and fixture factories for Pytest. + + :pypi:`pytest-elasticsearch-test` + *last release*: Apr 20, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7.0 Elasticsearch fixtures and fixture factories for Pytest. @@ -4460,7 +5194,7 @@ This list contains 1487 plugins. An eliot plugin for pytest. :pypi:`pytest-elk-reporter` - *last release*: Apr 04, 2024, + *last release*: Jul 25, 2024, *status*: 4 - Beta, *requires*: pytest>=3.5.0 @@ -4474,56 +5208,63 @@ This list contains 1487 plugins. Send execution result email :pypi:`pytest-embedded` - *last release*: May 31, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=7.0 A pytest plugin that designed for embedded testing. :pypi:`pytest-embedded-arduino` - *last release*: May 23, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with Arduino. :pypi:`pytest-embedded-idf` - *last release*: May 23, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with ESP-IDF. :pypi:`pytest-embedded-jtag` - *last release*: May 23, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with JTAG. + :pypi:`pytest-embedded-nuttx` + *last release*: Oct 27, 2025, + *status*: 5 - Production/Stable, + *requires*: N/A + + Make pytest-embedded plugin work with NuttX. + :pypi:`pytest-embedded-qemu` - *last release*: May 23, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with QEMU. :pypi:`pytest-embedded-serial` - *last release*: May 31, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with Serial. :pypi:`pytest-embedded-serial-esp` - *last release*: May 31, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A Make pytest-embedded plugin work with Espressif target boards. :pypi:`pytest-embedded-wokwi` - *last release*: May 23, 2024, + *last release*: Oct 27, 2025, *status*: 5 - Production/Stable, *requires*: N/A @@ -4551,9 +5292,9 @@ This list contains 1487 plugins. Pytest plugin to represent test output with emoji support :pypi:`pytest-enabler` - *last release*: Mar 21, 2024, + *last release*: May 16, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=6; extra == "testing" + *requires*: pytest!=8.1.*,>=6; extra == "test" Enable installed pytest plugins @@ -4600,9 +5341,9 @@ This list contains 1487 plugins. Improvements for pytest (rejected upstream) :pypi:`pytest-env` - *last release*: Nov 28, 2023, + *last release*: Oct 09, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=7.4.3 + *requires*: pytest>=8.4.2 pytest plugin that allows you to add environment variables. @@ -4641,6 +5382,13 @@ This list contains 1487 plugins. Pytest plugin to validate use of envvars on your tests + :pypi:`pytest-envx` + *last release*: Jun 28, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.4.1 + + Pytest plugin for managing environment variables with interpolation and .env file support. + :pypi:`pytest-env-yaml` *last release*: Apr 02, 2019, *status*: N/A, @@ -4669,6 +5417,20 @@ This list contains 1487 plugins. Pytest plugin to treat skipped tests a test failure + :pypi:`pytest-errxfail` + *last release*: Jan 06, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + pytest plugin to mark a test as xfailed if it fails with the specified error message in the captured output + + :pypi:`pytest-essentials` + *last release*: May 19, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=7.0 + + A Pytest plugin providing essential utilities like soft assertions. + :pypi:`pytest-eth` *last release*: Aug 14, 2020, *status*: 1 - Planning, @@ -4690,6 +5452,13 @@ This list contains 1487 plugins. Pytest Plugin for BDD + :pypi:`pytest-evals` + *last release*: Feb 02, 2025, + *status*: N/A, + *requires*: pytest>=7.0.0 + + A pytest plugin for running and analyzing LLM evaluation tests + :pypi:`pytest-eventlet` *last release*: Oct 04, 2021, *status*: N/A, @@ -4697,8 +5466,15 @@ This list contains 1487 plugins. Applies eventlet monkey-patch as a pytest plugin. - :pypi:`pytest-evm` - *last release*: Apr 22, 2024, + :pypi:`pytest-everyfunc` + *last release*: Apr 30, 2025, + *status*: 4 - Beta, + *requires*: pytest + + A pytest plugin to detect completely untested functions using coverage + + :pypi:`pytest_evm` + *last release*: Sep 23, 2024, *status*: 4 - Beta, *requires*: pytest<9.0.0,>=8.1.1 @@ -4712,30 +5488,51 @@ This list contains 1487 plugins. Parse queries in Lucene and Elasticsearch syntaxes :pypi:`pytest-examples` - *last release*: Jul 02, 2024, - *status*: 4 - Beta, + *last release*: May 06, 2025, + *status*: N/A, *requires*: pytest>=7 Pytest plugin for testing examples in docstrings and markdown files. + :pypi:`pytest-exasol-backend` + *last release*: Oct 29, 2025, + *status*: N/A, + *requires*: pytest<9,>=7 + + + + :pypi:`pytest-exasol-extension` + *last release*: Oct 29, 2025, + *status*: N/A, + *requires*: pytest<9,>=7 + + + :pypi:`pytest-exasol-itde` - *last release*: Jul 01, 2024, + *last release*: Nov 22, 2024, *status*: N/A, *requires*: pytest<9,>=7 :pypi:`pytest-exasol-saas` - *last release*: Jun 07, 2024, + *last release*: Nov 22, 2024, + *status*: N/A, + *requires*: pytest<9,>=7 + + + + :pypi:`pytest-exasol-slc` + *last release*: Oct 30, 2025, *status*: N/A, *requires*: pytest<9,>=7 :pypi:`pytest-excel` - *last release*: Jun 18, 2024, + *last release*: Jul 22, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>3.6 + *requires*: pytest pytest plugin for generating excel reports @@ -4774,6 +5571,13 @@ This list contains 1487 plugins. A pytest plugin that overrides the built-in exit codes to retain more information about the test results. + :pypi:`pytest-exit-status` + *last release*: Jan 25, 2025, + *status*: N/A, + *requires*: pytest>=8.0.0 + + Enhance. + :pypi:`pytest-expect` *last release*: Apr 21, 2016, *status*: 4 - Beta, @@ -4788,6 +5592,13 @@ This list contains 1487 plugins. A pytest plugin to provide initial/expected directories, and check a test transforms the initial directory to the expected one + :pypi:`pytest-expected` + *last release*: Feb 26, 2025, + *status*: N/A, + *requires*: pytest + + Record and play back your expectations + :pypi:`pytest-expecter` *last release*: Sep 18, 2022, *status*: 5 - Production/Stable, @@ -4824,9 +5635,9 @@ This list contains 1487 plugins. A Pytest plugin to ignore certain marked tests by default :pypi:`pytest-exploratory` - *last release*: Aug 18, 2023, + *last release*: Sep 18, 2024, *status*: N/A, - *requires*: pytest (>=6.2) + *requires*: pytest>=6.2 Interactive console for pytest. @@ -4844,6 +5655,13 @@ This list contains 1487 plugins. pytest plugin for automation test + :pypi:`pytest-extended-mock` + *last release*: Mar 12, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.3.5 + + a pytest extension for easy mock setup + :pypi:`pytest-extensions` *last release*: Aug 17, 2022, *status*: 4 - Beta, @@ -4879,6 +5697,13 @@ This list contains 1487 plugins. Additional pytest markers to dynamically enable/disable tests viia CLI flags + :pypi:`pytest-f3ts` + *last release*: Jul 15, 2025, + *status*: N/A, + *requires*: pytest<8.0.0,>=7.2.1 + + Pytest Plugin for communicating test results and information to a FixturFab Test Runner GUI + :pypi:`pytest-fabric` *last release*: Sep 12, 2018, *status*: 5 - Production/Stable, @@ -4886,13 +5711,6 @@ This list contains 1487 plugins. Provides test utilities to run fabric task tests by using docker containers - :pypi:`pytest-factor` - *last release*: Feb 20, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-factory` *last release*: Sep 06, 2020, *status*: 3 - Alpha, @@ -4901,9 +5719,9 @@ This list contains 1487 plugins. Use factories for test setup with py.test :pypi:`pytest-factoryboy` - *last release*: Mar 05, 2024, + *last release*: Jul 01, 2025, *status*: 6 - Mature, - *requires*: pytest (>=6.2) + *requires*: pytest>=7.0 Factory Boy support for pytest. @@ -4949,6 +5767,13 @@ This list contains 1487 plugins. Fail tests that take too long to run + :pypi:`pytest-failure-tracker` + *last release*: Jul 17, 2024, + *status*: N/A, + *requires*: pytest>=6.0.0 + + A pytest plugin for tracking test failures over multiple runs + :pypi:`pytest-faker` *last release*: Dec 19, 2016, *status*: 6 - Mature, @@ -4963,13 +5788,6 @@ This list contains 1487 plugins. Pytest helpers for Falcon. - :pypi:`pytest-falcon-client` - *last release*: Feb 21, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-fantasy` *last release*: Mar 14, 2019, *status*: N/A, @@ -5013,7 +5831,7 @@ This list contains 1487 plugins. py.test plugin that activates the fault handler module for tests (dummy package) :pypi:`pytest-fauna` - *last release*: May 30, 2024, + *last release*: Jan 03, 2025, *status*: N/A, *requires*: N/A @@ -5083,9 +5901,9 @@ This list contains 1487 plugins. Pytest plugin for filtering based on sub-packages :pypi:`pytest-find-dependencies` - *last release*: Mar 16, 2024, - *status*: 4 - Beta, - *requires*: pytest >=4.3.0 + *last release*: Jul 16, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=6.2.4 A pytest plugin to find dependencies between tests @@ -5097,19 +5915,33 @@ This list contains 1487 plugins. A pytest plugin to treat non-assertion failures as test errors. :pypi:`pytest-firefox` - *last release*: Aug 08, 2017, + *last release*: Feb 28, 2025, + *status*: N/A, + *requires*: N/A + + + + :pypi:`pytest-fixturecheck` + *last release*: Jun 02, 2025, *status*: 3 - Alpha, - *requires*: pytest (>=3.0.2) + *requires*: pytest>=6.0.0 - pytest plugin to manipulate firefox + A pytest plugin to check fixture validity before test execution :pypi:`pytest-fixture-classes` - *last release*: Sep 02, 2023, + *last release*: Oct 12, 2025, *status*: 5 - Production/Stable, - *requires*: pytest + *requires*: N/A Fixtures as classes that work well with dependency injection, autocompletetion, type checkers, and language servers + :pypi:`pytest-fixture-collect` + *last release*: Jul 25, 2025, + *status*: N/A, + *requires*: pytest; extra == "test" + + A utility to collect pytest fixture file paths. + :pypi:`pytest-fixturecollection` *last release*: Feb 22, 2024, *status*: 4 - Beta, @@ -5118,12 +5950,19 @@ This list contains 1487 plugins. A pytest plugin to collect tests based on fixtures being used by tests :pypi:`pytest-fixture-config` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest Fixture configuration utils for py.test + :pypi:`pytest-fixture-forms` + *last release*: Dec 06, 2024, + *status*: N/A, + *requires*: pytest<9.0.0,>=7.0.0 + + A pytest plugin for creating fixtures that holds different forms between tests. + :pypi:`pytest-fixture-maker` *last release*: Sep 21, 2021, *status*: N/A, @@ -5139,9 +5978,9 @@ This list contains 1487 plugins. A pytest plugin to add markers based on fixtures used. :pypi:`pytest-fixture-order` - *last release*: May 16, 2022, + *last release*: Oct 22, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=3.0) + *requires*: pytest>=3.0 pytest plugin to control fixture evaluation order @@ -5173,8 +6012,15 @@ This list contains 1487 plugins. Common fixtures for pytest + :pypi:`pytest-fixtures-fixtures` + *last release*: Sep 14, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.4.1 + + Handy fixtues to access your fixtures from your _pytest tests. + :pypi:`pytest-fixture-tools` - *last release*: Aug 18, 2020, + *last release*: Apr 30, 2025, *status*: 6 - Mature, *requires*: pytest @@ -5188,14 +6034,14 @@ This list contains 1487 plugins. A pytest plugin to assert type annotations at runtime. :pypi:`pytest-flake8` - *last release*: Mar 18, 2022, - *status*: 4 - Beta, - *requires*: pytest (>=7.0) + *last release*: Nov 09, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest>=7.0 pytest plugin to check FLAKE8 requirements :pypi:`pytest-flake8-path` - *last release*: Jul 10, 2023, + *last release*: Sep 09, 2025, *status*: 5 - Production/Stable, *requires*: pytest @@ -5208,6 +6054,13 @@ This list contains 1487 plugins. pytest plugin to check FLAKE8 requirements + :pypi:`pytest-flake-detection` + *last release*: Nov 29, 2024, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + Continuously runs your tests to detect flaky tests + :pypi:`pytest-flakefinder` *last release*: Oct 26, 2022, *status*: 4 - Beta, @@ -5265,7 +6118,7 @@ This list contains 1487 plugins. :pypi:`pytest-fluent` - *last release*: Jun 05, 2024, + *last release*: Aug 14, 2024, *status*: 4 - Beta, *requires*: pytest>=7.0.0 @@ -5279,11 +6132,11 @@ This list contains 1487 plugins. A pytest plugin in order to provide logs via fluentbit :pypi:`pytest-fly` - *last release*: Apr 14, 2024, + *last release*: Jun 07, 2025, *status*: 3 - Alpha, *requires*: pytest - pytest observer + pytest runner and observer :pypi:`pytest-flyte` *last release*: May 03, 2021, @@ -5292,6 +6145,13 @@ This list contains 1487 plugins. Pytest fixtures for simplifying Flyte integration testing + :pypi:`pytest-fmu-filter` + *last release*: Jun 23, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + A pytest plugin to filter fmus + :pypi:`pytest-focus` *last release*: May 04, 2019, *status*: 4 - Beta, @@ -5313,13 +6173,6 @@ This list contains 1487 plugins. py.test plugin to make the test failing regardless of pytest.mark.xfail - :pypi:`pytest-forks` - *last release*: Mar 05, 2024, - *status*: N/A, - *requires*: N/A - - Fork helper for pytest - :pypi:`pytest-forward-compatability` *last release*: Sep 06, 2020, *status*: N/A, @@ -5335,14 +6188,21 @@ This list contains 1487 plugins. A pytest plugin to shim pytest commandline options for fowards compatibility :pypi:`pytest-frappe` - *last release*: Oct 29, 2023, + *last release*: Jul 30, 2024, *status*: 4 - Beta, *requires*: pytest>=7.0.0 Pytest Frappe Plugin - A set of pytest fixtures to test Frappe applications + :pypi:`pytest-freethreaded` + *last release*: Oct 03, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest + + pytest plugin for running parallel tests + :pypi:`pytest-freezeblaster` - *last release*: Jul 10, 2024, + *last release*: Oct 13, 2025, *status*: N/A, *requires*: pytest>=6.2.5 @@ -5356,9 +6216,9 @@ This list contains 1487 plugins. Wrap tests with fixtures in freeze_time :pypi:`pytest-freezer` - *last release*: Jun 21, 2023, + *last release*: Dec 12, 2024, *status*: N/A, - *requires*: pytest >= 3.6 + *requires*: pytest>=3.6 Pytest plugin providing a fixture interface for spulec/freezegun @@ -5383,6 +6243,13 @@ This list contains 1487 plugins. Pytest plugin for measuring function coverage + :pypi:`pytest-funcnodes` + *last release*: Mar 19, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + Testing plugin for funcnodes + :pypi:`pytest-funparam` *last release*: Dec 02, 2021, *status*: 4 - Beta, @@ -5390,6 +6257,13 @@ This list contains 1487 plugins. An alternative way to parametrize test cases. + :pypi:`pytest-fv` + *last release*: Jun 06, 2025, + *status*: N/A, + *requires*: pytest + + pytest extensions to support running functional-verification jobs + :pypi:`pytest-fxa` *last release*: Aug 28, 2018, *status*: 5 - Production/Stable, @@ -5397,6 +6271,13 @@ This list contains 1487 plugins. pytest plugin for Firefox Accounts + :pypi:`pytest-fxa-mte` + *last release*: Oct 02, 2024, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for Firefox Accounts + :pypi:`pytest-fxtest` *last release*: Oct 27, 2020, *status*: N/A, @@ -5405,7 +6286,7 @@ This list contains 1487 plugins. :pypi:`pytest-fzf` - *last release*: Jul 03, 2024, + *last release*: Jan 06, 2025, *status*: 4 - Beta, *requires*: pytest>=6.0.0 @@ -5418,10 +6299,17 @@ This list contains 1487 plugins. pytest plugin for apps written with Google's AppEngine + :pypi:`pytest-gak` + *last release*: Apr 10, 2025, + *status*: N/A, + *requires*: N/A + + A Pytest plugin and command line tool for interactive testing with Pytest + :pypi:`pytest-gather-fixtures` - *last release*: Apr 12, 2022, + *last release*: Aug 18, 2024, *status*: N/A, - *requires*: pytest (>=6.0.0) + *requires*: pytest>=7.0.0 set up asynchronous pytest fixtures concurrently @@ -5440,14 +6328,14 @@ This list contains 1487 plugins. Uses gcov to measure test coverage of a C library :pypi:`pytest-gcs` - *last release*: Mar 01, 2024, + *last release*: Jan 24, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=6.2 + *requires*: pytest>=6.2 GCS fixtures and fixture factories for Pytest. :pypi:`pytest-gee` - *last release*: Jun 30, 2024, + *last release*: Oct 16, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -5482,25 +6370,25 @@ This list contains 1487 plugins. For finding/executing Ghost Inspector tests :pypi:`pytest-girder` - *last release*: Jul 08, 2024, + *last release*: Sep 30, 2025, *status*: N/A, *requires*: pytest>=3.6 A set of pytest fixtures for testing Girder applications. :pypi:`pytest-git` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest Git repository fixture for py.test :pypi:`pytest-gitconfig` - *last release*: Oct 15, 2023, + *last release*: Oct 12, 2025, *status*: 4 - Beta, *requires*: pytest>=7.1.2 - Provide a gitconfig sandbox for testing + Provide a Git config sandbox for testing :pypi:`pytest-gitcov` *last release*: Jan 11, 2020, @@ -5531,9 +6419,9 @@ This list contains 1487 plugins. Plugin for py.test that associates tests with github issues using a marker. :pypi:`pytest-github-actions-annotate-failures` - *last release*: May 04, 2023, + *last release*: Jan 17, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=4.0.0) + *requires*: pytest>=6.0.0 pytest plugin to annotate failed tests with a workflow command for GitHub Actions @@ -5551,6 +6439,13 @@ This list contains 1487 plugins. py.test plugin to ignore the same files as git + :pypi:`pytest-gitlab` + *last release*: Oct 16, 2024, + *status*: N/A, + *requires*: N/A + + Pytest Plugin for Gitlab + :pypi:`pytest-gitlabci-parallelized` *last release*: Mar 08, 2023, *status*: N/A, @@ -5559,7 +6454,7 @@ This list contains 1487 plugins. Parallelize pytest across GitLab CI workers. :pypi:`pytest-gitlab-code-quality` - *last release*: Apr 03, 2024, + *last release*: Sep 09, 2024, *status*: N/A, *requires*: pytest>=8.1.1 @@ -5572,6 +6467,13 @@ This list contains 1487 plugins. Folds output sections in GitLab CI build log + :pypi:`pytest-gitscope` + *last release*: Sep 24, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7.0.0 + + A pragmatic pytest plugin that runs only the tests that matter, and ship faster + :pypi:`pytest-git-selector` *last release*: Nov 17, 2022, *status*: N/A, @@ -5580,9 +6482,9 @@ This list contains 1487 plugins. Utility to select tests that have had its dependencies modified (as identified by git diff) :pypi:`pytest-glamor-allure` - *last release*: Apr 30, 2024, + *last release*: Jul 20, 2025, *status*: 4 - Beta, - *requires*: pytest<=8.2.0 + *requires*: pytest<=8.4.1 Extends allure-pytest functionality @@ -5614,6 +6516,27 @@ This list contains 1487 plugins. Notify google chat channel for test results + :pypi:`pytest-google-cloud-storage` + *last release*: Sep 11, 2025, + *status*: N/A, + *requires*: pytest>=8.0.0 + + Pytest custom features, e.g. fixtures and various tests. Aimed to emulate Google Cloud Storage service + + :pypi:`pytest-grader` + *last release*: Aug 25, 2025, + *status*: N/A, + *requires*: pytest>=8 + + Pytest extension for scoring programming assignments. + + :pypi:`pytest-gradescope` + *last release*: Apr 29, 2025, + *status*: N/A, + *requires*: N/A + + A pytest plugin for Gradescope integration + :pypi:`pytest-graphql-schema` *last release*: Oct 18, 2019, *status*: N/A, @@ -5628,6 +6551,20 @@ This list contains 1487 plugins. Green progress dots + :pypi:`pytest-greener` + *last release*: Oct 18, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.3.3 + + Pytest plugin for Greener + + :pypi:`pytest-greet` + *last release*: Oct 21, 2025, + *status*: N/A, + *requires*: N/A + + + :pypi:`pytest-group-by-class` *last release*: Jun 27, 2023, *status*: 5 - Production/Stable, @@ -5649,10 +6586,17 @@ This list contains 1487 plugins. pytest plugin for grpc + :pypi:`pytest-grpc-aio` + *last release*: Oct 28, 2025, + *status*: N/A, + *requires*: pytest>=3.6.0 + + pytest plugin for grpc.aio + :pypi:`pytest-grunnur` - *last release*: Feb 05, 2023, + *last release*: Jul 26, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest>=6 Py.Test plugin for Grunnur-based packages. @@ -5692,14 +6636,14 @@ This list contains 1487 plugins. Store data created during your pytest tests execution, and retrieve it at the end of the session, e.g. for applicative benchmarking purposes. :pypi:`pytest-helm-charts` - *last release*: Feb 07, 2024, + *last release*: Oct 31, 2024, *status*: 4 - Beta, - *requires*: pytest (>=8.0.0,<9.0.0) + *requires*: pytest<9.0.0,>=8.0.0 A plugin to provide different types and configs of Kubernetes clusters that can be used for testing. :pypi:`pytest-helm-templates` - *last release*: May 08, 2024, + *last release*: Aug 07, 2024, *status*: N/A, *requires*: pytest~=7.4.0; extra == "dev" @@ -5769,7 +6713,7 @@ This list contains 1487 plugins. Pytest plugin to keep a history of your pytest runs :pypi:`pytest-home` - *last release*: Oct 09, 2023, + *last release*: Jul 28, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -5783,9 +6727,9 @@ This list contains 1487 plugins. A pytest plugin for use with homeassistant custom components. :pypi:`pytest-homeassistant-custom-component` - *last release*: Jul 11, 2024, + *last release*: Oct 31, 2025, *status*: 3 - Alpha, - *requires*: pytest==8.2.0 + *requires*: pytest==8.4.2 Experimental package to automatically extract test plugins for Home Assistant custom components @@ -5804,7 +6748,7 @@ This list contains 1487 plugins. Report on tests that honor constraints, and guard against regressions :pypi:`pytest-hot-reloading` - *last release*: Apr 18, 2024, + *last release*: Sep 23, 2024, *status*: N/A, *requires*: N/A @@ -5818,7 +6762,7 @@ This list contains 1487 plugins. A plugin that tracks test changes :pypi:`pytest-houdini` - *last release*: Jul 05, 2024, + *last release*: Jul 15, 2024, *status*: N/A, *requires*: pytest @@ -5852,10 +6796,17 @@ This list contains 1487 plugins. pytest plugin for generating HTML reports + :pypi:`pytest-html5` + *last release*: Oct 11, 2025, + *status*: N/A, + *requires*: N/A + + the best report for pytest + :pypi:`pytest-html-cn` - *last release*: Aug 01, 2023, + *last release*: Aug 19, 2024, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest!=6.0.0,>=5.0 pytest plugin for generating HTML reports @@ -5873,6 +6824,13 @@ This list contains 1487 plugins. Pytest HTML reports merging utility + :pypi:`pytest-html-nova-act` + *last release*: Sep 05, 2025, + *status*: N/A, + *requires*: N/A + + A Pytest Plugin for Amazon Nova Act Python SDK. + :pypi:`pytest-html-object-storage` *last release*: Jan 17, 2024, *status*: 5 - Production/Stable, @@ -5880,6 +6838,13 @@ This list contains 1487 plugins. Pytest report plugin for send HTML report on object-storage + :pypi:`pytest-html-plus` + *last release*: Oct 30, 2025, + *status*: N/A, + *requires*: N/A + + Get started with rich pytest reports in under 3 seconds. Just install the plugin — no setup required. The simplest, fastest reporter for pytest. + :pypi:`pytest-html-profiling` *last release*: Feb 11, 2020, *status*: 5 - Production/Stable, @@ -5887,6 +6852,13 @@ This list contains 1487 plugins. Pytest plugin for generating HTML reports with per-test profiling and optionally call graph visualizations. Based on pytest-html by Dave Hunt. + :pypi:`pytest-html-report` + *last release*: Jun 24, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.0 + + Enhanced HTML reporting for pytest with categories, specifications, and detailed logging + :pypi:`pytest-html-reporter` *last release*: Feb 13, 2022, *status*: N/A, @@ -5908,26 +6880,75 @@ This list contains 1487 plugins. pytest plugin for generating HTML reports + :pypi:`pytest-htmlx` + *last release*: Sep 09, 2025, + *status*: 4 - Beta, + *requires*: pytest + + Custom HTML report plugin for Pytest with charts and tables + :pypi:`pytest-http` - *last release*: Dec 05, 2019, + *last release*: Aug 22, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest Fixture "http" for http requests :pypi:`pytest-httpbin` - *last release*: May 08, 2023, + *last release*: Sep 18, 2024, *status*: 5 - Production/Stable, - *requires*: pytest ; extra == 'test' + *requires*: pytest; extra == "test" Easily test your HTTP library against a local copy of httpbin + :pypi:`pytest-httpchain` + *last release*: Aug 16, 2025, + *status*: 5 - Production/Stable, + *requires*: N/A + + pytest plugin for HTTP testing using JSON files + + :pypi:`pytest-httpchain-jsonref` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: N/A + + JSON reference ($ref) support for pytest-httpchain + + :pypi:`pytest-httpchain-mcp` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: N/A + + MCP server for pytest-httpchain + + :pypi:`pytest-httpchain-models` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: N/A + + Pydantic models for pytest-httpchain + + :pypi:`pytest-httpchain-templates` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: N/A + + Templating support for pytest-httpchain + + :pypi:`pytest-httpchain-userfunc` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: N/A + + User functions support for pytest-httpchain + :pypi:`pytest-httpdbg` - *last release*: Jan 10, 2024, - *status*: 3 - Alpha, - *requires*: pytest >=7.0.0 + *last release*: Oct 26, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 - A pytest plugin to record HTTP(S) requests with stack trace + A pytest plugin to record HTTP(S) requests with stack trace. :pypi:`pytest-http-mocker` *last release*: Oct 20, 2019, @@ -5944,23 +6965,23 @@ This list contains 1487 plugins. A thin wrapper of HTTPretty for pytest :pypi:`pytest_httpserver` - *last release*: Feb 24, 2024, + *last release*: Apr 10, 2025, *status*: 3 - Alpha, *requires*: N/A pytest-httpserver is a httpserver for pytest :pypi:`pytest-httptesting` - *last release*: May 08, 2024, + *last release*: Dec 19, 2024, *status*: N/A, - *requires*: pytest<9.0.0,>=8.2.0 + *requires*: pytest>=8.2.0 http_testing framework on top of pytest :pypi:`pytest-httpx` - *last release*: Feb 21, 2024, + *last release*: Nov 28, 2024, *status*: 5 - Production/Stable, - *requires*: pytest <9,>=7 + *requires*: pytest==8.* Send responses to httpx. @@ -6000,16 +7021,16 @@ This list contains 1487 plugins. help hypo module for pytest :pypi:`pytest-iam` - *last release*: Apr 22, 2024, - *status*: 3 - Alpha, + *last release*: Jul 25, 2025, + *status*: 4 - Beta, *requires*: pytest>=7.0.0 - A fully functional OAUTH2 / OpenID Connect (OIDC) server to be used in your testsuite + A fully functional OAUTH2 / OpenID Connect (OIDC) / SCIM server to be used in your testsuite :pypi:`pytest-ibutsu` - *last release*: Aug 05, 2022, + *last release*: Oct 21, 2025, *status*: 4 - Beta, - *requires*: pytest>=7.1 + *requires*: pytest A plugin to sent pytest results to an Ibutsu server @@ -6049,26 +7070,40 @@ This list contains 1487 plugins. ignore failures from flaky tests (pytest plugin) :pypi:`pytest-ignore-test-results` - *last release*: Aug 17, 2023, - *status*: 2 - Pre-Alpha, + *last release*: Feb 03, 2025, + *status*: 5 - Production/Stable, *requires*: pytest>=7.0 A pytest plugin to ignore test results. :pypi:`pytest-image-diff` - *last release*: Mar 09, 2023, + *last release*: Dec 31, 2024, *status*: 3 - Alpha, *requires*: pytest :pypi:`pytest-image-snapshot` - *last release*: Jul 01, 2024, + *last release*: Jul 16, 2025, *status*: 4 - Beta, *requires*: pytest>=3.5.0 A pytest plugin for image snapshot management and comparison. + :pypi:`pytest-impacted` + *last release*: Sep 11, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.0.0 + + A pytest plugin that selectively runs tests impacted by codechanges via git introspection, ASL parsing, and dependency graph analysis. + + :pypi:`pytest-import-check` + *last release*: Jul 19, 2024, + *status*: 3 - Alpha, + *requires*: pytest>=8.1 + + pytest plugin to check whether Python modules can be imported + :pypi:`pytest-incremental` *last release*: Apr 24, 2021, *status*: 5 - Production/Stable, @@ -6083,6 +7118,13 @@ This list contains 1487 plugins. + :pypi:`pytest-influx` + *last release*: Oct 16, 2024, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.3.3 + + Pytest plugin for managing your influx instance between test runs + :pypi:`pytest-influxdb` *last release*: Apr 20, 2021, *status*: N/A, @@ -6111,6 +7153,13 @@ This list contains 1487 plugins. display more node ininformation. + :pypi:`pytest-infrahouse` + *last release*: Oct 29, 2025, + *status*: 4 - Beta, + *requires*: pytest~=8.3 + + A set of fixtures to use with pytest + :pypi:`pytest-infrastructure` *last release*: Apr 12, 2020, *status*: 4 - Beta, @@ -6133,35 +7182,42 @@ This list contains 1487 plugins. Plugin for sending automation test data from Pytest to the initry :pypi:`pytest-inline` - *last release*: Oct 19, 2023, + *last release*: Oct 24, 2024, *status*: 4 - Beta, - *requires*: pytest >=7.0.0 + *requires*: pytest<9.0,>=7.0 - A pytest plugin for writing inline tests. + A pytest plugin for writing inline tests :pypi:`pytest-inmanta` - *last release*: Jul 05, 2024, + *last release*: Apr 09, 2025, *status*: 5 - Production/Stable, *requires*: pytest A py.test plugin providing fixtures to simplify inmanta modules testing. :pypi:`pytest-inmanta-extensions` - *last release*: Jul 05, 2024, + *last release*: Jul 04, 2025, *status*: 5 - Production/Stable, *requires*: N/A Inmanta tests package :pypi:`pytest-inmanta-lsm` - *last release*: Jul 06, 2024, + *last release*: Aug 26, 2025, *status*: 5 - Production/Stable, *requires*: N/A Common fixtures for inmanta LSM related modules + :pypi:`pytest-inmanta-srlinux` + *last release*: Apr 22, 2025, + *status*: 3 - Alpha, + *requires*: N/A + + Pytest library to facilitate end to end testing of inmanta projects + :pypi:`pytest-inmanta-yang` - *last release*: Feb 22, 2024, + *last release*: Oct 28, 2025, *status*: 4 - Beta, *requires*: pytest @@ -6175,7 +7231,7 @@ This list contains 1487 plugins. A simple image diff plugin for pytest :pypi:`pytest-in-robotframework` - *last release*: Mar 02, 2024, + *last release*: Nov 23, 2024, *status*: N/A, *requires*: pytest @@ -6209,6 +7265,13 @@ This list contains 1487 plugins. pytest plugin to instrument tests + :pypi:`pytest-insubprocess` + *last release*: Jul 01, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.4 + + A pytest plugin to execute test cases in a subprocess + :pypi:`pytest-integration` *last release*: Nov 17, 2022, *status*: N/A, @@ -6238,16 +7301,16 @@ This list contains 1487 plugins. Pytest plugin for intercepting outgoing connection requests during pytest run. :pypi:`pytest-interface-tester` - *last release*: Feb 09, 2024, + *last release*: Oct 09, 2025, *status*: 4 - Beta, *requires*: pytest Pytest plugin for checking charm relation interface protocol compliance. :pypi:`pytest-invenio` - *last release*: Jun 27, 2024, + *last release*: Jul 09, 2025, *status*: 5 - Production/Stable, - *requires*: pytest<7.2.0,>=6 + *requires*: pytest<9.0.0,>=6 Pytest fixtures for Invenio. @@ -6258,6 +7321,13 @@ This list contains 1487 plugins. Run tests covering a specific file or changeset + :pypi:`pytest-iovis` + *last release*: Nov 06, 2024, + *status*: 4 - Beta, + *requires*: pytest>=7.1.0 + + A Pytest plugin to enable Jupyter Notebook testing with Papermill + :pypi:`pytest-ipdb` *last release*: Mar 20, 2013, *status*: 2 - Pre-Alpha, @@ -6272,19 +7342,33 @@ This list contains 1487 plugins. THIS PROJECT IS ABANDONED + :pypi:`pytest-ipynb2` + *last release*: Mar 09, 2025, + *status*: N/A, + *requires*: pytest + + Pytest plugin to run tests in Jupyter Notebooks + :pypi:`pytest-ipywidgets` - *last release*: Jul 11, 2024, + *last release*: Oct 24, 2025, *status*: N/A, *requires*: pytest :pypi:`pytest-isolate` - *last release*: Feb 20, 2023, + *last release*: Sep 08, 2025, *status*: 4 - Beta, *requires*: pytest + Run pytest tests in isolated subprocesses + + :pypi:`pytest-isolate-mpi` + *last release*: Feb 24, 2025, + *status*: 4 - Beta, + *requires*: pytest>=5 + pytest-isolate-mpi allows for MPI-parallel tests being executed in a segfault and MPI_Abort safe manner :pypi:`pytest-isort` *last release*: Mar 05, 2024, @@ -6300,6 +7384,13 @@ This list contains 1487 plugins. Pytest plugin to display test reports as a plaintext spec, inspired by Rspec: https://github.com/mattduck/pytest-it. + :pypi:`pytest-item-dict` + *last release*: Nov 14, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.3.0 + + Get a hierarchical dict of session.items + :pypi:`pytest-iterassert` *last release*: May 11, 2020, *status*: 3 - Alpha, @@ -6307,6 +7398,13 @@ This list contains 1487 plugins. Nicer list and iterable assertion messages for pytest + :pypi:`pytest-iteration` + *last release*: Aug 22, 2024, + *status*: N/A, + *requires*: pytest + + Add iteration mark for tests + :pypi:`pytest-iters` *last release*: May 24, 2022, *status*: N/A, @@ -6350,7 +7448,7 @@ This list contains 1487 plugins. A plugin to generate customizable jinja-based HTML reports in pytest :pypi:`pytest-jira` - *last release*: Apr 30, 2024, + *last release*: Apr 15, 2025, *status*: 3 - Alpha, *requires*: N/A @@ -6364,7 +7462,7 @@ This list contains 1487 plugins. Plugin skips (xfail) tests if unresolved Jira issue(s) linked :pypi:`pytest-jira-xray` - *last release*: Mar 27, 2024, + *last release*: Oct 11, 2025, *status*: 4 - Beta, *requires*: pytest>=6.2.4 @@ -6399,7 +7497,7 @@ This list contains 1487 plugins. Generate JSON test reports :pypi:`pytest-json-ctrf` - *last release*: Jun 15, 2024, + *last release*: Oct 10, 2024, *status*: N/A, *requires*: pytest>6.0.0 @@ -6427,28 +7525,49 @@ This list contains 1487 plugins. A pytest plugin to report test results as JSON files :pypi:`pytest-json-report-wip` - *last release*: Oct 28, 2023, + *last release*: Jul 23, 2025, *status*: 4 - Beta, *requires*: pytest >=3.8.0 A pytest plugin to report test results as JSON files :pypi:`pytest-jsonschema` - *last release*: Mar 27, 2024, + *last release*: Apr 20, 2025, *status*: 4 - Beta, *requires*: pytest>=6.2.0 A pytest plugin to perform JSONSchema validations + :pypi:`pytest-jsonschema-snapshot` + *last release*: Sep 13, 2025, + *status*: N/A, + *requires*: pytest + + Pytest plugin for automatic JSON Schema generation and validation from examples + :pypi:`pytest-jtr` - *last release*: Jun 04, 2024, + *last release*: Jul 21, 2024, *status*: N/A, *requires*: pytest<8.0.0,>=7.1.2 pytest plugin supporting json test report output + :pypi:`pytest-jubilant` + *last release*: Jul 28, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Add your description here + + :pypi:`pytest-junit-xray-xml` + *last release*: Jan 01, 2025, + *status*: 4 - Beta, + *requires*: pytest + + Export test results in an augmented JUnit format for usage with Xray () + :pypi:`pytest-jupyter` - *last release*: Apr 04, 2024, + *last release*: Oct 16, 2025, *status*: 4 - Beta, *requires*: pytest>=7.0 @@ -6461,8 +7580,22 @@ This list contains 1487 plugins. A reusable JupyterHub pytest plugin + :pypi:`pytest-jux` + *last release*: Oct 24, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=7.4 + + A pytest plugin for signing and publishing JUnit XML test reports to the Jux REST API + + :pypi:`pytest-k8s` + *last release*: Jul 07, 2025, + *status*: N/A, + *requires*: pytest>=8.4.1 + + Kubernetes-based testing for pytest + :pypi:`pytest-kafka` - *last release*: Jun 14, 2023, + *last release*: Aug 14, 2024, *status*: N/A, *requires*: pytest @@ -6475,6 +7608,13 @@ This list contains 1487 plugins. A plugin to send pytest events to Kafka + :pypi:`pytest-kairos` + *last release*: Aug 08, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest>=5.0.0 + + Pytest plugin with random number generation, reproducibility, and test repetition + :pypi:`pytest-kasima` *last release*: Jan 26, 2023, *status*: 5 - Production/Stable, @@ -6497,9 +7637,9 @@ This list contains 1487 plugins. :pypi:`pytest-keyring` - *last release*: Oct 01, 2023, + *last release*: Dec 08, 2024, *status*: N/A, - *requires*: pytest (>=7.1) + *requires*: pytest>=8.0.2 A Pytest plugin to access the system's keyring to provide credentials for tests @@ -6532,7 +7672,7 @@ This list contains 1487 plugins. Run Konira DSL tests with py.test :pypi:`pytest-kookit` - *last release*: May 16, 2024, + *last release*: Sep 10, 2024, *status*: N/A, *requires*: N/A @@ -6553,11 +7693,18 @@ This list contains 1487 plugins. pytest krtech common library :pypi:`pytest-kubernetes` - *last release*: Sep 14, 2023, + *last release*: Oct 23, 2025, *status*: N/A, - *requires*: pytest (>=7.2.1,<8.0.0) + *requires*: pytest<9.0.0,>=8.3.0 + + + :pypi:`pytest_kustomize` + *last release*: Oct 02, 2025, + *status*: N/A, + *requires*: N/A + Parse and validate kustomize output :pypi:`pytest-kuunda` *last release*: Feb 25, 2024, @@ -6601,6 +7748,13 @@ This list contains 1487 plugins. Create fancy and clear HTML test reports. + :pypi:`pytest-latin-hypercube` + *last release*: Jun 26, 2025, + *status*: N/A, + *requires*: pytest + + Implementation of Latin Hypercube Sampling for pytest. + :pypi:`pytest-launchable` *last release*: Apr 05, 2023, *status*: N/A, @@ -6623,9 +7777,9 @@ This list contains 1487 plugins. It helps to use fixtures in pytest.mark.parametrize :pypi:`pytest-lazy-fixtures` - *last release*: Mar 16, 2024, + *last release*: Sep 16, 2025, *status*: N/A, - *requires*: pytest (>=7) + *requires*: pytest>=7 Allows you to use fixtures in @pytest.mark.parametrize. @@ -6657,6 +7811,13 @@ This list contains 1487 plugins. A simple plugin to use with pytest + :pypi:`pytest-leo-interface` + *last release*: Mar 19, 2025, + *status*: N/A, + *requires*: N/A + + Pytest extension tool for leo projects. + :pypi:`pytest-level` *last release*: Oct 21, 2019, *status*: N/A, @@ -6664,6 +7825,13 @@ This list contains 1487 plugins. Select tests of a given level or lower + :pypi:`pytest-lf-skip` + *last release*: Oct 14, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3.5 + + A pytest plugin which makes \`--last-failed\` skip instead of deselect tests. + :pypi:`pytest-libfaketime` *last release*: Apr 12, 2024, *status*: 4 - Beta, @@ -6672,11 +7840,11 @@ This list contains 1487 plugins. A python-libfaketime plugin for pytest :pypi:`pytest-libiio` - *last release*: Dec 22, 2023, - *status*: 4 - Beta, - *requires*: N/A + *last release*: Aug 15, 2025, + *status*: N/A, + *requires*: pytest>=3.5.0 - A pytest plugin to manage interfacing with libiio contexts + A pytest plugin for testing libiio based devices :pypi:`pytest-libnotify` *last release*: Apr 02, 2021, @@ -6721,7 +7889,7 @@ This list contains 1487 plugins. Pytest plugin for organizing tests. :pypi:`pytest-listener` - *last release*: May 28, 2019, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -6748,6 +7916,27 @@ This list contains 1487 plugins. Live results for pytest + :pypi:`pytest-llm` + *last release*: Oct 03, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=7.0.0 + + pytest-llm: A pytest plugin for testing LLM outputs with success rate thresholds. + + :pypi:`pytest-llmeval` + *last release*: Mar 19, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A pytest plugin to evaluate/benchmark LLM prompts + + :pypi:`pytest-lobster` + *last release*: Jul 26, 2025, + *status*: N/A, + *requires*: pytest>=7.0 + + Pytest to generate lobster tracing files + :pypi:`pytest-local-badge` *last release*: Jan 15, 2023, *status*: N/A, @@ -6763,7 +7952,7 @@ This list contains 1487 plugins. A PyTest plugin which provides an FTP fixture for your tests :pypi:`pytest-localserver` - *last release*: Oct 12, 2023, + *last release*: Oct 06, 2024, *status*: 4 - Beta, *requires*: N/A @@ -6784,16 +7973,16 @@ This list contains 1487 plugins. pytest-lock is a pytest plugin that allows you to "lock" the results of unit tests, storing them in a local cache. This is particularly useful for tests that are resource-intensive or don't need to be run every time. When the tests are run subsequently, pytest-lock will compare the current results with the locked results and issue a warning if there are any discrepancies. :pypi:`pytest-lockable` - *last release*: Jan 24, 2024, + *last release*: Sep 08, 2025, *status*: 5 - Production/Stable, *requires*: pytest lockable resource plugin for pytest :pypi:`pytest-locker` - *last release*: Oct 29, 2021, + *last release*: Dec 20, 2024, *status*: N/A, - *requires*: pytest (>=5.4) + *requires*: pytest>=5.4 Used to lock object during testing. Essentially changing assertions from being hard coded to asserting that nothing changed @@ -6832,6 +8021,13 @@ This list contains 1487 plugins. Plugin configuring handlers for loggers from Python logging module. + :pypi:`pytest-logger-db` + *last release*: Sep 14, 2025, + *status*: N/A, + *requires*: N/A + + Add your description here + :pypi:`pytest-logging` *last release*: Nov 04, 2015, *status*: 4 - Beta, @@ -6846,10 +8042,17 @@ This list contains 1487 plugins. + :pypi:`pytest-logging-strict` + *last release*: May 20, 2025, + *status*: 3 - Alpha, + *requires*: pytest + + pytest fixture logging configured from packaged YAML + :pypi:`pytest-logikal` - *last release*: Jun 27, 2024, + *last release*: Sep 11, 2025, *status*: 5 - Production/Stable, - *requires*: pytest==8.2.2 + *requires*: pytest==8.4.2 Common testing environment @@ -6860,6 +8063,13 @@ This list contains 1487 plugins. Package for creating a pytest test run reprot + :pypi:`pytest-logscanner` + *last release*: Sep 30, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.2.2 + + Pytest plugin for logscanner (A logger for python logging outputting to easily viewable (and filterable) html files. Good for people not grep savey, and color higlighting and quickly changing filters might even bye useful for commandline wizards.) + :pypi:`pytest-loguru` *last release*: Mar 20, 2024, *status*: 5 - Production/Stable, @@ -6868,19 +8078,33 @@ This list contains 1487 plugins. Pytest Loguru :pypi:`pytest-loop` - *last release*: Mar 30, 2024, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest pytest plugin for looping tests :pypi:`pytest-lsp` - *last release*: May 22, 2024, - *status*: 3 - Alpha, - *requires*: pytest + *last release*: Oct 25, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8.0 A pytest plugin for end-to-end testing of language servers + :pypi:`pytest-lw-realtime-result` + *last release*: Mar 13, 2025, + *status*: N/A, + *requires*: pytest>=3.5.0 + + Pytest plugin to generate realtime test results to a file + + :pypi:`pytest-manifest` + *last release*: Apr 07, 2025, + *status*: N/A, + *requires*: pytest + + PyTest plugin for recording and asserting against a manifest file + :pypi:`pytest-manual-marker` *last release*: Aug 04, 2022, *status*: 3 - Alpha, @@ -6888,6 +8112,13 @@ This list contains 1487 plugins. pytest marker for marking manual tests + :pypi:`pytest-mark-count` + *last release*: Nov 13, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.0.0 + + Get a count of the number of tests marked, unmarked, and unique tests if tests have multiple markers + :pypi:`pytest-markdoctest` *last release*: Jul 22, 2022, *status*: 4 - Beta, @@ -6903,26 +8134,33 @@ This list contains 1487 plugins. Test your markdown docs with pytest :pypi:`pytest-markdown-docs` - *last release*: Mar 05, 2024, + *last release*: Apr 09, 2025, *status*: N/A, - *requires*: pytest (>=7.0.0) + *requires*: pytest>=7.0.0 Run markdown code fences through pytest :pypi:`pytest-marker-bugzilla` - *last release*: Jan 09, 2020, - *status*: N/A, - *requires*: N/A + *last release*: Apr 02, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=2.2.4 py.test bugzilla integration plugin, using markers :pypi:`pytest-markers-presence` - *last release*: Feb 04, 2021, + *last release*: Oct 30, 2024, *status*: 4 - Beta, - *requires*: pytest (>=6.0) + *requires*: pytest>=6.0 A simple plugin to detect missed pytest tags and markers" + :pypi:`pytest-mark-filter` + *last release*: May 11, 2025, + *status*: N/A, + *requires*: pytest>=8.3.0 + + Filter pytest marks by name using match kw + :pypi:`pytest-markfiltration` *last release*: Nov 08, 2011, *status*: 3 - Alpha, @@ -6931,7 +8169,7 @@ This list contains 1487 plugins. UNKNOWN :pypi:`pytest-mark-manage` - *last release*: Jul 08, 2024, + *last release*: Aug 15, 2024, *status*: N/A, *requires*: pytest @@ -6951,13 +8189,27 @@ This list contains 1487 plugins. UNKNOWN + :pypi:`pytest-mask-secrets` + *last release*: Jan 28, 2025, + *status*: N/A, + *requires*: N/A + + Pytest plugin to hide sensitive data in test reports + :pypi:`pytest-matcher` - *last release*: Mar 15, 2024, + *last release*: Aug 07, 2025, *status*: 5 - Production/Stable, *requires*: pytest Easy way to match captured \`pytest\` output against expectations stored in files + :pypi:`pytest-matchers` + *last release*: Feb 11, 2025, + *status*: N/A, + *requires*: pytest<9.0,>=7.0 + + Matchers for pytest + :pypi:`pytest-match-skip` *last release*: May 15, 2019, *status*: 4 - Beta, @@ -6986,6 +8238,13 @@ This list contains 1487 plugins. Compute the maximum coverage available through pytest with the minimum execution time cost + :pypi:`pytest-max-warnings` + *last release*: Oct 23, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.3.3 + + A Pytest plugin to exit non-zero exit code when the configured maximum warnings has been exceeded. + :pypi:`pytest-maybe-context` *last release*: Apr 16, 2023, *status*: N/A, @@ -7007,6 +8266,13 @@ This list contains 1487 plugins. pytest plugin to run the mccabe code complexity checker. + :pypi:`pytest-mcp` + *last release*: Jul 07, 2025, + *status*: N/A, + *requires*: pytest>=8.4.0 + + Pytest-style framework for evaluating Model Context Protocol (MCP) servers. + :pypi:`pytest-md` *last release*: Jul 11, 2019, *status*: 3 - Alpha, @@ -7015,16 +8281,16 @@ This list contains 1487 plugins. Plugin for generating Markdown reports for pytest results :pypi:`pytest-md-report` - *last release*: May 18, 2024, + *last release*: May 02, 2025, *status*: 4 - Beta, *requires*: pytest!=6.0.0,<9,>=3.3.2 A pytest plugin to generate test outcomes reports with markdown table format. :pypi:`pytest-meilisearch` - *last release*: Feb 15, 2024, + *last release*: Oct 08, 2024, *status*: N/A, - *requires*: pytest (>=7.4.3) + *requires*: pytest>=7.4.3 Pytest helpers for testing projects using Meilisearch @@ -7043,7 +8309,7 @@ This list contains 1487 plugins. Estimates memory consumption of test functions :pypi:`pytest-memray` - *last release*: Apr 18, 2024, + *last release*: Aug 18, 2025, *status*: N/A, *requires*: pytest>=7.2 @@ -7063,6 +8329,13 @@ This list contains 1487 plugins. pytest plugin to write integration tests for projects using Mercurial Python internals + :pypi:`pytest-mergify` + *last release*: Oct 23, 2025, + *status*: N/A, + *requires*: pytest>=6.0.0 + + Pytest plugin for Mergify + :pypi:`pytest-mesh` *last release*: Aug 05, 2022, *status*: N/A, @@ -7091,6 +8364,13 @@ This list contains 1487 plugins. pytest plugin for test session metadata + :pypi:`pytest-metaexport` + *last release*: Jun 24, 2025, + *status*: N/A, + *requires*: pytest>=7.1.0 + + Pytest plugin for exporting custom test metadata to JSON. + :pypi:`pytest-metrics` *last release*: Apr 04, 2020, *status*: N/A, @@ -7098,8 +8378,22 @@ This list contains 1487 plugins. Custom metrics report for pytest + :pypi:`pytest-mfd-config` + *last release*: Jul 11, 2025, + *status*: N/A, + *requires*: pytest<9,>=7.2.1 + + Pytest Plugin that handles test and topology configs and all their belongings like helper fixtures. + + :pypi:`pytest-mfd-logging` + *last release*: Jul 09, 2025, + *status*: N/A, + *requires*: pytest<9,>=7.2.1 + + Module for handling PyTest logging. + :pypi:`pytest-mh` - *last release*: Jul 02, 2024, + *last release*: Oct 16, 2025, *status*: N/A, *requires*: pytest @@ -7112,6 +8406,13 @@ This list contains 1487 plugins. Mimesis integration with the pytest test runner + :pypi:`pytest-mimic` + *last release*: Apr 24, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + Easily record function calls while testing + :pypi:`pytest-minecraft` *last release*: Apr 06, 2022, *status*: N/A, @@ -7127,12 +8428,19 @@ This list contains 1487 plugins. A plugin to test mp :pypi:`pytest-minio-mock` - *last release*: May 26, 2024, + *last release*: Aug 06, 2025, *status*: N/A, *requires*: pytest>=5.0.0 A pytest plugin for mocking Minio S3 interactions + :pypi:`pytest-mirror` + *last release*: Jul 30, 2025, + *status*: 4 - Beta, + *requires*: N/A + + A pluggy-based pytest plugin and CLI tool for ensuring your test suite mirrors your source code structure + :pypi:`pytest-missing-fixtures` *last release*: Oct 14, 2020, *status*: 4 - Beta, @@ -7140,13 +8448,27 @@ This list contains 1487 plugins. Pytest plugin that creates missing fixtures + :pypi:`pytest-missing-modules` + *last release*: Sep 03, 2024, + *status*: N/A, + *requires*: pytest>=8.3.2 + + Pytest plugin to easily fake missing modules + :pypi:`pytest-mitmproxy` - *last release*: May 28, 2024, + *last release*: Nov 13, 2024, *status*: N/A, *requires*: pytest>=7.0 pytest plugin for mitmproxy tests + :pypi:`pytest-mitmproxy-plugin` + *last release*: Apr 10, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.2.0 + + Use MITM Proxy in autotests with full control from code + :pypi:`pytest-ml` *last release*: May 04, 2019, *status*: 4 - Beta, @@ -7162,7 +8484,7 @@ This list contains 1487 plugins. pytest plugin to display test execution output like a mochajs :pypi:`pytest-mock` - *last release*: Mar 21, 2024, + *last release*: Sep 16, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=6.2.5 @@ -7204,7 +8526,7 @@ This list contains 1487 plugins. An in-memory mock of a Redis server that runs in a separate thread. This is to be used for unit-tests that require a Redis database. :pypi:`pytest-mock-resources` - *last release*: Jun 20, 2024, + *last release*: Sep 17, 2025, *status*: N/A, *requires*: pytest>=1.0 @@ -7238,6 +8560,13 @@ This list contains 1487 plugins. Massively distributed pytest runs using modal.com + :pypi:`pytest-modern` + *last release*: Aug 19, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8 + + A more modern pytest + :pypi:`pytest-modified-env` *last release*: Jan 29, 2022, *status*: 4 - Beta, @@ -7267,9 +8596,9 @@ This list contains 1487 plugins. PyTest Molecule Plugin :: discover and run molecule tests :pypi:`pytest-mongo` - *last release*: Mar 13, 2024, + *last release*: Aug 01, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=6.2 + *requires*: pytest>=6.2 MongoDB process and client fixtures plugin for Pytest. @@ -7280,6 +8609,20 @@ This list contains 1487 plugins. pytest plugin for MongoDB fixtures + :pypi:`pytest-mongodb-nono` + *last release*: Jan 07, 2025, + *status*: N/A, + *requires*: N/A + + pytest plugin for MongoDB + + :pypi:`pytest-mongodb-ry` + *last release*: Sep 25, 2025, + *status*: N/A, + *requires*: N/A + + pytest plugin for MongoDB + :pypi:`pytest-monitor` *last release*: Jun 25, 2023, *status*: 5 - Production/Stable, @@ -7308,6 +8651,13 @@ This list contains 1487 plugins. Fixtures for integration tests of AWS services,uses moto mocking library. + :pypi:`pytest-moto-fixtures` + *last release*: Feb 04, 2025, + *status*: 1 - Planning, + *requires*: pytest<9,>=8.3; extra == "pytest" + + Fixtures for testing code that interacts with AWS + :pypi:`pytest-motor` *last release*: Jul 21, 2021, *status*: 3 - Alpha, @@ -7330,7 +8680,7 @@ This list contains 1487 plugins. pytest plugin to collect information from tests :pypi:`pytest-mpiexec` - *last release*: Apr 13, 2023, + *last release*: Jul 29, 2024, *status*: 3 - Alpha, *requires*: pytest @@ -7351,8 +8701,8 @@ This list contains 1487 plugins. low-startup-overhead, scalable, distributed-testing pytest plugin :pypi:`pytest-mqtt` - *last release*: May 08, 2024, - *status*: 4 - Beta, + *last release*: Sep 10, 2025, + *status*: 5 - Production/Stable, *requires*: pytest<9; extra == "test" pytest-mqtt supports testing systems based on MQTT @@ -7365,14 +8715,14 @@ This list contains 1487 plugins. Utility for writing multi-host tests for pytest :pypi:`pytest-multilog` - *last release*: Jan 17, 2023, + *last release*: Sep 21, 2025, *status*: N/A, *requires*: pytest Multi-process logs handling and other helpers for pytest :pypi:`pytest-multithreading` - *last release*: Dec 07, 2022, + *last release*: Aug 05, 2024, *status*: N/A, *requires*: N/A @@ -7399,12 +8749,19 @@ This list contains 1487 plugins. + :pypi:`pytest-my-plugin` + *last release*: Jan 27, 2025, + *status*: N/A, + *requires*: pytest>=6.0 + + A pytest plugin that does awesome things + :pypi:`pytest-mypy` - *last release*: Dec 18, 2022, - *status*: 4 - Beta, - *requires*: pytest (>=6.2) ; python_version >= "3.10" + *last release*: Apr 02, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7.0 - Mypy static type checker plugin for Pytest + A Pytest Plugin for Mypy :pypi:`pytest-mypyd` *last release*: Aug 20, 2019, @@ -7414,14 +8771,14 @@ This list contains 1487 plugins. Mypy static type checker plugin for Pytest :pypi:`pytest-mypy-plugins` - *last release*: Mar 31, 2024, + *last release*: Dec 21, 2024, *status*: 4 - Beta, *requires*: pytest>=7.0.0 pytest plugin for writing tests for mypy plugins :pypi:`pytest-mypy-plugins-shim` - *last release*: Apr 12, 2021, + *last release*: Feb 14, 2025, *status*: N/A, *requires*: pytest>=6.0.0 @@ -7442,12 +8799,19 @@ This list contains 1487 plugins. Pytest plugin to check mypy output. :pypi:`pytest-mysql` - *last release*: May 23, 2024, + *last release*: Dec 10, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=6.2 MySQL process and client fixtures for pytest + :pypi:`pytest-nb` + *last release*: Jul 26, 2025, + *status*: N/A, + *requires*: pytest==8.4.1 + + Seedable Jupyter Notebook testing tool + :pypi:`pytest-ndb` *last release*: Apr 28, 2024, *status*: N/A, @@ -7470,16 +8834,23 @@ This list contains 1487 plugins. pytest-neo is a plugin for pytest that shows tests like screen of Matrix. :pypi:`pytest-neos` - *last release*: Jun 11, 2024, - *status*: 1 - Planning, - *requires*: N/A + *last release*: Sep 10, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest<8.0,>=7.2; extra == "dev" Pytest plugin for neos + :pypi:`pytest-netconf` + *last release*: Jan 06, 2025, + *status*: N/A, + *requires*: N/A + + A pytest plugin that provides a mock NETCONF (RFC6241/RFC6242) server for local testing. + :pypi:`pytest-netdut` - *last release*: Jul 05, 2024, + *last release*: Oct 09, 2025, *status*: N/A, - *requires*: pytest<7.3,>=3.5.0 + *requires*: pytest>=3.5.0 "Automated software testing for switches using pytest" @@ -7505,9 +8876,9 @@ This list contains 1487 plugins. pytest plugin helps to avoid adding tests without mock \`time.sleep\` :pypi:`pytest-nginx` - *last release*: Aug 12, 2017, + *last release*: May 03, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest>=3.0.0 nginx fixture for pytest @@ -7533,7 +8904,7 @@ This list contains 1487 plugins. pytest ngs fixtures :pypi:`pytest-nhsd-apim` - *last release*: Jul 01, 2024, + *last release*: Oct 29, 2025, *status*: N/A, *requires*: pytest<9.0.0,>=8.2.0 @@ -7554,14 +8925,14 @@ This list contains 1487 plugins. A small snippet for nicer PyTest's Parametrize :pypi:`pytest_nlcov` - *last release*: Apr 11, 2024, + *last release*: Aug 05, 2024, *status*: N/A, *requires*: N/A Pytest plugin to get the coverage of the new lines (based on git diff) only :pypi:`pytest-nocustom` - *last release*: Apr 11, 2024, + *last release*: Aug 05, 2024, *status*: 5 - Production/Stable, *requires*: N/A @@ -7582,12 +8953,19 @@ This list contains 1487 plugins. Test-driven source code search for Python. :pypi:`pytest-nogarbage` - *last release*: Aug 29, 2021, + *last release*: Feb 24, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=4.6.0) + *requires*: pytest>=4.6.0 Ensure a test produces no garbage + :pypi:`pytest-no-problem` + *last release*: Oct 18, 2025, + *status*: N/A, + *requires*: pytest>=7.0 + + Pytest plugin to tell you when there's no problem + :pypi:`pytest-nose-attrib` *last release*: Aug 13, 2023, *status*: N/A, @@ -7652,12 +9030,19 @@ This list contains 1487 plugins. A pytest plugin for generating NUnit3 test result XML output :pypi:`pytest-oar` - *last release*: May 02, 2023, + *last release*: May 12, 2025, *status*: N/A, *requires*: pytest>=6.0.1 PyTest plugin for the OAR testing framework + :pypi:`pytest-oarepo` + *last release*: Oct 23, 2025, + *status*: N/A, + *requires*: pytest>=7.1.2; extra == "dev" + + + :pypi:`pytest-object-getter` *last release*: Jul 31, 2022, *status*: 5 - Production/Stable, @@ -7680,9 +9065,9 @@ This list contains 1487 plugins. A pytest plugin for simplifying ODC database tests :pypi:`pytest-odoo` - *last release*: Jul 06, 2023, - *status*: 4 - Beta, - *requires*: pytest (>=7.2.0) + *last release*: May 20, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8 py.test plugin to run Odoo tests @@ -7693,6 +9078,13 @@ This list contains 1487 plugins. Project description + :pypi:`pytest-oduit` + *last release*: Oct 06, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8 + + py.test plugin to run Odoo tests + :pypi:`pytest-oerp` *last release*: Feb 28, 2012, *status*: 3 - Alpha, @@ -7721,6 +9113,13 @@ This list contains 1487 plugins. The ultimate pytest output plugin + :pypi:`pytest-once` + *last release*: Oct 10, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=8.4.0 + + xdist-safe 'run once' fixture decorator for pytest (setup/teardown across workers) + :pypi:`pytest-only` *last release*: May 27, 2024, *status*: 5 - Production/Stable, @@ -7749,17 +9148,24 @@ This list contains 1487 plugins. Pytest plugin for detecting inadvertent open file handles + :pypi:`pytest-open-html` + *last release*: Mar 31, 2025, + *status*: N/A, + *requires*: pytest>=6.0 + + Auto-open HTML reports after pytest runs + :pypi:`pytest-opentelemetry` - *last release*: Oct 01, 2023, + *last release*: Apr 25, 2025, *status*: N/A, *requires*: pytest A pytest plugin for instrumenting test runs via OpenTelemetry :pypi:`pytest-opentmi` - *last release*: Jun 02, 2022, + *last release*: Mar 22, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=5.0) + *requires*: pytest>=5.0 pytest plugin for publish results to opentmi @@ -7768,7 +9174,7 @@ This list contains 1487 plugins. *status*: N/A, *requires*: pytest - Fixtures for Operators + Fixtures for Charmed Operators :pypi:`pytest-optional` *last release*: Oct 07, 2015, @@ -7778,9 +9184,9 @@ This list contains 1487 plugins. include/exclude values of fixtures in pytest :pypi:`pytest-optional-tests` - *last release*: Jul 09, 2019, + *last release*: Jul 21, 2025, *status*: 4 - Beta, - *requires*: pytest (>=4.5.0) + *requires*: pytest; extra == "dev" Easy declaration of optional tests (i.e., that are not run by default) @@ -7792,12 +9198,19 @@ This list contains 1487 plugins. A pytest plugin for orchestrating tests :pypi:`pytest-order` - *last release*: Apr 02, 2024, - *status*: 4 - Beta, + *last release*: Aug 22, 2024, + *status*: 5 - Production/Stable, *requires*: pytest>=5.0; python_version < "3.10" pytest plugin to run your tests in a specific order + :pypi:`pytest-ordered` + *last release*: Oct 07, 2024, + *status*: N/A, + *requires*: pytest>=6.2.0 + + Declare the order in which tests should run in your pytest.ini + :pypi:`pytest-ordering` *last release*: Nov 14, 2018, *status*: 4 - Beta, @@ -7827,12 +9240,19 @@ This list contains 1487 plugins. A pytest plugin for instrumenting test runs via OpenTelemetry :pypi:`pytest-otel` - *last release*: Mar 18, 2024, + *last release*: Apr 24, 2025, *status*: N/A, - *requires*: pytest==8.1.1 + *requires*: pytest==8.3.5 OpenTelemetry plugin for Pytest + :pypi:`pytest-otelmark` + *last release*: Sep 14, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=8.3.5 + + Pytest plugin for otelmark. + :pypi:`pytest-override-env-var` *last release*: Feb 25, 2023, *status*: N/A, @@ -7841,9 +9261,9 @@ This list contains 1487 plugins. Pytest mark to override a value of an environment variable. :pypi:`pytest-owner` - *last release*: Apr 25, 2022, + *last release*: Aug 19, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest Add owner mark for tests @@ -7854,6 +9274,13 @@ This list contains 1487 plugins. A simple plugin to use with pytest + :pypi:`pytest-pagerduty` + *last release*: Mar 22, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=7.4.0 + + Pytest plugin for PagerDuty integration via automation testing. + :pypi:`pytest-pahrametahrize` *last release*: Nov 24, 2021, *status*: 4 - Beta, @@ -7889,13 +9316,6 @@ This list contains 1487 plugins. pytest plugin to test all, first, last or random params - :pypi:`pytest-paramark` - *last release*: Jan 10, 2020, - *status*: 4 - Beta, - *requires*: pytest (>=4.5.0) - - Configure pytest fixtures using a combination of"parametrize" and markers - :pypi:`pytest-parametrization` *last release*: May 22, 2022, *status*: 5 - Production/Stable, @@ -7903,6 +9323,20 @@ This list contains 1487 plugins. Simpler PyTest parametrization + :pypi:`pytest-parametrization-annotation` + *last release*: Dec 10, 2024, + *status*: 5 - Production/Stable, + *requires*: pytest>=7 + + A pytest library for parametrizing tests using type hints. + + :pypi:`pytest-parametrize` + *last release*: Sep 25, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest<9.0.0,>=8.3.0 + + pytest decorator for parametrizing test cases in a dict-way + :pypi:`pytest-parametrize-cases` *last release*: Mar 13, 2022, *status*: N/A, @@ -7911,7 +9345,7 @@ This list contains 1487 plugins. A more user-friendly way to write parametrized tests. :pypi:`pytest-parametrized` - *last release*: Nov 03, 2023, + *last release*: Dec 21, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -7931,6 +9365,13 @@ This list contains 1487 plugins. Create pytest parametrize decorators from external files. + :pypi:`pytest-params` + *last release*: Apr 27, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7.0.0 + + Simplified pytest test case parameters. + :pypi:`pytest-param-scope` *last release*: Oct 18, 2023, *status*: N/A, @@ -7981,9 +9422,9 @@ This list contains 1487 plugins. A contextmanager pytest fixture for handling multiple mock patches :pypi:`pytest-patterns` - *last release*: Jun 14, 2024, + *last release*: Oct 22, 2024, *status*: 4 - Beta, - *requires*: N/A + *requires*: pytest>=6 pytest plugin to make testing complicated long string output easy to write and easy to debug @@ -8044,30 +9485,30 @@ This list contains 1487 plugins. A simple plugin to ensure the execution of critical sections of code has not been impacted :pypi:`pytest-performancetotal` - *last release*: Mar 19, 2024, - *status*: 4 - Beta, + *last release*: Aug 05, 2025, + *status*: 5 - Production/Stable, *requires*: N/A A performance plugin for pytest :pypi:`pytest-persistence` - *last release*: May 23, 2024, + *last release*: Aug 21, 2024, *status*: N/A, *requires*: N/A Pytest tool for persistent objects :pypi:`pytest-pexpect` - *last release*: Mar 27, 2024, + *last release*: Sep 10, 2025, *status*: 4 - Beta, *requires*: pytest>=6.2.0 Pytest pexpect plugin. :pypi:`pytest-pg` - *last release*: May 21, 2024, + *last release*: May 18, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=6.0.0 + *requires*: pytest>=7.4 A tiny plugin for pytest which runs PostgreSQL in Docker @@ -8085,13 +9526,27 @@ This list contains 1487 plugins. pytest plugin to test Python examples in Markdown using phmdoctest. + :pypi:`pytest-phoenix-interface` + *last release*: Mar 19, 2025, + *status*: N/A, + *requires*: N/A + + Pytest extension tool for phoenix projects. + :pypi:`pytest-picked` - *last release*: Jul 27, 2023, + *last release*: Nov 06, 2024, *status*: N/A, - *requires*: pytest (>=3.7.0) + *requires*: pytest>=3.7.0 Run the tests related to the changed files + :pypi:`pytest-pickle-cache` + *last release*: Feb 17, 2025, + *status*: N/A, + *requires*: pytest>=7 + + A pytest plugin for caching test results using pickle. + :pypi:`pytest-pigeonhole` *last release*: Jun 25, 2018, *status*: 5 - Production/Stable, @@ -8155,6 +9610,13 @@ This list contains 1487 plugins. runs tests in an order such that coverage increases as fast as possible + :pypi:`pytest-platform-adapter` + *last release*: Feb 18, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=6.2.5 + + Pytest集成自动化平台插件 + :pypi:`pytest-platform-markers` *last release*: Sep 09, 2019, *status*: 4 - Beta, @@ -8177,25 +9639,32 @@ This list contains 1487 plugins. Pytest plugin for reading playbooks. :pypi:`pytest-playwright` - *last release*: Jul 03, 2024, + *last release*: Sep 08, 2025, *status*: N/A, - *requires*: N/A + *requires*: pytest<9.0.0,>=6.2.4 A pytest wrapper with fixtures for Playwright to automate web browsers :pypi:`pytest_playwright_async` - *last release*: May 24, 2024, + *last release*: Sep 28, 2024, *status*: N/A, *requires*: N/A ASYNC Pytest plugin for Playwright :pypi:`pytest-playwright-asyncio` - *last release*: Aug 29, 2023, + *last release*: Sep 08, 2025, *status*: N/A, - *requires*: N/A + *requires*: pytest<9.0.0,>=6.2.4 + + A pytest wrapper with async fixtures for Playwright to automate web browsers + :pypi:`pytest-playwright-axe` + *last release*: Nov 01, 2025, + *status*: 5 - Production/Stable, + *requires*: N/A + An axe-core integration for accessibility testing using Playwright Python. :pypi:`pytest-playwright-enhanced` *last release*: Mar 24, 2024, @@ -8218,15 +9687,29 @@ This list contains 1487 plugins. A pytest wrapper for snapshot testing with playwright - :pypi:`pytest-playwright-visual` - *last release*: Apr 28, 2022, + :pypi:`pytest-playwright-visual` + *last release*: Apr 28, 2022, + *status*: N/A, + *requires*: N/A + + A pytest fixture for visual testing with Playwright + + :pypi:`pytest-playwright-visual-snapshot` + *last release*: Jul 02, 2025, *status*: N/A, *requires*: N/A - A pytest fixture for visual testing with Playwright + Easy pytest visual regression testing using playwright + + :pypi:`pytest-pl-grader` + *last release*: Nov 01, 2025, + *status*: 3 - Alpha, + *requires*: pytest + + A pytest plugin for autograding Python code. Designed for use with the PrairieLearn platform. :pypi:`pytest-plone` - *last release*: May 15, 2024, + *last release*: Jun 11, 2025, *status*: 3 - Alpha, *requires*: pytest<8.0.0 @@ -8246,8 +9729,15 @@ This list contains 1487 plugins. A plugin to help developing and testing other plugins + :pypi:`pytest-plugins` + *last release*: Oct 23, 2025, + *status*: N/A, + *requires*: pytest + + A Python package for managing pytest plugins. + :pypi:`pytest-plus` - *last release*: Mar 26, 2024, + *last release*: Feb 02, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=7.4.2 @@ -8261,8 +9751,8 @@ This list contains 1487 plugins. :pypi:`pytest-pogo` - *last release*: May 22, 2024, - *status*: 1 - Planning, + *last release*: May 05, 2025, + *status*: 4 - Beta, *requires*: pytest<9,>=7 Pytest plugin for pogo-migrate @@ -8302,6 +9792,13 @@ This list contains 1487 plugins. Provides Polecat pytest fixtures + :pypi:`pytest-polymeric-report` + *last release*: Oct 20, 2025, + *status*: N/A, + *requires*: N/A + + A polymeric test report plugin for pytest + :pypi:`pytest-ponyorm` *last release*: Oct 31, 2018, *status*: N/A, @@ -8337,12 +9834,12 @@ This list contains 1487 plugins. A pytest plugin to help with testing pop projects - :pypi:`pytest-porringer` - *last release*: Jan 18, 2024, - *status*: N/A, - *requires*: pytest>=7.4.4 - + :pypi:`pytest-porcochu` + *last release*: Nov 28, 2024, + *status*: 5 - Production/Stable, + *requires*: N/A + Show surprise when tests are passing :pypi:`pytest-portion` *last release*: Jan 28, 2021, @@ -8359,9 +9856,9 @@ This list contains 1487 plugins. Run PostgreSQL in Docker container in Pytest. :pypi:`pytest-postgresql` - *last release*: Mar 11, 2024, + *last release*: May 17, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=6.2 + *requires*: pytest>=7.2 Postgresql fixtures and fixture factories for Pytest. @@ -8373,11 +9870,11 @@ This list contains 1487 plugins. pytest plugin with powerful fixtures :pypi:`pytest-powerpack` - *last release*: Mar 17, 2024, + *last release*: Jan 04, 2025, *status*: N/A, - *requires*: pytest (>=8.1.1,<9.0.0) - + *requires*: pytest<9.0.0,>=8.1.1 + A plugin containing extra batteries for pytest :pypi:`pytest-prefer-nested-dup-tests` *last release*: Apr 27, 2022, @@ -8387,7 +9884,7 @@ This list contains 1487 plugins. A Pytest plugin to drop duplicated tests during collection, but will prefer keeping nested packages. :pypi:`pytest-pretty` - *last release*: Apr 05, 2023, + *last release*: Jun 04, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=7 @@ -8408,21 +9905,21 @@ This list contains 1487 plugins. Minitest-style test colors :pypi:`pytest-print` - *last release*: Aug 25, 2023, + *last release*: Oct 09, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=7.4 + *requires*: pytest>=8.4.2 pytest-print adds the printer fixture you can use to print messages to the user (directly to the pytest runner, not stdout) :pypi:`pytest-priority` - *last release*: Jul 23, 2023, + *last release*: Aug 19, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest pytest plugin for add priority for tests :pypi:`pytest-proceed` - *last release*: Apr 10, 2024, + *last release*: Oct 01, 2024, *status*: N/A, *requires*: pytest @@ -8436,7 +9933,7 @@ This list contains 1487 plugins. pytest plugin for configuration profiles :pypi:`pytest-profiling` - *last release*: May 28, 2019, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -8463,6 +9960,20 @@ This list contains 1487 plugins. Pytest report plugin for Zulip + :pypi:`pytest-prometheus-pushgw` + *last release*: May 19, 2025, + *status*: N/A, + *requires*: pytest>=6.0.0 + + Pytest plugin to export test metrics to Prometheus Pushgateway + + :pypi:`pytest-proofy` + *last release*: Oct 17, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + Pytest plugin for Proofy test reporting + :pypi:`pytest-prosper` *last release*: Sep 24, 2018, *status*: N/A, @@ -8471,9 +9982,9 @@ This list contains 1487 plugins. Test helpers for Prosper projects :pypi:`pytest-prysk` - *last release*: Mar 12, 2024, + *last release*: Dec 10, 2024, *status*: 4 - Beta, - *requires*: pytest (>=7.3.2) + *requires*: pytest>=7.3.2 Pytest plugin for prysk @@ -8492,8 +10003,8 @@ This list contains 1487 plugins. pytest plugin for testing applications that use psqlgraph :pypi:`pytest-pt` - *last release*: May 15, 2024, - *status*: 4 - Beta, + *last release*: Sep 22, 2024, + *status*: 5 - Production/Stable, *requires*: pytest pytest plugin to use \*.pt files as tests @@ -8555,12 +10066,19 @@ This list contains 1487 plugins. Plugin for py.test to enter PyCharm debugger on uncaught exceptions :pypi:`pytest-pycodestyle` - *last release*: Oct 28, 2022, + *last release*: Jul 20, 2025, *status*: 3 - Alpha, - *requires*: N/A + *requires*: pytest>=7.0 pytest plugin to run pycodestyle + :pypi:`pytest-pydantic-schema-sync` + *last release*: Aug 29, 2024, + *status*: N/A, + *requires*: pytest>=6 + + Pytest plugin to synchronise Pydantic model schemas with JSONSchema files + :pypi:`pytest-pydev` *last release*: Nov 15, 2017, *status*: 3 - Alpha, @@ -8569,12 +10087,19 @@ This list contains 1487 plugins. py.test plugin to connect to a remote debug server with PyDev or PyCharm. :pypi:`pytest-pydocstyle` - *last release*: Jan 05, 2023, + *last release*: Oct 09, 2024, *status*: 3 - Alpha, - *requires*: N/A + *requires*: pytest>=7.0 pytest plugin to run pydocstyle + :pypi:`pytest-pylembic` + *last release*: Jul 22, 2025, + *status*: 3 - Alpha, + *requires*: N/A + + This package provides pytest plugin for validating Alembic migrations using the pylembic package. + :pypi:`pytest-pylint` *last release*: Oct 06, 2023, *status*: 5 - Production/Stable, @@ -8582,6 +10107,13 @@ This list contains 1487 plugins. pytest plugin to check source code with pylint + :pypi:`pytest-pylyzer` + *last release*: Feb 15, 2025, + *status*: 4 - Beta, + *requires*: N/A + + A pytest plugin for pylyzer + :pypi:`pytest-pymysql-autorecord` *last release*: Sep 02, 2022, *status*: N/A, @@ -8590,7 +10122,7 @@ This list contains 1487 plugins. Record PyMySQL queries and mock with the stored data. :pypi:`pytest-pyodide` - *last release*: Jun 12, 2024, + *last release*: Oct 24, 2025, *status*: N/A, *requires*: pytest @@ -8625,14 +10157,14 @@ This list contains 1487 plugins. Pytest fixture "q" for pyq :pypi:`pytest-pyramid` - *last release*: Oct 11, 2023, + *last release*: Sep 30, 2025, *status*: 5 - Production/Stable, *requires*: pytest pytest_pyramid - provides fixtures for testing pyramid applications with pytest test suite :pypi:`pytest-pyramid-server` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -8652,20 +10184,34 @@ This list contains 1487 plugins. Pytest plugin for type checking code with Pyright + :pypi:`pytest-pyspark-plugin` + *last release*: Jul 28, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.0.0 + + Pytest pyspark plugin (p3) + :pypi:`pytest-pyspec` - *last release*: Jan 02, 2024, + *last release*: Aug 17, 2024, *status*: N/A, - *requires*: pytest (>=7.2.1,<8.0.0) + *requires*: pytest<9.0.0,>=8.3.2 A plugin that transforms the pytest output into a result similar to the RSpec. It enables the use of docstrings to display results and also enables the use of the prefixes "describe", "with" and "it". :pypi:`pytest-pystack` - *last release*: Jan 04, 2024, + *last release*: Nov 16, 2024, *status*: N/A, - *requires*: pytest >=3.5.0 + *requires*: pytest>=3.5.0 Plugin to run pystack after a timeout for a test suite. + :pypi:`pytest-pytestdb` + *last release*: Sep 14, 2025, + *status*: N/A, + *requires*: N/A + + Add your description here + :pypi:`pytest-pytestrail` *last release*: Aug 27, 2020, *status*: 4 - Beta, @@ -8673,8 +10219,15 @@ This list contains 1487 plugins. Pytest plugin for interaction with TestRail + :pypi:`pytest-pytestrail-internal` + *last release*: Jun 12, 2025, + *status*: 4 - Beta, + *requires*: pytest>=3.8.0 + + Pytest plugin for interaction with TestRail, Pytest plugin for TestRail (internal fork from: https://github.com/tolstislon/pytest-pytestrail with PR #25 fix) + :pypi:`pytest-pythonhashseed` - *last release*: Feb 25, 2024, + *last release*: Sep 28, 2025, *status*: 4 - Beta, *requires*: pytest>=3.0.0 @@ -8709,22 +10262,22 @@ This list contains 1487 plugins. A package for create venv in tests :pypi:`pytest-pyvista` - *last release*: Sep 29, 2023, + *last release*: Oct 06, 2025, *status*: 4 - Beta, - *requires*: pytest>=3.5.0 + *requires*: pytest>=6.2.0 - Pytest-pyvista package + Pytest-pyvista package. :pypi:`pytest-qanova` - *last release*: May 26, 2024, + *last release*: Sep 05, 2024, *status*: 3 - Alpha, *requires*: pytest A pytest plugin to collect test information :pypi:`pytest-qaseio` - *last release*: May 30, 2024, - *status*: 4 - Beta, + *last release*: Oct 01, 2025, + *status*: 5 - Production/Stable, *requires*: pytest<9.0.0,>=7.2.2 Pytest plugin for Qase.io integration @@ -8765,16 +10318,16 @@ This list contains 1487 plugins. pytest plugin to generate test result QR codes :pypi:`pytest-qt` - *last release*: Feb 07, 2024, + *last release*: Jul 01, 2025, *status*: 5 - Production/Stable, *requires*: pytest pytest support for PyQt and PySide applications :pypi:`pytest-qt-app` - *last release*: Dec 23, 2015, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest QT app fixture for py.test @@ -8800,7 +10353,7 @@ This list contains 1487 plugins. Run test suites with pytest-quickify. :pypi:`pytest-rabbitmq` - *last release*: May 08, 2024, + *last release*: Oct 15, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=6.2 @@ -8863,7 +10416,7 @@ This list contains 1487 plugins. py.test plugin to randomize tests :pypi:`pytest-randomly` - *last release*: Aug 15, 2023, + *last release*: Sep 12, 2025, *status*: 5 - Production/Stable, *requires*: pytest @@ -8884,46 +10437,60 @@ This list contains 1487 plugins. Randomise the order in which pytest tests are run with some control over the randomness :pypi:`pytest-random-order` - *last release*: Jan 20, 2024, + *last release*: Jun 22, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=3.0.0 + *requires*: pytest Randomise the order in which pytest tests are run with some control over the randomness :pypi:`pytest-ranking` - *last release*: Jun 07, 2024, + *last release*: Apr 08, 2025, *status*: 4 - Beta, *requires*: pytest>=7.4.3 - A Pytest plugin for automatically prioritizing/ranking tests to speed up failure detection + A Pytest plugin for faster fault detection via regression test prioritization + + :pypi:`pytest-rca-report` + *last release*: Aug 04, 2025, + *status*: N/A, + *requires*: N/A + + Interactive RCA report generator for pytest runs, with AI-based analysis and visual dashboard :pypi:`pytest-readme` - *last release*: Sep 02, 2022, + *last release*: Aug 01, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest Test your README.md file :pypi:`pytest-reana` - *last release*: Mar 14, 2024, + *last release*: Oct 10, 2025, *status*: 3 - Alpha, *requires*: N/A Pytest fixtures for REANA. + :pypi:`pytest-recap` + *last release*: Jun 16, 2025, + *status*: N/A, + *requires*: pytest>=6.2.0 + + Capture your test sessions. Recap the results. + :pypi:`pytest-recorder` - *last release*: Jun 27, 2024, + *last release*: Oct 28, 2025, *status*: N/A, - *requires*: N/A + *requires*: pytest>=8.4.1 Pytest plugin, meant to facilitate unit tests writing for tools consumming Web APIs. :pypi:`pytest-recording` - *last release*: Jul 09, 2024, + *last release*: May 08, 2025, *status*: 4 - Beta, *requires*: pytest>=3.5.0 - A pytest plugin that allows you recording of network interactions via VCR.py + A pytest plugin powered by VCR.py to record and replay HTTP traffic :pypi:`pytest-recordings` *last release*: Aug 13, 2020, @@ -8932,8 +10499,15 @@ This list contains 1487 plugins. Provides pytest plugins for reporting request/response traffic, screenshots, and more to ReportPortal + :pypi:`pytest-record-video` + *last release*: Oct 31, 2024, + *status*: N/A, + *requires*: N/A + + 用例执行过程中录制视频 + :pypi:`pytest-redis` - *last release*: Jun 19, 2024, + *last release*: Nov 27, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=6.2 @@ -8982,14 +10556,14 @@ This list contains 1487 plugins. Management of Pytest dependencies via regex patterns :pypi:`pytest-regressions` - *last release*: Aug 31, 2023, + *last release*: Sep 05, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=6.2.0 + *requires*: pytest>=6.2.0 Easy to use fixtures to write regression tests. :pypi:`pytest-regtest` - *last release*: Feb 26, 2024, + *last release*: Oct 11, 2025, *status*: N/A, *requires*: pytest>7.2 @@ -9002,6 +10576,13 @@ This list contains 1487 plugins. a pytest plugin that sorts tests using "before" and "after" markers + :pypi:`pytest-relative-path` + *last release*: Aug 30, 2024, + *status*: N/A, + *requires*: pytest + + Handle relative path in pytest options or ini configs + :pypi:`pytest-relaxed` *last release*: Mar 29, 2024, *status*: 5 - Production/Stable, @@ -9045,7 +10626,7 @@ This list contains 1487 plugins. Reorder tests depending on their paths and names. :pypi:`pytest-repeat` - *last release*: Oct 09, 2023, + *last release*: Apr 07, 2025, *status*: 5 - Production/Stable, *requires*: pytest @@ -9059,14 +10640,14 @@ This list contains 1487 plugins. py.test plugin for repeating single test multiple times. :pypi:`pytest-replay` - *last release*: Jan 11, 2024, + *last release*: Feb 05, 2025, *status*: 5 - Production/Stable, *requires*: pytest Saves previous test runs and allow re-execute previous pytest runs to reproduce crashes or flaky tests :pypi:`pytest-repo-health` - *last release*: Apr 17, 2023, + *last release*: May 05, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -9087,19 +10668,33 @@ This list contains 1487 plugins. Generate Pytest reports with templates :pypi:`pytest-reporter-html1` - *last release*: Jun 28, 2024, + *last release*: Oct 10, 2025, *status*: 4 - Beta, *requires*: N/A A basic HTML report template for Pytest :pypi:`pytest-reporter-html-dots` - *last release*: Jan 22, 2023, + *last release*: Apr 26, 2025, *status*: N/A, *requires*: N/A A basic HTML report for pytest using Jinja2 template engine. + :pypi:`pytest-reporter-plus` + *last release*: Jul 16, 2025, + *status*: N/A, + *requires*: N/A + + Lightweight enhanced HTML reporter for Pytest + + :pypi:`pytest-report-extras` + *last release*: Aug 08, 2025, + *status*: N/A, + *requires*: pytest>=8.4.0 + + Pytest plugin to enhance pytest-html and allure reports by adding comments, screenshots, webpage sources and attachments. + :pypi:`pytest-reportinfra` *last release*: Aug 11, 2019, *status*: 3 - Alpha, @@ -9136,9 +10731,9 @@ This list contains 1487 plugins. pytest plugin for adding tests' parameters to junit report :pypi:`pytest-reportportal` - *last release*: Mar 27, 2024, + *last release*: Jul 08, 2025, *status*: N/A, - *requires*: pytest>=3.8.0 + *requires*: pytest>=4.6.10 Agent for Reporting results of tests to the Report Portal @@ -9156,6 +10751,20 @@ This list contains 1487 plugins. Pytest Repo Structure + :pypi:`pytest-req` + *last release*: Sep 08, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8.4.2 + + pytest requests plugin + + :pypi:`pytest-reqcov` + *last release*: Jul 04, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=6.0 + + A pytest plugin for requirement coverage tracking + :pypi:`pytest-reqs` *last release*: May 12, 2019, *status*: N/A, @@ -9184,6 +10793,13 @@ This list contains 1487 plugins. Pytest Plugin to Mock Requests Futures + :pypi:`pytest-requirements` + *last release*: Feb 28, 2025, + *status*: N/A, + *requires*: pytest + + pytest plugin for using custom markers to relate tests to requirements and usecases + :pypi:`pytest-requires` *last release*: Dec 21, 2021, *status*: 4 - Beta, @@ -9191,6 +10807,13 @@ This list contains 1487 plugins. A pytest plugin to elegantly skip tests with optional requirements + :pypi:`pytest-reqyaml` + *last release*: Aug 16, 2025, + *status*: N/A, + *requires*: pytest>=8.4.1 + + This is a plugin where generate requests test cases from yaml. + :pypi:`pytest-reraise` *last release*: Sep 20, 2022, *status*: 5 - Production/Stable, @@ -9206,9 +10829,9 @@ This list contains 1487 plugins. Re-run only changed files in specified branch :pypi:`pytest-rerun-all` - *last release*: Nov 16, 2023, + *last release*: Jul 30, 2025, *status*: 3 - Alpha, - *requires*: pytest (>=7.0.0) + *requires*: pytest>=7.0.0 Rerun testsuite for a certain time or iterations @@ -9220,9 +10843,9 @@ This list contains 1487 plugins. pytest rerun class failures plugin :pypi:`pytest-rerunfailures` - *last release*: Mar 13, 2024, + *last release*: Oct 10, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=7.2 + *requires*: pytest!=8.2.2,>=7.4 pytest plugin to re-run tests to eliminate flaky failures @@ -9234,16 +10857,16 @@ This list contains 1487 plugins. pytest plugin to re-run tests to eliminate flaky failures :pypi:`pytest-reserial` - *last release*: May 23, 2024, + *last release*: Dec 22, 2024, *status*: 4 - Beta, *requires*: pytest Pytest fixture for recording and replaying serial port traffic. :pypi:`pytest-resilient-circuits` - *last release*: May 17, 2024, + *last release*: Jul 29, 2025, *status*: N/A, - *requires*: pytest~=4.6; python_version == "2.7" + *requires*: pytest~=7.0 Resilient Circuits fixtures for PyTest @@ -9255,9 +10878,9 @@ This list contains 1487 plugins. Load resource fixture plugin to use with pytest :pypi:`pytest-resource-path` - *last release*: May 01, 2021, + *last release*: Sep 18, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=3.5.0) + *requires*: pytest>=3.5.0 Provides path for uniform access to test resources in isolated directory @@ -9268,6 +10891,13 @@ This list contains 1487 plugins. Pytest plugin for reporting running time and peak memory usage + :pypi:`pytest-respect` + *last release*: Oct 21, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8.0.0 + + Pytest plugin to load resource files relative to test code and to expect values to match them. + :pypi:`pytest-responsemock` *last release*: Mar 10, 2022, *status*: 5 - Production/Stable, @@ -9290,7 +10920,7 @@ This list contains 1487 plugins. :pypi:`pytest-restrict` - *last release*: Jul 10, 2023, + *last release*: Sep 09, 2025, *status*: 5 - Production/Stable, *requires*: pytest @@ -9303,6 +10933,20 @@ This list contains 1487 plugins. A pytest plugin that records the start, end, and result information of each use case in a log file + :pypi:`pytest-result-notify` + *last release*: Apr 27, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Default template for PDM package + + :pypi:`pytest-results` + *last release*: Oct 08, 2025, + *status*: 4 - Beta, + *requires*: pytest + + Easily spot regressions in your tests. + :pypi:`pytest-result-sender` *last release*: Apr 20, 2023, *status*: N/A, @@ -9310,6 +10954,34 @@ This list contains 1487 plugins. + :pypi:`pytest-result-sender-jms` + *last release*: May 22, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Default template for PDM package + + :pypi:`pytest-result-sender-lj` + *last release*: Dec 17, 2024, + *status*: N/A, + *requires*: pytest>=8.3.4 + + Default template for PDM package + + :pypi:`pytest-result-sender-lyt` + *last release*: Mar 14, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Default template for PDM package + + :pypi:`pytest-result-sender-misszhang` + *last release*: Mar 21, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Default template for PDM package + :pypi:`pytest-resume` *last release*: Apr 22, 2023, *status*: 4 - Beta, @@ -9325,16 +10997,16 @@ This list contains 1487 plugins. A RethinkDB plugin for pytest. :pypi:`pytest-retry` - *last release*: May 14, 2024, + *last release*: Jan 19, 2025, *status*: N/A, *requires*: pytest>=7.0.0 Adds the ability to retry flaky tests in CI environments :pypi:`pytest-retry-class` - *last release*: Mar 25, 2023, + *last release*: Nov 24, 2024, *status*: N/A, - *requires*: pytest (>=5.3) + *requires*: pytest>=5.3 A pytest plugin to rerun entire class on failure @@ -9345,17 +11017,24 @@ This list contains 1487 plugins. + :pypi:`pytest-revealtype-injector` + *last release*: Oct 23, 2025, + *status*: 4 - Beta, + *requires*: pytest<9,>=7.0 + + Pytest plugin for replacing reveal_type() calls inside test functions with static and runtime type checking result comparison, for confirming type annotation validity. + :pypi:`pytest-reverse` - *last release*: Jul 10, 2023, + *last release*: Sep 09, 2025, *status*: 5 - Production/Stable, *requires*: pytest Pytest plugin to reverse test order. :pypi:`pytest-rich` - *last release*: Mar 03, 2022, + *last release*: Dec 12, 2024, *status*: 4 - Beta, - *requires*: pytest (>=7.0) + *requires*: pytest>=7.0 Leverage rich for richer test session output @@ -9394,6 +11073,13 @@ This list contains 1487 plugins. Sycronise pytest results to Jira RMsis + :pypi:`pytest-rmysql` + *last release*: Aug 17, 2025, + *status*: N/A, + *requires*: pytest>=8.4.1 + + This is a plugin which is able to connet MySQL easyly. + :pypi:`pytest-rng` *last release*: Aug 08, 2019, *status*: 5 - Production/Stable, @@ -9408,8 +11094,8 @@ This list contains 1487 plugins. pytest plugin for ROAST configuration override and fixtures - :pypi:`pytest_robotframework` - *last release*: Jul 01, 2024, + :pypi:`pytest-robotframework` + *last release*: Oct 06, 2025, *status*: N/A, *requires*: pytest<9,>=7 @@ -9458,7 +11144,7 @@ This list contains 1487 plugins. Coverage-based regression test selection (RTS) plugin for pytest :pypi:`pytest-ruff` - *last release*: Jul 09, 2024, + *last release*: Jun 19, 2025, *status*: 4 - Beta, *requires*: pytest>=5 @@ -9478,6 +11164,13 @@ This list contains 1487 plugins. implement a --failed option for pytest + :pypi:`pytest-run-parallel` + *last release*: Oct 23, 2025, + *status*: 4 - Beta, + *requires*: pytest>=6.2.0 + + A simple pytest plugin to run tests concurrently + :pypi:`pytest-run-subprocess` *last release*: Nov 12, 2022, *status*: 5 - Production/Stable, @@ -9493,8 +11186,8 @@ This list contains 1487 plugins. Checks type annotations on runtime while running tests. :pypi:`pytest-runtime-xfail` - *last release*: Aug 26, 2021, - *status*: N/A, + *last release*: Oct 10, 2025, + *status*: 5 - Production/Stable, *requires*: pytest>=5.0.0 Call runtime_xfail() to mark running test as xfail. @@ -9528,9 +11221,9 @@ This list contains 1487 plugins. A Pytest plugin that builds and creates docker containers :pypi:`pytest-salt-factories` - *last release*: Mar 22, 2024, + *last release*: Jul 08, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=7.0.0 + *requires*: pytest>=7.4.0 Pytest Salt Plugin @@ -9562,6 +11255,13 @@ This list contains 1487 plugins. a pytest plugin for Sanic + :pypi:`pytest-sanitizer` + *last release*: Mar 16, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=6.0.0 + + A pytest plugin to sanitize output for LLMs (personal tool, no warranty or liability) + :pypi:`pytest-sanity` *last release*: Dec 07, 2020, *status*: N/A, @@ -9584,7 +11284,7 @@ This list contains 1487 plugins. pytest_sauce provides sane and helpful methods worked out in clearcode to run py.test tests with selenium/saucelabs :pypi:`pytest-sbase` - *last release*: Jul 08, 2024, + *last release*: Nov 01, 2025, *status*: 5 - Production/Stable, *requires*: N/A @@ -9598,18 +11298,25 @@ This list contains 1487 plugins. pytest plugin for test scenarios :pypi:`pytest-scenario-files` - *last release*: May 19, 2024, + *last release*: Sep 03, 2025, *status*: 5 - Production/Stable, - *requires*: pytest>=7.2.0 + *requires*: pytest<9,>=7.4 A pytest plugin that generates unit test scenarios from data files. + :pypi:`pytest-scenarios` + *last release*: Oct 29, 2025, + *status*: N/A, + *requires*: N/A + + Add your description here + :pypi:`pytest-schedule` - *last release*: Jan 07, 2023, - *status*: 5 - Production/Stable, + *last release*: Oct 31, 2024, + *status*: N/A, *requires*: N/A - The job of test scheduling for humans. + Automate and customize test scheduling effortlessly on local machines. :pypi:`pytest-schema` *last release*: Feb 16, 2024, @@ -9618,6 +11325,13 @@ This list contains 1487 plugins. 👍 Validate return values against a schema-like object in testing + :pypi:`pytest-scim2-server` + *last release*: May 14, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3.4 + + SCIM2 server fixture for Pytest + :pypi:`pytest-screenshot-on-failure` *last release*: Jul 21, 2023, *status*: 4 - Beta, @@ -9625,6 +11339,13 @@ This list contains 1487 plugins. Saves a screenshot when a test case from a pytest execution fails + :pypi:`pytest-scrutinize` + *last release*: Aug 19, 2024, + *status*: 4 - Beta, + *requires*: pytest>=6 + + Scrutinize your pytest test suites for slow fixtures, tests and more. + :pypi:`pytest-securestore` *last release*: Nov 08, 2021, *status*: 4 - Beta, @@ -9654,7 +11375,7 @@ This list contains 1487 plugins. pytest plugin to automatically capture screenshots upon selenium webdriver events :pypi:`pytest-seleniumbase` - *last release*: Jul 08, 2024, + *last release*: Nov 01, 2025, *status*: 5 - Production/Stable, *requires*: N/A @@ -9675,21 +11396,21 @@ This list contains 1487 plugins. A pytest package implementing perceptualdiff for Selenium tests. :pypi:`pytest-selfie` - *last release*: Apr 05, 2024, + *last release*: Dec 16, 2024, *status*: N/A, - *requires*: pytest<9.0.0,>=8.0.0 + *requires*: pytest>=8.0.0 A pytest plugin for selfie snapshot testing. :pypi:`pytest-send-email` - *last release*: Dec 04, 2019, + *last release*: Sep 02, 2024, *status*: N/A, - *requires*: N/A + *requires*: pytest Send pytest execution result email :pypi:`pytest-sentry` - *last release*: Apr 25, 2024, + *last release*: Jul 01, 2025, *status*: N/A, *requires*: pytest @@ -9703,18 +11424,18 @@ This list contains 1487 plugins. Pytest plugin for sequencing markers for execution of tests :pypi:`pytest-server` - *last release*: Jun 24, 2024, + *last release*: Sep 09, 2024, *status*: N/A, *requires*: N/A test server exec cmd :pypi:`pytest-server-fixtures` - *last release*: Dec 19, 2023, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest - Extensible server fixures for py.test + Extensible server fixtures for py.test :pypi:`pytest-serverless` *last release*: May 09, 2022, @@ -9724,23 +11445,23 @@ This list contains 1487 plugins. Automatically mocks resources from serverless.yml in pytest using moto. :pypi:`pytest-servers` - *last release*: Jun 17, 2024, + *last release*: Aug 04, 2025, *status*: 3 - Alpha, *requires*: pytest>=6.2 pytest servers :pypi:`pytest-service` - *last release*: May 11, 2024, + *last release*: Aug 06, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=6.0.0 :pypi:`pytest-services` - *last release*: Oct 30, 2020, + *last release*: Jul 16, 2025, *status*: 6 - Mature, - *requires*: N/A + *requires*: pytest Services plugin for pytest testing framework @@ -9786,6 +11507,20 @@ This list contains 1487 plugins. + :pypi:`pytest-shard-fork` + *last release*: Jun 13, 2025, + *status*: 4 - Beta, + *requires*: pytest + + Shard tests to support parallelism across multiple machines + + :pypi:`pytest-shared-session-scope` + *last release*: Oct 31, 2025, + *status*: N/A, + *requires*: pytest>=7.0.0 + + Pytest session-scoped fixture that works with xdist + :pypi:`pytest-share-hdf` *last release*: Sep 21, 2022, *status*: 4 - Beta, @@ -9808,9 +11543,9 @@ This list contains 1487 plugins. A pytest plugin to help with testing shell scripts / black box commands :pypi:`pytest-shell-utilities` - *last release*: Feb 23, 2024, + *last release*: Oct 22, 2024, *status*: 5 - Production/Stable, - *requires*: pytest >=7.4.0 + *requires*: pytest>=7.4.0 Pytest plugin to simplify running shell commands against the system @@ -9836,12 +11571,19 @@ This list contains 1487 plugins. Expand command-line shortcuts listed in pytest configuration :pypi:`pytest-shutil` - *last release*: May 28, 2019, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest A goodie-bag of unix shell and environment tools for py.test + :pypi:`pytest-sigil` + *last release*: Oct 21, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=7.0.0 + + Proper fixture resource cleanup by handling signals + :pypi:`pytest-simbind` *last release*: Mar 28, 2024, *status*: N/A, @@ -9877,10 +11619,17 @@ This list contains 1487 plugins. Allow for multiple processes to log to a single file + :pypi:`pytest-skip` + *last release*: Sep 12, 2025, + *status*: 3 - Alpha, + *requires*: pytest + + A pytest plugin which allows to (de-)select or skip tests from a file. + :pypi:`pytest-skip-markers` - *last release*: Jan 04, 2024, + *last release*: Aug 09, 2024, *status*: 5 - Production/Stable, - *requires*: pytest >=7.1.0 + *requires*: pytest>=7.1.0 Pytest Salt Plugin @@ -9941,9 +11690,9 @@ This list contains 1487 plugins. Prioritize running the slowest tests first. :pypi:`pytest-slow-last` - *last release*: Dec 10, 2022, + *last release*: Mar 16, 2025, *status*: 4 - Beta, - *requires*: pytest (>=3.5.0) + *requires*: pytest>=3.5.0 Run tests in order of execution time (faster tests first) @@ -9961,12 +11710,33 @@ This list contains 1487 plugins. Smart coverage plugin for pytest. + :pypi:`pytest-smart-debugger-backend` + *last release*: Sep 17, 2025, + *status*: N/A, + *requires*: N/A + + Backend server for Pytest Smart Debugger + + :pypi:`pytest-smart-rerun` + *last release*: Oct 12, 2025, + *status*: 3 - Alpha, + *requires*: N/A + + A Pytest plugin for intelligent retrying of flaky tests. + :pypi:`pytest-smell` *last release*: Jun 26, 2022, *status*: N/A, *requires*: N/A - Automated bad smell detection tool for Pytest + Automated bad smell detection tool for Pytest + + :pypi:`pytest-smoke` + *last release*: Oct 08, 2025, + *status*: 4 - Beta, + *requires*: pytest<9,>=7.0.0 + + Pytest plugin for smoke testing :pypi:`pytest-smtp` *last release*: Feb 20, 2021, @@ -10003,6 +11773,20 @@ This list contains 1487 plugins. Plugin for adding a marker to slow running tests. 🐌 + :pypi:`pytest-snap` + *last release*: Aug 25, 2025, + *status*: N/A, + *requires*: pytest>=8.0.0 + + A text-based snapshot testing library implemented as a pytest plugin + + :pypi:`pytest-snapcheck` + *last release*: Sep 07, 2025, + *status*: N/A, + *requires*: pytest>=8.0 + + Minimal deterministic test-run snapshot capture for pytest. + :pypi:`pytest-snapci` *last release*: Nov 12, 2015, *status*: N/A, @@ -10010,6 +11794,13 @@ This list contains 1487 plugins. py.test plugin for Snap-CI + :pypi:`pytest-snapmock` + *last release*: Nov 15, 2024, + *status*: N/A, + *requires*: N/A + + Snapshots for your mocks. + :pypi:`pytest-snapshot` *last release*: Apr 23, 2022, *status*: 4 - Beta, @@ -10031,6 +11822,13 @@ This list contains 1487 plugins. + :pypi:`pytest-snob` + *last release*: Jan 12, 2025, + *status*: N/A, + *requires*: pytest + + A pytest plugin that only selects meaningful python tests to run. + :pypi:`pytest-snowflake-bdd` *last release*: Jan 05, 2022, *status*: 4 - Beta, @@ -10074,9 +11872,9 @@ This list contains 1487 plugins. Solr process and client fixtures for py.test. :pypi:`pytest-sort` - *last release*: Jan 07, 2024, + *last release*: Mar 22, 2025, *status*: N/A, - *requires*: pytest >=7.4.0 + *requires*: pytest>=7.4.0 Tools for sorting test cases @@ -10102,7 +11900,7 @@ This list contains 1487 plugins. Test-ordering plugin for pytest :pypi:`pytest-spark` - *last release*: Feb 23, 2020, + *last release*: May 21, 2025, *status*: 4 - Beta, *requires*: pytest @@ -10116,9 +11914,9 @@ This list contains 1487 plugins. py.test plugin to spawn process and communicate with them. :pypi:`pytest-spec` - *last release*: May 04, 2021, + *last release*: Oct 08, 2025, *status*: N/A, - *requires*: N/A + *requires*: pytest; extra == "test" Library pytest-spec is a pytest plugin to display test execution output like a SPECIFICATION. @@ -10165,7 +11963,7 @@ This list contains 1487 plugins. Pytest plugin for the splinter automation library :pypi:`pytest-split` - *last release*: Jun 19, 2024, + *last release*: Oct 16, 2024, *status*: 4 - Beta, *requires*: pytest<9,>=5 @@ -10200,14 +11998,14 @@ This list contains 1487 plugins. :pypi:`pytest-splunk-addon` - *last release*: Jul 11, 2024, + *last release*: Aug 19, 2025, *status*: N/A, *requires*: pytest<8,>5.4.0 A Dynamic test tool for Splunk Apps and Add-ons :pypi:`pytest-splunk-addon-ui-smartx` - *last release*: Jul 10, 2024, + *last release*: Aug 28, 2025, *status*: N/A, *requires*: N/A @@ -10228,14 +12026,14 @@ This list contains 1487 plugins. sqitch for pytest :pypi:`pytest-sqlalchemy` - *last release*: Mar 13, 2018, + *last release*: Apr 19, 2025, *status*: 3 - Alpha, - *requires*: N/A + *requires*: pytest>=8.0 pytest plugin with sqlalchemy related fixtures :pypi:`pytest-sqlalchemy-mock` - *last release*: May 21, 2024, + *last release*: Aug 10, 2024, *status*: 3 - Alpha, *requires*: pytest>=7.0.0 @@ -10262,6 +12060,13 @@ This list contains 1487 plugins. A pytest plugin to use sqlfluff to enable format checking of sql files. + :pypi:`pytest-sqlguard` + *last release*: Jun 06, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7 + + Pytest fixture to record and check SQL Queries made by SQLAlchemy + :pypi:`pytest-squadcast` *last release*: Feb 22, 2022, *status*: 5 - Production/Stable, @@ -10290,22 +12095,15 @@ This list contains 1487 plugins. Start pytest run from a given point - :pypi:`pytest-star-track-issue` - *last release*: Feb 20, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-static` - *last release*: Jun 20, 2024, - *status*: 1 - Planning, + *last release*: May 25, 2025, + *status*: 3 - Alpha, *requires*: pytest<8.0.0,>=7.4.3 pytest-static :pypi:`pytest-stats` - *last release*: Jul 03, 2024, + *last release*: Jul 18, 2024, *status*: N/A, *requires*: pytest>=8.0.0 @@ -10318,6 +12116,27 @@ This list contains 1487 plugins. pytest plugin for reporting to graphite + :pypi:`pytest-status` + *last release*: Aug 22, 2024, + *status*: N/A, + *requires*: pytest + + Add status mark for tests + + :pypi:`pytest-stderr-db` + *last release*: Sep 14, 2025, + *status*: N/A, + *requires*: N/A + + Add your description here + + :pypi:`pytest-stdout-db` + *last release*: Sep 14, 2025, + *status*: N/A, + *requires*: N/A + + Add your description here + :pypi:`pytest-stepfunctions` *last release*: May 08, 2021, *status*: 4 - Beta, @@ -10332,6 +12151,13 @@ This list contains 1487 plugins. Create step-wise / incremental tests in pytest. + :pypi:`pytest-stepthrough` + *last release*: Aug 14, 2025, + *status*: N/A, + *requires*: N/A + + Pause and wait for Enter after each test with --step + :pypi:`pytest-stepwise` *last release*: Dec 01, 2015, *status*: 4 - Beta, @@ -10340,12 +12166,19 @@ This list contains 1487 plugins. Run a test suite one failing test at a time. :pypi:`pytest-stf` - *last release*: Mar 25, 2024, + *last release*: Sep 23, 2025, *status*: N/A, *requires*: pytest>=5.0 pytest plugin for openSTF + :pypi:`pytest-stochastics` + *last release*: Dec 01, 2024, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.0.0 + + pytest plugin that allows selectively running tests several times and accepting \*some\* failures. + :pypi:`pytest-stoq` *last release*: Feb 09, 2021, *status*: 4 - Beta, @@ -10353,13 +12186,27 @@ This list contains 1487 plugins. A plugin to pytest stoq + :pypi:`pytest-storage` + *last release*: Sep 12, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=8.4.2 + + Pytest plugin to store test artifacts + :pypi:`pytest-store` - *last release*: Nov 16, 2023, + *last release*: Jul 30, 2025, *status*: 3 - Alpha, - *requires*: pytest (>=7.0.0) + *requires*: pytest>=7.0.0 Pytest plugin to store values from test runs + :pypi:`pytest-streaming` + *last release*: May 28, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8.3.5 + + Plugin for testing pubsub, pulsar, and kafka systems with pytest locally and in ci/cd + :pypi:`pytest-stress` *last release*: Dec 07, 2019, *status*: 4 - Beta, @@ -10368,7 +12215,7 @@ This list contains 1487 plugins. A Pytest plugin that allows you to loop tests for a user defined amount of time. :pypi:`pytest-structlog` - *last release*: Jun 09, 2024, + *last release*: Sep 10, 2025, *status*: N/A, *requires*: pytest @@ -10409,10 +12256,17 @@ This list contains 1487 plugins. Run pytest in a subinterpreter + :pypi:`pytest-subket` + *last release*: Jul 31, 2025, + *status*: 4 - Beta, + *requires*: N/A + + Pytest Plugin to disable socket calls during tests + :pypi:`pytest-subprocess` - *last release*: Jan 28, 2023, + *last release*: Jan 04, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=4.0.0) + *requires*: pytest>=4.0.0 A plugin to fake subprocess for pytest @@ -10424,9 +12278,9 @@ This list contains 1487 plugins. A hack to explicitly set up and tear down fixtures. :pypi:`pytest-subtests` - *last release*: Jul 07, 2024, + *last release*: Oct 20, 2025, *status*: 4 - Beta, - *requires*: pytest>=7.0 + *requires*: pytest>=7.4 unittest subTest() support and subtests fixture @@ -10438,9 +12292,9 @@ This list contains 1487 plugins. pytest-subunit is a plugin for py.test which outputs testsresult in subunit format. :pypi:`pytest-sugar` - *last release*: Feb 01, 2024, + *last release*: Aug 23, 2025, *status*: 4 - Beta, - *requires*: pytest >=6.2.0 + *requires*: pytest>=6.2.0 pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly). @@ -10466,7 +12320,7 @@ This list contains 1487 plugins. Pytest plugin for measuring explicit test-file to source-file coverage :pypi:`pytest-svn` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest @@ -10479,13 +12333,6 @@ This list contains 1487 plugins. pytest-symbols is a pytest plugin that adds support for passing test environment symbols into pytest tests. - :pypi:`pytest-synodic` - *last release*: Mar 09, 2024, - *status*: N/A, - *requires*: pytest>=8.0.2 - - Synodic Pytest utilities - :pypi:`pytest-system-statistics` *last release*: Feb 16, 2022, *status*: 5 - Production/Stable, @@ -10501,14 +12348,14 @@ This list contains 1487 plugins. Pyst - Pytest System-Test Plugin :pypi:`pytest_tagging` - *last release*: Apr 08, 2024, + *last release*: Nov 08, 2024, *status*: N/A, - *requires*: pytest<8.0.0,>=7.1.3 + *requires*: pytest>=7.1.3 a pytest plugin to tag tests :pypi:`pytest-takeltest` - *last release*: Feb 15, 2023, + *last release*: Sep 07, 2024, *status*: N/A, *requires*: N/A @@ -10529,9 +12376,9 @@ This list contains 1487 plugins. A Pytest plugin to generate realtime summary stats, and display them in-console using a text-based dashboard. :pypi:`pytest-tap` - *last release*: Jul 15, 2023, + *last release*: Jan 30, 2025, *status*: 5 - Production/Stable, - *requires*: pytest (>=3.0) + *requires*: pytest>=3.0 Test Anything Protocol (TAP) reporting plugin for pytest @@ -10549,6 +12396,13 @@ This list contains 1487 plugins. Pytest plugin for remote target orchestration. + :pypi:`pytest-taskgraph` + *last release*: Apr 09, 2025, + *status*: N/A, + *requires*: pytest + + Add your description here + :pypi:`pytest-tblineinfo` *last release*: Dec 01, 2015, *status*: 3 - Alpha, @@ -10577,6 +12431,13 @@ This list contains 1487 plugins. py.test plugin to introduce block structure in teamcity build log, if output is not captured + :pypi:`pytest-teardown` + *last release*: Apr 15, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=7.4.1 + + + :pypi:`pytest-telegram` *last release*: Apr 25, 2024, *status*: 5 - Production/Stable, @@ -10619,6 +12480,13 @@ This list contains 1487 plugins. generate terraform resources to use with pytest + :pypi:`pytest-test-analyzer` + *last release*: Jun 14, 2025, + *status*: 4 - Beta, + *requires*: N/A + + A powerful tool for analyzing pytest test files and generating detailed reports + :pypi:`pytest-testbook` *last release*: Dec 11, 2016, *status*: 3 - Alpha, @@ -10633,6 +12501,13 @@ This list contains 1487 plugins. Test configuration plugin for pytest. + :pypi:`pytest-testdata` + *last release*: Aug 30, 2024, + *status*: N/A, + *requires*: pytest + + Get and load testdata in pytest projects + :pypi:`pytest-testdirectory` *last release*: May 02, 2023, *status*: 5 - Production/Stable, @@ -10655,14 +12530,14 @@ This list contains 1487 plugins. A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. :pypi:`pytest-test-groups` - *last release*: Oct 25, 2016, + *last release*: May 08, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest>=7.0.0 A Pytest plugin for running a subset of your tests by splitting them in to equally sized groups. :pypi:`pytest-testinfra` - *last release*: May 26, 2024, + *last release*: Mar 30, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=6 @@ -10682,6 +12557,13 @@ This list contains 1487 plugins. Test infrastructures + :pypi:`pytest-testit-parametrize` + *last release*: Dec 04, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.3.3 + + A pytest plugin for uploading parameterized tests parameters into TMS TestIT + :pypi:`pytest-testlink-adaptor` *last release*: Dec 20, 2018, *status*: 4 - Beta, @@ -10690,9 +12572,9 @@ This list contains 1487 plugins. pytest reporting plugin for testlink :pypi:`pytest-testmon` - *last release*: Feb 27, 2024, + *last release*: Dec 22, 2024, *status*: 4 - Beta, - *requires*: pytest <9,>=5 + *requires*: pytest<9,>=5 selects tests affected by changed files and methods @@ -10745,6 +12627,13 @@ This list contains 1487 plugins. A pytest plugin to upload results to TestRail. + :pypi:`pytest-testrail-api` + *last release*: Mar 17, 2025, + *status*: N/A, + *requires*: pytest + + TestRail Api Python Client + :pypi:`pytest-testrail-api-client` *last release*: Dec 14, 2021, *status*: N/A, @@ -10844,7 +12733,7 @@ This list contains 1487 plugins. A plugin that allows coll test data for use on Test Tracer :pypi:`pytest-test-tracer-for-pytest-bdd` - *last release*: Jul 01, 2024, + *last release*: Aug 20, 2024, *status*: 4 - Beta, *requires*: pytest>=6.2.0 @@ -10858,16 +12747,16 @@ This list contains 1487 plugins. :pypi:`pytest-tesults` - *last release*: Feb 15, 2024, + *last release*: Nov 12, 2024, *status*: 5 - Production/Stable, - *requires*: pytest >=3.5.0 + *requires*: pytest>=3.5.0 Tesults plugin for pytest :pypi:`pytest-textual-snapshot` - *last release*: Aug 23, 2023, - *status*: 4 - Beta, - *requires*: pytest (>=7.0.0) + *last release*: Jan 23, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=8.0.0 Snapshot testing for Textual apps @@ -10921,7 +12810,7 @@ This list contains 1487 plugins. Ticking on tests :pypi:`pytest-time` - *last release*: Jun 24, 2023, + *last release*: Jan 20, 2025, *status*: 3 - Alpha, *requires*: pytest @@ -10942,9 +12831,9 @@ This list contains 1487 plugins. A pytest plugin to time test function runs :pypi:`pytest-timeout` - *last release*: Mar 07, 2024, + *last release*: May 05, 2025, *status*: 5 - Production/Stable, - *requires*: pytest >=7.0.0 + *requires*: pytest>=7.0.0 pytest plugin to abort hanging tests @@ -10976,6 +12865,13 @@ This list contains 1487 plugins. A simple plugin to view timestamps for each test + :pypi:`pytest-timing-plugin` + *last release*: Jul 21, 2025, + *status*: N/A, + *requires*: N/A + + pytest插件开发demo + :pypi:`pytest-tiny-api-client` *last release*: Jan 04, 2024, *status*: 5 - Production/Stable, @@ -10984,9 +12880,9 @@ This list contains 1487 plugins. The companion pytest plugin for tiny-api-client :pypi:`pytest-tinybird` - *last release*: Jun 26, 2023, + *last release*: May 07, 2025, *status*: 4 - Beta, - *requires*: pytest (>=3.8.0) + *requires*: pytest>=3.8.0 A pytest plugin to report test results to tinybird @@ -11047,7 +12943,7 @@ This list contains 1487 plugins. this is a vue-element ui report for pytest :pypi:`pytest-tmux` - *last release*: Apr 22, 2023, + *last release*: Sep 01, 2025, *status*: 4 - Beta, *requires*: N/A @@ -11166,9 +13062,9 @@ This list contains 1487 plugins. Plugin for py.test that integrates trello using markers :pypi:`pytest-trepan` - *last release*: Jul 28, 2018, + *last release*: Sep 11, 2025, *status*: 5 - Production/Stable, - *requires*: N/A + *requires*: pytest>=4.0.0 Pytest plugin for trepan debugger. @@ -11221,6 +13117,20 @@ This list contains 1487 plugins. Text User Interface (TUI) and HTML report for Pytest test runs + :pypi:`pytest-tui-runner` + *last release*: Oct 23, 2025, + *status*: N/A, + *requires*: pytest>=8.3.5 + + Textual-based terminal UI for running pytest tests + + :pypi:`pytest-tuitest` + *last release*: Apr 11, 2025, + *status*: N/A, + *requires*: pytest>=7.4.0 + + pytest plugin for testing TUI and regular command-line applications. + :pypi:`pytest-tutorials` *last release*: Mar 11, 2023, *status*: N/A, @@ -11236,12 +13146,19 @@ This list contains 1487 plugins. :pypi:`pytest-twisted` - *last release*: Jul 10, 2024, + *last release*: Sep 10, 2024, *status*: 5 - Production/Stable, *requires*: pytest>=2.3 A twisted plugin for pytest. + :pypi:`pytest-ty` + *last release*: Oct 10, 2025, + *status*: 3 - Alpha, + *requires*: pytest>=7.0.0 + + A pytest plugin to run the ty type checker + :pypi:`pytest-typechecker` *last release*: Feb 04, 2022, *status*: N/A, @@ -11249,6 +13166,13 @@ This list contains 1487 plugins. Run type checkers on specified test files + :pypi:`pytest-typed-schema-shot` + *last release*: Jun 14, 2025, + *status*: N/A, + *requires*: pytest + + Pytest plugin for automatic JSON Schema generation and validation from examples + :pypi:`pytest-typhoon-config` *last release*: Apr 07, 2022, *status*: 5 - Production/Stable, @@ -11270,6 +13194,13 @@ This list contains 1487 plugins. Typhoon HIL plugin for pytest + :pypi:`pytest-typing-runner` + *last release*: May 31, 2025, + *status*: N/A, + *requires*: N/A + + Pytest plugin to make it easier to run and check python code against static type checkers + :pypi:`pytest-tytest` *last release*: May 25, 2020, *status*: 4 - Beta, @@ -11277,6 +13208,13 @@ This list contains 1487 plugins. Typhoon HIL plugin for pytest + :pypi:`pytest-tzshift` + *last release*: Jun 25, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0 + + A Pytest plugin that transparently re-runs tests under a matrix of timezones and locales. + :pypi:`pytest-ubersmith` *last release*: Apr 13, 2015, *status*: N/A, @@ -11306,7 +13244,7 @@ This list contains 1487 plugins. UI自动测试失败时自动截图,并将截图加入到Allure测试报告中 :pypi:`pytest-uncollect-if` - *last release*: Mar 24, 2024, + *last release*: Dec 26, 2024, *status*: 4 - Beta, *requires*: pytest>=6.2.0 @@ -11327,9 +13265,9 @@ This list contains 1487 plugins. Plugin for py.test set a different exit code on uncaught exceptions :pypi:`pytest-unique` - *last release*: Sep 15, 2023, + *last release*: Jun 10, 2025, *status*: N/A, - *requires*: pytest (>=7.4.2,<8.0.0) + *requires*: pytest<9.0.0,>=8.0.0 Pytest fixture to generate unique values. @@ -11340,6 +13278,20 @@ This list contains 1487 plugins. A pytest plugin for filtering unittest-based test classes + :pypi:`pytest-unittest-id-runner` + *last release*: Feb 09, 2025, + *status*: N/A, + *requires*: pytest>=6.0.0 + + A pytest plugin to run tests using unittest-style test IDs + + :pypi:`pytest-unmagic` + *last release*: Jul 14, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest + + Pytest fixtures with conventional import semantics + :pypi:`pytest-unmarked` *last release*: Aug 27, 2019, *status*: 5 - Production/Stable, @@ -11348,7 +13300,7 @@ This list contains 1487 plugins. Run only unmarked tests :pypi:`pytest-unordered` - *last release*: Jul 05, 2024, + *last release*: Jun 03, 2025, *status*: 4 - Beta, *requires*: pytest>=7.0.0 @@ -11362,12 +13314,19 @@ This list contains 1487 plugins. Set a test as unstable to return 0 even if it failed :pypi:`pytest-unused-fixtures` - *last release*: Apr 08, 2024, + *last release*: Mar 15, 2025, *status*: 4 - Beta, *requires*: pytest>7.3.2 A pytest plugin to list unused fixtures after a test run. + :pypi:`pytest-unused-port` + *last release*: Oct 22, 2025, + *status*: N/A, + *requires*: pytest + + pytest fixture finding an unused local port + :pypi:`pytest-upload-report` *last release*: Jun 18, 2021, *status*: 5 - Production/Stable, @@ -11446,12 +13405,19 @@ This list contains 1487 plugins. py.test fixture for creating a virtual environment :pypi:`pytest-verbose-parametrize` - *last release*: May 28, 2019, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest More descriptive output for parametrized py.test tests + :pypi:`pytest-verify` + *last release*: Oct 25, 2025, + *status*: 5 - Production/Stable, + *requires*: N/A + + A pytest plugin for snapshot verification with optional visual diff viewer. + :pypi:`pytest-vimqf` *last release*: Feb 08, 2021, *status*: 4 - Beta, @@ -11460,16 +13426,16 @@ This list contains 1487 plugins. A simple pytest plugin that will shrink pytest output when specified, to fit vim quickfix window. :pypi:`pytest-virtualenv` - *last release*: May 28, 2019, + *last release*: Nov 29, 2024, *status*: 5 - Production/Stable, *requires*: pytest Virtualenv fixture for py.test :pypi:`pytest-visual` - *last release*: Nov 01, 2023, - *status*: 3 - Alpha, - *requires*: pytest >=7.0.0 + *last release*: Nov 28, 2024, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 @@ -11501,6 +13467,13 @@ This list contains 1487 plugins. A PyTest helper to enable start remote debugger on test start or failure or when pytest.set_trace is used. + :pypi:`pytest-vtestify` + *last release*: Feb 04, 2025, + *status*: N/A, + *requires*: pytest + + A pytest plugin for visual assertion using SSIM and image comparison. + :pypi:`pytest-vts` *last release*: Jun 05, 2019, *status*: N/A, @@ -11509,9 +13482,9 @@ This list contains 1487 plugins. pytest plugin for automatic recording of http stubbed tests :pypi:`pytest-vulture` - *last release*: Jun 01, 2023, + *last release*: Nov 25, 2024, *status*: N/A, - *requires*: pytest (>=7.0.0) + *requires*: pytest>=7.0.0 A pytest plugin to checks dead code with vulture @@ -11537,7 +13510,7 @@ This list contains 1487 plugins. Pytest plugin for testing whatsapp bots with end to end tests :pypi:`pytest-wake` - *last release*: Mar 20, 2024, + *last release*: Nov 19, 2024, *status*: N/A, *requires*: pytest @@ -11551,12 +13524,19 @@ This list contains 1487 plugins. Local continuous test runner with pytest and watchdog. :pypi:`pytest-watcher` - *last release*: Apr 01, 2024, + *last release*: Aug 28, 2024, *status*: 4 - Beta, *requires*: N/A Automatically rerun your tests on file modifications + :pypi:`pytest-watch-plugin` + *last release*: Sep 12, 2024, + *status*: N/A, + *requires*: N/A + + Placeholder for internal package + :pypi:`pytest_wdb` *last release*: Jul 04, 2016, *status*: N/A, @@ -11579,18 +13559,18 @@ This list contains 1487 plugins. A pytest plugin to fetch test data from IPFS HTTP gateways during pytest execution. :pypi:`pytest-webdriver` - *last release*: May 28, 2019, + *last release*: Oct 17, 2024, *status*: 5 - Production/Stable, *requires*: pytest Selenium webdriver fixture for py.test - :pypi:`pytest-webtest-extras` - *last release*: Jun 08, 2024, + :pypi:`pytest-webstage` + *last release*: Sep 20, 2024, *status*: N/A, - *requires*: pytest>=7.0.0 + *requires*: pytest<9.0,>=7.0 - Pytest plugin to enhance pytest-html and allure reports of webtest projects by adding screenshots, comments and webpage sources. + Test web apps with pytest :pypi:`pytest-wetest` *last release*: Nov 10, 2018, @@ -11600,7 +13580,7 @@ This list contains 1487 plugins. Welian API Automation test framework pytest plugin :pypi:`pytest-when` - *last release*: May 28, 2024, + *last release*: Sep 25, 2025, *status*: N/A, *requires*: pytest>=7.3.1 @@ -11641,6 +13621,13 @@ This list contains 1487 plugins. A pytest plugin for programmatically using wiremock in integration tests + :pypi:`pytest-wiretap` + *last release*: Mar 18, 2025, + *status*: N/A, + *requires*: pytest + + \`pytest\` plugin for recording call stacks + :pypi:`pytest-with-docker` *last release*: Nov 09, 2021, *status*: N/A, @@ -11648,6 +13635,13 @@ This list contains 1487 plugins. pytest with docker helpers. + :pypi:`pytest-workaround-12888` + *last release*: Jan 15, 2025, + *status*: N/A, + *requires*: N/A + + forces an import of readline early in the process to work around pytest bug #12888 + :pypi:`pytest-workflow` *last release*: Mar 18, 2024, *status*: 5 - Production/Stable, @@ -11656,7 +13650,7 @@ This list contains 1487 plugins. A pytest plugin for configuring workflow/pipeline tests using YAML files :pypi:`pytest-xdist` - *last release*: Apr 28, 2024, + *last release*: Jul 01, 2025, *status*: 5 - Production/Stable, *requires*: pytest>=7.0.0 @@ -11676,6 +13670,13 @@ This list contains 1487 plugins. forked from pytest-xdist + :pypi:`pytest-xdist-gnumake` + *last release*: Jun 22, 2025, + *status*: N/A, + *requires*: pytest + + A small example package + :pypi:`pytest-xdist-tracker` *last release*: Nov 18, 2021, *status*: 3 - Alpha, @@ -11684,12 +13685,19 @@ This list contains 1487 plugins. pytest plugin helps to reproduce failures for particular xdist node :pypi:`pytest-xdist-worker-stats` - *last release*: Apr 16, 2024, + *last release*: Mar 15, 2025, *status*: 4 - Beta, *requires*: pytest>=7.0.0 A pytest plugin to list worker statistics after a xdist run. + :pypi:`pytest-xdocker` + *last release*: Jun 10, 2025, + *status*: N/A, + *requires*: pytest<9.0.0,>=8.0.0 + + Pytest fixture to run docker across test runs. + :pypi:`pytest-xfaillist` *last release*: Sep 17, 2021, *status*: N/A, @@ -11704,6 +13712,20 @@ This list contains 1487 plugins. Pytest fixtures providing data read from function, module or package related (x)files. + :pypi:`pytest-xflaky` + *last release*: Oct 14, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.2.1 + + A simple plugin to use with pytest + + :pypi:`pytest-xhtml` + *last release*: Oct 18, 2025, + *status*: 5 - Production/Stable, + *requires*: pytest>=7 + + pytest plugin for generating HTML reports + :pypi:`pytest-xiuyu` *last release*: Jul 25, 2023, *status*: 5 - Production/Stable, @@ -11719,14 +13741,21 @@ This list contains 1487 plugins. Extended logging for test and decorators :pypi:`pytest-xlsx` - *last release*: Apr 23, 2024, + *last release*: Aug 07, 2024, *status*: N/A, - *requires*: pytest~=7.0 + *requires*: pytest~=8.2.2 pytest plugin for generating test cases by xlsx(excel) + :pypi:`pytest-xml` + *last release*: Nov 14, 2024, + *status*: 4 - Beta, + *requires*: pytest>=8.0.0 + + Create simple XML results for parsing + :pypi:`pytest-xpara` - *last release*: Oct 30, 2017, + *last release*: Aug 07, 2024, *status*: 3 - Alpha, *requires*: pytest @@ -11753,6 +13782,13 @@ This list contains 1487 plugins. + :pypi:`pytest-xray-reporter` + *last release*: May 21, 2025, + *status*: 4 - Beta, + *requires*: pytest>=7.0.0 + + Pytest plugin for generating Xray JSON reports + :pypi:`pytest-xray-server` *last release*: May 03, 2022, *status*: 3 - Alpha, @@ -11760,13 +13796,6 @@ This list contains 1487 plugins. - :pypi:`pytest-xskynet` - *last release*: Feb 20, 2024, - *status*: N/A, - *requires*: N/A - - A package to prevent Dependency Confusion attacks against Yandex. - :pypi:`pytest-xstress` *last release*: Jun 01, 2024, *status*: N/A, @@ -11774,15 +13803,22 @@ This list contains 1487 plugins. + :pypi:`pytest-xtime` + *last release*: Jun 05, 2025, + *status*: 4 - Beta, + *requires*: pytest + + pytest plugin for recording execution time + :pypi:`pytest-xvfb` - *last release*: May 29, 2023, + *last release*: Mar 12, 2025, *status*: 4 - Beta, - *requires*: pytest (>=2.8.1) + *requires*: pytest>=2.8.1 A pytest plugin to run Xvfb (or Xephyr/Xvnc) for tests. :pypi:`pytest-xvirt` - *last release*: Jul 03, 2024, + *last release*: Dec 15, 2024, *status*: 4 - Beta, *requires*: pytest>=7.2.2 @@ -11795,12 +13831,19 @@ This list contains 1487 plugins. This plugin is used to load yaml output to your test using pytest framework. + :pypi:`pytest-yaml-fei` + *last release*: Aug 03, 2025, + *status*: N/A, + *requires*: pytest + + a pytest yaml allure package + :pypi:`pytest-yaml-sanmu` - *last release*: Jul 12, 2024, + *last release*: Sep 16, 2025, *status*: N/A, - *requires*: pytest>=7.4.0 + *requires*: pytest>=8.2.2 - pytest plugin for generating test cases by yaml + Pytest plugin for generating test cases with YAML. In test cases, you can use markers, fixtures, variables, and even call Python functions. :pypi:`pytest-yamltree` *last release*: Mar 02, 2020, @@ -11845,9 +13888,9 @@ This list contains 1487 plugins. PyTest plugin to run tests concurrently, each \`yield\` switch context to other one :pypi:`pytest-yls` - *last release*: Mar 30, 2024, + *last release*: Apr 09, 2025, *status*: N/A, - *requires*: pytest<8.0.0,>=7.2.2 + *requires*: pytest<9.0.0,>=8.3.3 Pytest plugin to test the YLS as a whole. @@ -11900,6 +13943,20 @@ This list contains 1487 plugins. Pytest fixtures for testing Camunda 8 processes using a Zeebe test engine. + :pypi:`pytest-zephyr-scale-integration` + *last release*: Jun 26, 2025, + *status*: N/A, + *requires*: pytest + + A library for integrating Jira Zephyr Scale (Adaptavist\TM4J) with pytest + + :pypi:`pytest-zephyr-telegram` + *last release*: Sep 30, 2024, + *status*: N/A, + *requires*: pytest==8.3.2 + + Плагин для отправки данных автотестов в Телеграм и Зефир + :pypi:`pytest-zest` *last release*: Nov 17, 2022, *status*: N/A, @@ -11934,3 +13991,10 @@ This list contains 1487 plugins. *requires*: pytest~=7.2.0 接口自动化测试框架 + + :pypi:`tursu` + *last release*: May 05, 2025, + *status*: 4 - Beta, + *requires*: pytest>=8.3.5 + + 🎬 A pytest plugin that transpiles Gherkin feature files to Python using AST, enforcing typing for ease of use and debugging. diff --git a/doc/en/reference/reference.rst b/doc/en/reference/reference.rst index d1222728e13..3760add53cf 100644 --- a/doc/en/reference/reference.rst +++ b/doc/en/reference/reference.rst @@ -18,8 +18,18 @@ The current pytest version, as a string:: >>> import pytest >>> pytest.__version__ - '7.0.0' + '9.0.2' +.. _`hidden-param`: + +pytest.HIDDEN_PARAM +~~~~~~~~~~~~~~~~~~~ + +.. versionadded:: 8.4 + +Can be passed to ``ids`` of :py:func:`Metafunc.parametrize ` +or to ``id`` of :func:`pytest.param` to hide a parameter set from the test name. +Can only be used at most 1 time, as test names need to be unique. .. _`version-tuple`: @@ -54,7 +64,7 @@ pytest.fail **Tutorial**: :ref:`skipping` -.. autofunction:: pytest.fail(reason, [pytrace=True, msg=None]) +.. autofunction:: pytest.fail(reason, [pytrace=True]) .. class:: pytest.fail.Exception @@ -63,7 +73,7 @@ pytest.fail pytest.skip ~~~~~~~~~~~ -.. autofunction:: pytest.skip(reason, [allow_module_level=False, msg=None]) +.. autofunction:: pytest.skip(reason, [allow_module_level=False]) .. class:: pytest.skip.Exception @@ -88,7 +98,7 @@ pytest.xfail pytest.exit ~~~~~~~~~~~ -.. autofunction:: pytest.exit(reason, [returncode=None, msg=None]) +.. autofunction:: pytest.exit(reason, [returncode=None]) .. class:: pytest.exit.Exception @@ -251,7 +261,7 @@ pytest.mark.xfail Marks a test function as *expected to fail*. -.. py:function:: pytest.mark.xfail(condition=False, *, reason=None, raises=None, run=True, strict=xfail_strict) +.. py:function:: pytest.mark.xfail(condition=False, *, reason=None, raises=None, run=True, strict=strict_xfail) :keyword Union[bool, str] condition: Condition for marking the test function as xfail (``True/False`` or a @@ -276,7 +286,7 @@ Marks a test function as *expected to fail*. that are always failing and there should be a clear indication if they unexpectedly start to pass (for example a new release of a library fixes a known bug). - Defaults to :confval:`xfail_strict`, which is ``False`` by default. + Defaults to :confval:`strict_xfail`, which is ``False`` by default. Custom marks @@ -402,6 +412,16 @@ capsys .. autoclass:: pytest.CaptureFixture() :members: +.. fixture:: capteesys + +capteesys +~~~~~~~~~ + +**Tutorial**: :ref:`captures` + +.. autofunction:: _pytest.capture.capteesys() + :no-auto-options: + .. fixture:: capsysbinary capsysbinary @@ -529,13 +549,14 @@ record_testsuite_property recwarn ~~~~~~~ -**Tutorial**: :ref:`assertwarnings` +**Tutorial**: :ref:`recwarn` .. autofunction:: _pytest.recwarn.recwarn() :no-auto-options: .. autoclass:: pytest.WarningsRecorder() :members: + :special-members: __getitem__, __iter__, __len__ .. fixture:: request @@ -551,6 +572,19 @@ The ``request`` fixture is a special fixture providing information of the reques :members: +.. fixture:: subtests + +subtests +~~~~~~~~ + +The ``subtests`` fixture enables declaring subtests inside test functions. + +**Tutorial**: :ref:`subtests` + +.. autoclass:: pytest.Subtests() + :members: + + .. fixture:: testdir testdir @@ -723,6 +757,7 @@ items, delete or otherwise amend the test items: If this hook is implemented in ``conftest.py`` files, it always receives all collected items, not only those under the ``conftest.py`` where it is implemented. +.. hook:: pytest_collection_finish .. autofunction:: pytest_collection_finish Test running (runtest) hooks @@ -1013,6 +1048,30 @@ PytestPluginManager :inherited-members: :show-inheritance: +RaisesExc +~~~~~~~~~ + +.. autoclass:: pytest.RaisesExc() + :members: + + .. autoattribute:: fail_reason + +RaisesGroup +~~~~~~~~~~~ +**Tutorial**: :ref:`assert-matching-exception-groups` + +.. autoclass:: pytest.RaisesGroup() + :members: + + .. autoattribute:: fail_reason + +TerminalReporter +~~~~~~~~~~~~~~~~ + +.. autoclass:: pytest.TerminalReporter + :members: + :inherited-members: + TestReport ~~~~~~~~~~ @@ -1120,69 +1179,77 @@ Environment variables that can be used to change pytest's behavior. .. envvar:: CI -When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to ``BUILD_NUMBER`` variable. See also :ref:`ci-pipelines`. + When set to a non-empty value, pytest acknowledges that it is running in a CI process. See also :ref:`ci-pipelines`. .. envvar:: BUILD_NUMBER -When set (regardless of value), pytest acknowledges that is running in a CI process. Alternative to CI variable. See also :ref:`ci-pipelines`. + When set to a non-empty value, pytest acknowledges that it is running in a CI process. Alternative to :envvar:`CI`. See also :ref:`ci-pipelines`. .. envvar:: PYTEST_ADDOPTS -This contains a command-line (parsed by the py:mod:`shlex` module) that will be **prepended** to the command line given -by the user, see :ref:`adding default options` for more information. + This contains a command-line (parsed by the py:mod:`shlex` module) that will be **prepended** to the command line given + by the user, see :ref:`adding default options` for more information. .. envvar:: PYTEST_VERSION -This environment variable is defined at the start of the pytest session and is undefined afterwards. -It contains the value of ``pytest.__version__``, and among other things can be used to easily check if a code is running from within a pytest run. + This environment variable is defined at the start of the pytest session and is undefined afterwards. + It contains the value of ``pytest.__version__``, and among other things can be used to easily check if a code is running from within a pytest run. .. envvar:: PYTEST_CURRENT_TEST -This is not meant to be set by users, but is set by pytest internally with the name of the current test so other -processes can inspect it, see :ref:`pytest current test env` for more information. + This is not meant to be set by users, but is set by pytest internally with the name of the current test so other + processes can inspect it, see :ref:`pytest current test env` for more information. .. envvar:: PYTEST_DEBUG -When set, pytest will print tracing and debug information. + When set, pytest will print tracing and debug information. + +.. envvar:: PYTEST_DEBUG_TEMPROOT + + Root for temporary directories produced by fixtures like :fixture:`tmp_path` + as discussed in :ref:`temporary directory location and retention`. .. envvar:: PYTEST_DISABLE_PLUGIN_AUTOLOAD -When set, disables plugin auto-loading through :std:doc:`entry point packaging -metadata `. Only explicitly -specified plugins will be loaded. + When set, disables plugin auto-loading through :std:doc:`entry point packaging + metadata `. Only plugins + explicitly specified in :envvar:`PYTEST_PLUGINS` or with :option:`-p` will be loaded. + See also :ref:`--disable-plugin-autoload `. .. envvar:: PYTEST_PLUGINS -Contains comma-separated list of modules that should be loaded as plugins: + Contains comma-separated list of modules that should be loaded as plugins: + + .. code-block:: bash -.. code-block:: bash + export PYTEST_PLUGINS=mymodule.plugin,xdist - export PYTEST_PLUGINS=mymodule.plugin,xdist + See also :option:`-p`. .. envvar:: PYTEST_THEME -Sets a `pygment style `_ to use for the code output. + Sets a `pygment style `_ to use for the code output. .. envvar:: PYTEST_THEME_MODE -Sets the :envvar:`PYTEST_THEME` to be either *dark* or *light*. + Sets the :envvar:`PYTEST_THEME` to be either *dark* or *light*. .. envvar:: PY_COLORS -When set to ``1``, pytest will use color in terminal output. -When set to ``0``, pytest will not use color. -``PY_COLORS`` takes precedence over ``NO_COLOR`` and ``FORCE_COLOR``. + When set to ``1``, pytest will use color in terminal output. + When set to ``0``, pytest will not use color. + ``PY_COLORS`` takes precedence over ``NO_COLOR`` and ``FORCE_COLOR``. .. envvar:: NO_COLOR -When set to a non-empty string (regardless of value), pytest will not use color in terminal output. -``PY_COLORS`` takes precedence over ``NO_COLOR``, which takes precedence over ``FORCE_COLOR``. -See `no-color.org `__ for other libraries supporting this community standard. + When set to a non-empty string (regardless of value), pytest will not use color in terminal output. + ``PY_COLORS`` takes precedence over ``NO_COLOR``, which takes precedence over ``FORCE_COLOR``. + See `no-color.org `__ for other libraries supporting this community standard. .. envvar:: FORCE_COLOR -When set to a non-empty string (regardless of value), pytest will use color in terminal output. -``PY_COLORS`` and ``NO_COLOR`` take precedence over ``FORCE_COLOR``. + When set to a non-empty string (regardless of value), pytest will use color in terminal output. + ``PY_COLORS`` and ``NO_COLOR`` take precedence over ``FORCE_COLOR``. Exceptions ---------- @@ -1227,9 +1294,6 @@ Custom warnings generated in some situations such as improper usage or deprecate .. autoclass:: pytest.PytestRemovedIn9Warning :show-inheritance: -.. autoclass:: pytest.PytestUnhandledCoroutineWarning - :show-inheritance: - .. autoclass:: pytest.PytestUnknownMarkWarning :show-inheritance: @@ -1251,13 +1315,13 @@ Configuration Options Here is a list of builtin configuration options that may be written in a ``pytest.ini`` (or ``.pytest.ini``), ``pyproject.toml``, ``tox.ini``, or ``setup.cfg`` file, usually located at the root of your repository. -To see each file format in details, see :ref:`config file formats`. +To see each file format in detail, see :ref:`config file formats`. .. warning:: Usage of ``setup.cfg`` is not recommended except for very simple use cases. ``.cfg`` files use a different parser than ``pytest.ini`` and ``tox.ini`` which might cause hard to track down problems. - When possible, it is recommended to use the latter files, or ``pyproject.toml``, to hold your pytest configuration. + When possible, it is recommended to use the latter files, or ``pytest.toml`` or ``pyproject.toml``, to hold your pytest configuration. Configuration options may be overwritten in the command-line by using ``-o/--override-ini``, which can also be passed multiple times. The expected format is ``name=value``. For example:: @@ -1268,13 +1332,13 @@ passed multiple times. The expected format is ``name=value``. For example:: .. confval:: addopts Add the specified ``OPTS`` to the set of command line arguments as if they - had been specified by the user. Example: if you have this ini file content: + had been specified by the user. Example: if you have this configuration file content: - .. code-block:: ini + .. code-block:: toml - # content of pytest.ini + # content of pytest.toml [pytest] - addopts = --maxfail=2 -rf # exit after 2 failures, report fail info + addopts = ["--maxfail=2", "-rf"] # exit after 2 failures, report fail info issuing ``pytest test_hello.py`` actually means: @@ -1287,19 +1351,63 @@ passed multiple times. The expected format is ``name=value``. For example:: .. confval:: cache_dir - Sets a directory where stores content of cache plugin. Default directory is + Sets the directory where the cache plugin's content is stored. Default directory is ``.pytest_cache`` which is created in :ref:`rootdir `. Directory may be relative or absolute path. If setting relative path, then directory is created - relative to :ref:`rootdir `. Additionally path may contain environment + relative to :ref:`rootdir `. Additionally, a path may contain environment variables, that will be expanded. For more information about cache plugin please refer to :ref:`cache_provider`. +.. confval:: collect_imported_tests + + .. versionadded:: 8.4 + + Setting this to ``false`` will make pytest collect classes/functions from test + files **only** if they are defined in that file (as opposed to imported there). + + .. tab:: toml + + .. code-block:: toml + + [pytest] + collect_imported_tests = false + + .. tab:: ini + + .. code-block:: ini + + [pytest] + collect_imported_tests = false + + Default: ``true`` + + pytest traditionally collects classes/functions in the test module namespace even if they are imported from another file. + + For example: + + .. code-block:: python + + # contents of src/domain.py + class Testament: ... + + + # contents of tests/test_testament.py + from domain import Testament + + + def test_testament(): ... + + In this scenario, with the default options, pytest will collect the class `Testament` from `tests/test_testament.py` because it starts with `Test`, even though in this case it is a production class being imported in the test module namespace. + + Set ``collected_imported_tests`` to ``false`` in the configuration file prevents that. + .. confval:: consider_namespace_packages Controls if pytest should attempt to identify `namespace packages `__ when collecting Python modules. Default is ``False``. Set to ``True`` if the package you are testing is part of a namespace package. + Namespace packages are also supported as :option:`--pyargs` target. Only `native namespace packages `__ are supported, with no plans to support `legacy namespace packages `__. @@ -1314,16 +1422,57 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``progress``: like classic pytest output, but with a progress indicator. * ``progress-even-when-capture-no``: allows the use of the progress indicator even when ``capture=no``. * ``count``: like progress, but shows progress as the number of tests completed instead of a percent. + * ``times``: show tests duration. The default is ``progress``, but you can fallback to ``classic`` if you prefer or the new mode is causing unexpected problems: - .. code-block:: ini + .. tab:: toml - # content of pytest.ini - [pytest] - console_output_style = classic + .. code-block:: toml + + [pytest] + console_output_style = "classic" + + .. tab:: ini + .. code-block:: ini + + [pytest] + console_output_style = classic + + +.. confval:: disable_test_id_escaping_and_forfeit_all_rights_to_community_support + + .. versionadded:: 4.4 + + pytest by default escapes any non-ascii characters used in unicode strings + for the parametrization because it has several downsides. + If however you would like to use unicode strings in parametrization + and see them in the terminal as is (non-escaped), use this option + in your configuration file: + + .. tab:: toml + + .. code-block:: toml + + [pytest] + disable_test_id_escaping_and_forfeit_all_rights_to_community_support = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + disable_test_id_escaping_and_forfeit_all_rights_to_community_support = true + + Keep in mind however that this might cause unwanted side effects and + even bugs depending on the OS used and plugins currently installed, + so use it at your own risk. + + Default: ``False``. + + See :ref:`parametrizemark`. .. confval:: doctest_encoding @@ -1349,11 +1498,19 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``xfail`` marks tests with an empty parameterset as xfail(run=False) * ``fail_at_collect`` raises an exception if parametrize collects an empty parameter set - .. code-block:: ini + .. tab:: toml + + .. code-block:: toml + + [pytest] + empty_parameter_set_mark = "xfail" - # content of pytest.ini - [pytest] - empty_parameter_set_mark = xfail + .. tab:: ini + + .. code-block:: ini + + [pytest] + empty_parameter_set_mark = xfail .. note:: @@ -1361,17 +1518,72 @@ passed multiple times. The expected format is ``name=value``. For example:: as this is considered less error prone, see :issue:`3155` for more details. +.. confval:: enable_assertion_pass_hook + + Enables the :hook:`pytest_assertion_pass` hook. + Make sure to delete any previously generated ``.pyc`` cache files. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + enable_assertion_pass_hook = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + enable_assertion_pass_hook = true + + +.. confval:: faulthandler_exit_on_timeout + + Exit the pytest process after the per-test timeout is reached by passing + `exit=True` to the :func:`faulthandler.dump_traceback_later` function. This + is particularly useful to avoid wasting CI resources for test suites that + are prone to putting the main Python interpreter into a deadlock state. + + This option is set to 'false' by default. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + faulthandler_timeout = 5 + faulthandler_exit_on_timeout = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + faulthandler_timeout = 5 + faulthandler_exit_on_timeout = true + + + .. confval:: faulthandler_timeout Dumps the tracebacks of all threads if a test takes longer than ``X`` seconds to run (including fixture setup and teardown). Implemented using the :func:`faulthandler.dump_traceback_later` function, so all caveats there apply. - .. code-block:: ini + .. tab:: toml - # content of pytest.ini - [pytest] - faulthandler_timeout=5 + .. code-block:: toml + + [pytest] + faulthandler_timeout = 5 + + .. tab:: ini + + .. code-block:: ini + + [pytest] + faulthandler_timeout = 5 For more information please refer to :ref:`faulthandler`. @@ -1383,13 +1595,21 @@ passed multiple times. The expected format is ``name=value``. For example:: warnings. By default all warnings emitted during the test session will be displayed in a summary at the end of the test session. - .. code-block:: ini + .. tab:: toml - # content of pytest.ini - [pytest] - filterwarnings = - error - ignore::DeprecationWarning + .. code-block:: toml + + [pytest] + filterwarnings = ["error", "ignore::DeprecationWarning"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + filterwarnings = + error + ignore::DeprecationWarning This tells pytest to ignore deprecation warnings and turn all other warnings into errors. For more information please refer to :ref:`warnings`. @@ -1404,10 +1624,19 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``total`` (the default): duration times reported include setup, call, and teardown times. * ``call``: duration times reported include only call times, excluding setup and teardown. - .. code-block:: ini + .. tab:: toml - [pytest] - junit_duration_report = call + .. code-block:: toml + + [pytest] + junit_duration_report = "call" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + junit_duration_report = call .. confval:: junit_family @@ -1421,10 +1650,41 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``xunit1`` (or ``legacy``): produces old style output, compatible with the xunit 1.0 format. * ``xunit2``: produces `xunit 2.0 style output `__, which should be more compatible with latest Jenkins versions. **This is the default**. - .. code-block:: ini + .. tab:: toml - [pytest] - junit_family = xunit2 + .. code-block:: toml + + [pytest] + junit_family = "xunit2" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + junit_family = xunit2 + + +.. confval:: junit_log_passing_tests + + .. versionadded:: 4.6 + + If ``junit_logging != "no"``, configures if the captured output should be written + to the JUnit XML file for **passing** tests. Default is ``True``. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + junit_log_passing_tests = false + + .. tab:: ini + + .. code-block:: ini + + [pytest] + junit_log_passing_tests = False .. confval:: junit_logging @@ -1442,39 +1702,44 @@ passed multiple times. The expected format is ``name=value``. For example:: * ``all``: write captured ``logging``, ``stdout`` and ``stderr`` contents. * ``no`` (the default): no captured output is written. - .. code-block:: ini + .. tab:: toml - [pytest] - junit_logging = system-out - - -.. confval:: junit_log_passing_tests + .. code-block:: toml - .. versionadded:: 4.6 + [pytest] + junit_logging = "system-out" - If ``junit_logging != "no"``, configures if the captured output should be written - to the JUnit XML file for **passing** tests. Default is ``True``. + .. tab:: ini - .. code-block:: ini + .. code-block:: ini - [pytest] - junit_log_passing_tests = False + [pytest] + junit_logging = system-out .. confval:: junit_suite_name To set the name of the root test suite xml item, you can configure the ``junit_suite_name`` option in your config file: - .. code-block:: ini + .. tab:: toml - [pytest] - junit_suite_name = my_suite + .. code-block:: toml + + [pytest] + junit_suite_name = "my_suite" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + junit_suite_name = my_suite .. confval:: log_auto_indent Allow selective auto-indentation of multiline log messages. - Supports command line option ``--log-auto-indent [value]`` + Supports command line option :option:`--log-auto-indent=[value]` and config option ``log_auto_indent = [value]`` to set the auto-indentation behavior for all logging. @@ -1483,10 +1748,19 @@ passed multiple times. The expected format is ``name=value``. For example:: * False or "Off" or 0 - Do not auto-indent multiline log messages (the default behavior) * [positive integer] - auto-indent multiline log messages by [value] spaces - .. code-block:: ini + .. tab:: toml - [pytest] - log_auto_indent = False + .. code-block:: toml + + [pytest] + log_auto_indent = false + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_auto_indent = false Supports passing kwarg ``extra={"auto_indent": [value]}`` to calls to ``logging.log()`` to specify auto-indentation behavior for @@ -1498,10 +1772,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Enable log display during test run (also known as :ref:`"live logging" `). The default is ``False``. - .. code-block:: ini + .. tab:: toml - [pytest] - log_cli = True + .. code-block:: toml + + [pytest] + log_cli = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_cli = true .. confval:: log_cli_date_format @@ -1509,10 +1792,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for live logging. - .. code-block:: ini + .. tab:: toml - [pytest] - log_cli_date_format = %Y-%m-%d %H:%M:%S + .. code-block:: toml + + [pytest] + log_cli_date_format = "%Y-%m-%d %H:%M:%S" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_cli_date_format = %Y-%m-%d %H:%M:%S For more information, see :ref:`live_logs`. @@ -1522,10 +1814,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:mod:`logging`-compatible string used to format live logging messages. - .. code-block:: ini + .. tab:: toml - [pytest] - log_cli_format = %(asctime)s %(levelname)s %(message)s + .. code-block:: toml + + [pytest] + log_cli_format = "%(asctime)s %(levelname)s %(message)s" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_cli_format = %(asctime)s %(levelname)s %(message)s For more information, see :ref:`live_logs`. @@ -1535,12 +1836,24 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets the minimum log message level that should be captured for live logging. The integer value or - the names of the levels can be used. + the names of the levels can be used. Note in TOML the integer must be quoted, as there is no support + for config parameters of mixed type. - .. code-block:: ini + .. tab:: toml - [pytest] - log_cli_level = INFO + .. code-block:: toml + + [pytest] + log_cli_level = "INFO" + log_cli_level = "10" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_cli_level = INFO + log_cli_level = 10 For more information, see :ref:`live_logs`. @@ -1551,10 +1864,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for logging capture. - .. code-block:: ini + .. tab:: toml - [pytest] - log_date_format = %Y-%m-%d %H:%M:%S + .. code-block:: toml + + [pytest] + log_date_format = "%Y-%m-%d %H:%M:%S" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_date_format = %Y-%m-%d %H:%M:%S For more information, see :ref:`logging`. @@ -1566,10 +1888,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a file name relative to the current working directory where log messages should be written to, in addition to the other logging facilities that are active. - .. code-block:: ini + .. tab:: toml - [pytest] - log_file = logs/pytest-logs.txt + .. code-block:: toml + + [pytest] + log_file = "logs/pytest-logs.txt" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_file = logs/pytest-logs.txt For more information, see :ref:`logging`. @@ -1580,10 +1911,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:func:`time.strftime`-compatible string that will be used when formatting dates for the logging file. - .. code-block:: ini + .. tab:: toml - [pytest] - log_file_date_format = %Y-%m-%d %H:%M:%S + .. code-block:: toml + + [pytest] + log_file_date_format = "%Y-%m-%d %H:%M:%S" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_file_date_format = %Y-%m-%d %H:%M:%S For more information, see :ref:`logging`. @@ -1593,10 +1933,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:mod:`logging`-compatible string used to format logging messages redirected to the logging file. - .. code-block:: ini + .. tab:: toml - [pytest] - log_file_format = %(asctime)s %(levelname)s %(message)s + .. code-block:: toml + + [pytest] + log_file_format = "%(asctime)s %(levelname)s %(message)s" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_file_format = %(asctime)s %(levelname)s %(message)s For more information, see :ref:`logging`. @@ -1605,12 +1954,46 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets the minimum log message level that should be captured for the logging file. The integer value or - the names of the levels can be used. + the names of the levels can be used. Note in TOML the integer must be quoted, as there is no support + for config parameters of mixed type. - .. code-block:: ini + .. tab:: toml - [pytest] - log_file_level = INFO + .. code-block:: toml + + [pytest] + log_file_level = "INFO" + log_cli_level = "10" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_file_level = INFO + log_cli_level = 10 + + For more information, see :ref:`logging`. + + +.. confval:: log_file_mode + + Sets the mode that the logging file is opened with. + The options are ``"w"`` to recreate the file (the default) or ``"a"`` to append to the file. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + log_file_mode = "a" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_file_mode = a For more information, see :ref:`logging`. @@ -1621,10 +2004,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets a :py:mod:`logging`-compatible string used to format captured logging messages. - .. code-block:: ini + .. tab:: toml - [pytest] - log_format = %(asctime)s %(levelname)s %(message)s + .. code-block:: toml + + [pytest] + log_format = "%(asctime)s %(levelname)s %(message)s" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_format = %(asctime)s %(levelname)s %(message)s For more information, see :ref:`logging`. @@ -1634,47 +2026,73 @@ passed multiple times. The expected format is ``name=value``. For example:: Sets the minimum log message level that should be captured for logging capture. The integer value or - the names of the levels can be used. + the names of the levels can be used. Note in TOML the integer must be quoted, as there is no support + for config parameters of mixed type. - .. code-block:: ini + .. tab:: toml - [pytest] - log_level = INFO + .. code-block:: toml + + [pytest] + log_level = "INFO" + log_cli_level = "10" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + log_level = INFO + log_cli_level = 10 For more information, see :ref:`logging`. .. confval:: markers - When the ``--strict-markers`` or ``--strict`` command-line arguments are used, + When the :confval:`strict_markers` configuration option is set, only known markers - defined in code by core pytest or some plugin - are allowed. You can list additional markers in this setting to add them to the whitelist, - in which case you probably want to add ``--strict-markers`` to ``addopts`` + in which case you probably want to set :confval:`strict_markers` to ``true`` to avoid future regressions: - .. code-block:: ini + .. tab:: toml - [pytest] - addopts = --strict-markers - markers = - slow - serial + .. code-block:: toml + + [pytest] + addopts = ["--strict-markers"] + markers = ["slow", "serial"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict_markers = true + markers = + slow + serial - .. note:: - The use of ``--strict-markers`` is highly preferred. ``--strict`` was kept for - backward compatibility only and may be confusing for others as it only applies to - markers and not to other options. .. confval:: minversion Specifies a minimal pytest version required for running tests. - .. code-block:: ini + .. tab:: toml - # content of pytest.ini - [pytest] - minversion = 3.0 # will fail if we run with pytest-2.8 + .. code-block:: toml + + [pytest] + minversion = 3.0 # will fail if we run with pytest-2.8 + + .. tab:: ini + + .. code-block:: ini + + [pytest] + minversion = 3.0 # will fail if we run with pytest-2.8 .. confval:: norecursedirs @@ -1694,10 +2112,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Setting a ``norecursedirs`` replaces the default. Here is an example of how to avoid certain directories: - .. code-block:: ini + .. tab:: toml - [pytest] - norecursedirs = .svn _build tmp* + .. code-block:: toml + + [pytest] + norecursedirs = [".svn", "_build", "tmp*"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + norecursedirs = .svn _build tmp* This would tell ``pytest`` to not look into typical subversion or sphinx-build directories or into any ``tmp`` prefixed directory. @@ -1705,7 +2132,7 @@ passed multiple times. The expected format is ``name=value``. For example:: Additionally, ``pytest`` will attempt to intelligently identify and ignore a virtualenv. Any directory deemed to be the root of a virtual environment will not be considered during test collection unless - ``--collect-in-virtualenv`` is given. Note also that ``norecursedirs`` + :option:`--collect-in-virtualenv` is given. Note also that ``norecursedirs`` takes precedence over ``--collect-in-virtualenv``; e.g. if you intend to run tests in a virtualenv with a base directory that matches ``'.*'`` you *must* override ``norecursedirs`` in addition to using the @@ -1720,10 +2147,19 @@ passed multiple times. The expected format is ``name=value``. For example:: class prefixed with ``Test`` as a test collection. Here is an example of how to collect tests from classes that end in ``Suite``: - .. code-block:: ini + .. tab:: toml - [pytest] - python_classes = *Suite + .. code-block:: toml + + [pytest] + python_classes = ["*Suite"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + python_classes = *Suite Note that ``unittest.TestCase`` derived classes are always collected regardless of this option, as ``unittest``'s own collection framework is used @@ -1736,20 +2172,29 @@ passed multiple times. The expected format is ``name=value``. For example:: are considered as test modules. Search for multiple glob patterns by adding a space between patterns: - .. code-block:: ini + .. tab:: toml - [pytest] - python_files = test_*.py check_*.py example_*.py + .. code-block:: toml - Or one per line: + [pytest] + python_files = ["test_*.py", "check_*.py", "example_*.py"] - .. code-block:: ini + .. tab:: ini - [pytest] - python_files = - test_*.py - check_*.py - example_*.py + .. code-block:: ini + + [pytest] + python_files = test_*.py check_*.py example_*.py + + Or one per line: + + .. code-block:: ini + + [pytest] + python_files = + test_*.py + check_*.py + example_*.py By default, files matching ``test_*.py`` and ``*_test.py`` will be considered test modules. @@ -1763,10 +2208,19 @@ passed multiple times. The expected format is ``name=value``. For example:: function prefixed with ``test`` as a test. Here is an example of how to collect test functions and methods that end in ``_test``: - .. code-block:: ini + .. tab:: toml - [pytest] - python_functions = *_test + .. code-block:: toml + + [pytest] + python_functions = ["*_test"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + python_functions = *_test Note that this has no effect on methods that live on a ``unittest.TestCase`` derived class, as ``unittest``'s own collection framework is used @@ -1784,15 +2238,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Paths are relative to the :ref:`rootdir ` directory. Directories remain in path for the duration of the test session. - .. code-block:: ini + .. tab:: toml - [pytest] - pythonpath = src1 src2 + .. code-block:: toml - .. note:: + [pytest] + pythonpath = ["src1", "src2"] + + .. tab:: ini + + .. code-block:: ini - ``pythonpath`` does not affect some imports that happen very early, - most notably plugins loaded using the ``-p`` command line option. + [pytest] + pythonpath = src1 src2 .. confval:: required_plugins @@ -1802,10 +2260,173 @@ passed multiple times. The expected format is ``name=value``. For example:: their name. Whitespace between different version specifiers is not allowed. If any one of the plugins is not found, emit an error. - .. code-block:: ini + .. tab:: toml - [pytest] - required_plugins = pytest-django>=3.0.0,<4.0.0 pytest-html pytest-xdist>=1.0.0 + .. code-block:: toml + + [pytest] + required_plugins = ["pytest-django>=3.0.0,<4.0.0", "pytest-html", "pytest-xdist>=1.0.0"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + required_plugins = pytest-django>=3.0.0,<4.0.0 pytest-html pytest-xdist>=1.0.0 + + +.. confval:: strict + + If set to ``true``, enable "strict mode", which enables the following options: + + * :confval:`strict_config` + * :confval:`strict_markers` + * :confval:`strict_parametrization_ids` + * :confval:`strict_xfail` + + Plugins may also enable their own strictness options. + + If you explicitly set an individual strictness option, it takes precedence over ``strict``. + + .. note:: + If pytest adds new strictness options in the future, they will also be enabled in strict mode. + Therefore, you should only enable strict mode if you use a pinned/locked version of pytest, + or if you want to proactively adopt new strictness options as they are added. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + strict = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict = true + + .. versionadded:: 9.0 + + +.. confval:: strict_config + + If set to ``true``, any warnings encountered while parsing the ``pytest`` section of the configuration file will raise errors. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + strict_config = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict_config = true + + You can also enable this option via the :confval:`strict` option. + + +.. confval:: strict_markers + + If set to ``true``, markers not registered in the ``markers`` section of the configuration file will raise errors. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + strict_markers = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict_markers = true + + You can also enable this option via the :confval:`strict` option. + + +.. confval:: strict_parametrization_ids + + If set to ``true``, pytest emits an error if it detects non-unique parameter set IDs. + + If not set (the default), pytest automatically handles this by adding `0`, `1`, ... to duplicate IDs, + making them unique. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + strict_parametrization_ids = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict_parametrization_ids = true + + You can also enable this option via the :confval:`strict` option. + + For example, + + .. code-block:: python + + import pytest + + + @pytest.mark.parametrize("letter", ["a", "a"]) + def test_letter_is_ascii(letter): + assert letter.isascii() + + will emit an error because both cases (parameter sets) have the same auto-generated ID "a". + + To fix the error, if you decide to keep the duplicates, explicitly assign unique IDs: + + .. code-block:: python + + import pytest + + + @pytest.mark.parametrize("letter", ["a", "a"], ids=["a0", "a1"]) + def test_letter_is_ascii(letter): + assert letter.isascii() + + See :func:`parametrize ` and :func:`pytest.param` for other ways to set IDs. + + +.. confval:: strict_xfail + + If set to ``true``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the + test suite. + For more information, see :ref:`xfail strict tutorial`. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + strict_xfail = true + + .. tab:: ini + + .. code-block:: ini + + [pytest] + strict_xfail = true + + You can also enable this option via the :confval:`strict` option. + + .. versionchanged:: 9.0 + Renamed from ``xfail_strict`` to ``strict_xfail``. + ``xfail_strict`` is accepted as an alias for ``strict_xfail``. .. confval:: testpaths @@ -1819,10 +2440,19 @@ passed multiple times. The expected format is ``name=value``. For example:: Useful when all project tests are in a known location to speed up test collection and to avoid picking up undesired tests by accident. - .. code-block:: ini + .. tab:: toml - [pytest] - testpaths = testing doc + .. code-block:: toml + + [pytest] + testpaths = ["testing", "doc"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + testpaths = testing doc This configuration means that executing: @@ -1836,18 +2466,24 @@ passed multiple times. The expected format is ``name=value``. For example:: pytest testing doc - .. confval:: tmp_path_retention_count + How many sessions should we keep the `tmp_path` directories, + according to :confval:`tmp_path_retention_policy`. + .. tab:: toml - How many sessions should we keep the `tmp_path` directories, - according to `tmp_path_retention_policy`. + .. code-block:: toml - .. code-block:: ini + [pytest] + tmp_path_retention_count = "3" - [pytest] - tmp_path_retention_count = 3 + .. tab:: ini + + .. code-block:: ini + + [pytest] + tmp_path_retention_count = 3 Default: ``3`` @@ -1863,64 +2499,172 @@ passed multiple times. The expected format is ``name=value``. For example:: * `failed`: retains directories only for tests with outcome `error` or `failed`. * `none`: directories are always removed after each test ends, regardless of the outcome. - .. code-block:: ini + .. tab:: toml - [pytest] - tmp_path_retention_policy = "all" + .. code-block:: toml + + [pytest] + tmp_path_retention_policy = "all" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + tmp_path_retention_policy = all Default: ``all`` +.. confval:: truncation_limit_chars + + Controls maximum number of characters to truncate assertion message contents. + + Setting value to ``0`` disables the character limit for truncation. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + truncation_limit_chars = 640 + + .. tab:: ini + + .. code-block:: ini + + [pytest] + truncation_limit_chars = 640 + + pytest truncates the assert messages to a certain limit by default to prevent comparison with large data to overload the console output. + + Default: ``640`` + + .. note:: + + If pytest detects it is :ref:`running on CI `, truncation is disabled automatically. + + +.. confval:: truncation_limit_lines + + Controls maximum number of lines to truncate assertion message contents. + + Setting value to ``0`` disables the lines limit for truncation. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + truncation_limit_lines = 8 + + .. tab:: ini + + .. code-block:: ini + + [pytest] + truncation_limit_lines = 8 + + pytest truncates the assert messages to a certain limit by default to prevent comparison with large data to overload the console output. + + Default: ``8`` + + .. note:: + + If pytest detects it is :ref:`running on CI `, truncation is disabled automatically. + + .. confval:: usefixtures List of fixtures that will be applied to all test functions; this is semantically the same to apply the ``@pytest.mark.usefixtures`` marker to all test functions. - .. code-block:: ini + .. tab:: toml - [pytest] - usefixtures = - clean_db + .. code-block:: toml + + [pytest] + usefixtures = ["clean_db"] + + .. tab:: ini + + .. code-block:: ini + + [pytest] + usefixtures = + clean_db .. confval:: verbosity_assertions Set a verbosity level specifically for assertion related output, overriding the application wide level. - .. code-block:: ini + .. tab:: toml - [pytest] - verbosity_assertions = 2 + .. code-block:: toml - Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of - "auto" can be used to explicitly use the global verbosity level. + [pytest] + verbosity_assertions = "2" + .. tab:: ini -.. confval:: verbosity_test_cases + .. code-block:: ini - Set a verbosity level specifically for test case execution related output, overriding the application wide level. + [pytest] + verbosity_assertions = 2 - .. code-block:: ini + If not set, defaults to application wide verbosity level (via the :option:`-v` command-line option). A special value of + ``"auto"`` can be used to explicitly use the global verbosity level. - [pytest] - verbosity_test_cases = 2 - Defaults to application wide verbosity level (via the ``-v`` command-line option). A special value of - "auto" can be used to explicitly use the global verbosity level. +.. confval:: verbosity_subtests + Set the verbosity level specifically for **passed** subtests. -.. confval:: xfail_strict + .. tab:: toml - If set to ``True``, tests marked with ``@pytest.mark.xfail`` that actually succeed will by default fail the - test suite. - For more information, see :ref:`xfail strict tutorial`. + .. code-block:: toml + [pytest] + verbosity_subtests = "1" - .. code-block:: ini + .. tab:: ini - [pytest] - xfail_strict = True + .. code-block:: ini + + [pytest] + verbosity_subtests = 1 + + A value of ``1`` or higher will show output for **passed** subtests (**failed** subtests are always reported). + Passed subtests output can be suppressed with the value ``0``, which overwrites the :option:`-v` command-line option. + + If not set, defaults to application wide verbosity level (via the :option:`-v` command-line option). A special value of + ``"auto"`` can be used to explicitly use the global verbosity level. + + See also: :ref:`subtests`. + + +.. confval:: verbosity_test_cases + + Set a verbosity level specifically for test case execution related output, overriding the application wide level. + + .. tab:: toml + + .. code-block:: toml + + [pytest] + verbosity_test_cases = "2" + + .. tab:: ini + + .. code-block:: ini + + [pytest] + verbosity_test_cases = 2 + + If not set, defaults to application wide verbosity level (via the :option:`-v` command-line option). A special value of + ``"auto"`` can be used to explicitly use the global verbosity level. .. _`command-line-flags`: @@ -1928,7 +2672,576 @@ passed multiple times. The expected format is ``name=value``. For example:: Command-line Flags ------------------ -All the command-line flags can be obtained by running ``pytest --help``:: +This section documents all command-line options provided by pytest's core plugins. + +.. note:: + + External plugins can add their own command-line options. + This reference documents only the options from pytest's core plugins. + To see all available options including those from installed plugins, run ``pytest --help``. + +Test Selection +~~~~~~~~~~~~~~ + +.. option:: -k EXPRESSION + + Only run tests which match the given substring expression. + An expression is a Python evaluable expression where all names are substring-matched against test names and their parent classes. + + Examples:: + + pytest -k "test_method or test_other" # matches names containing 'test_method' OR 'test_other' + pytest -k "not test_method" # matches names NOT containing 'test_method' + pytest -k "not test_method and not test_other" # excludes both + + The matching is case-insensitive. + Keywords are also matched to classes and functions containing extra names in their ``extra_keyword_matches`` set. + + See :ref:`select-tests` for more information and examples. + +.. option:: -m MARKEXPR + + Only run tests matching given mark expression. + Supports ``and``, ``or``, and ``not`` operators. + + Examples:: + + pytest -m slow # run tests marked with @pytest.mark.slow + pytest -m "not slow" # run tests NOT marked slow + pytest -m "mark1 and not mark2" # run tests marked mark1 but not mark2 + + See :ref:`mark` for more information on markers. + +.. option:: --markers + + Show all available markers (builtin, plugin, and per-project markers defined in configuration). + +Test Execution Control +~~~~~~~~~~~~~~~~~~~~~~~ + +.. option:: -x, --exitfirst + + Exit instantly on first error or failed test. + +.. option:: --maxfail=NUM + + Exit after first ``num`` failures or errors. + Useful for CI environments where you want to fail fast but see a few failures. + +.. option:: --last-failed, --lf + + Rerun only the tests that failed at the last run. + If no tests failed (or no cached data exists), all tests are run. + See also :confval:`cache_dir` and :ref:`cache`. + +.. option:: --failed-first, --ff + + Run all tests, but run the last failures first. + This may re-order tests and thus lead to repeated fixture setup/teardown. + +.. option:: --new-first, --nf + + Run tests from new files first, then the rest of the tests sorted by file modification time. + +.. option:: --stepwise, --sw + + Exit on test failure and continue from last failing test next time. + Useful for fixing multiple test failures one at a time. + + See :ref:`cache stepwise` for more information. + +.. option:: --stepwise-skip, --sw-skip + + Ignore the first failing test but stop on the next failing test. + Implicitly enables :option:`--stepwise`. + +.. option:: --stepwise-reset, --sw-reset + + Resets stepwise state, restarting the stepwise workflow. + Implicitly enables :option:`--stepwise`. + +.. option:: --last-failed-no-failures, --lfnf + + With :option:`--last-failed`, determines whether to execute tests when there are no previously known failures or when no cached ``lastfailed`` data was found. + + * ``all`` (default): runs the full test suite again + * ``none``: just emits a message about no known failures and exits successfully + +.. option:: --runxfail + + Report the results of xfail tests as if they were not marked. + Useful for debugging xfailed tests. + See :ref:`xfail`. + +Collection +~~~~~~~~~~ + +.. option:: --collect-only, --co + + Only collect tests, don't execute them. + Shows which tests would be collected and run. + +.. option:: --pyargs + + Try to interpret all arguments as Python packages. + Useful for running tests of installed packages:: + + pytest --pyargs pkg.testing + +.. option:: --ignore=PATH + + Ignore path during collection (multi-allowed). + Can be specified multiple times. + +.. option:: --ignore-glob=PATTERN + + Ignore path pattern during collection (multi-allowed). + Supports glob patterns. + +.. option:: --deselect=NODEID_PREFIX + + Deselect item (via node id prefix) during collection (multi-allowed). + +.. option:: --confcutdir=DIR + + Only load ``conftest.py`` files relative to specified directory. + +.. option:: --noconftest + + Don't load any ``conftest.py`` files. + +.. option:: --keep-duplicates + + Keep duplicate tests. By default, pytest removes duplicate test items. + +.. option:: --collect-in-virtualenv + + Don't ignore tests in a local virtualenv directory. + By default, pytest skips tests in virtualenv directories. + +.. option:: --continue-on-collection-errors + + Force test execution even if collection errors occur. + +.. option:: --import-mode + + Prepend/append to sys.path when importing test modules and conftest files. + + * ``prepend`` (default): prepend to sys.path + * ``append``: append to sys.path + * ``importlib``: use importlib to import test modules + + See :ref:`pythonpath` for more information. + +Fixtures +~~~~~~~~ + +.. option:: --fixtures, --funcargs + + Show available fixtures, sorted by plugin appearance. + Fixtures with leading ``_`` are only shown with :option:`--verbose`. + +.. option:: --fixtures-per-test + + Show fixtures per test. + +.. option:: --setup-only + + Only setup fixtures, do not execute tests. + See :ref:`how-to-fixtures`. + +.. option:: --setup-show + + Show setup of fixtures while executing tests. + +.. option:: --setup-plan + + Show what fixtures and tests would be executed but don't execute anything. + +Debugging +~~~~~~~~~ + +.. option:: --pdb + + Start the interactive Python debugger on errors or KeyboardInterrupt. + See :ref:`pdb-option`. + +.. option:: --pdbcls=MODULENAME:CLASSNAME + + Specify a custom interactive Python debugger for use with :option:`--pdb`. + + Example:: + + pytest --pdbcls=IPython.terminal.debugger:TerminalPdb + +.. option:: --trace + + Immediately break when running each test. + + See :ref:`trace-option` for more information. + +.. option:: --full-trace + + Don't cut any tracebacks (default is to cut). + + See :ref:`how-to-modifying-python-tb-printing` for more information. + +.. option:: --debug, --debug=DEBUG_FILE_NAME + + Store internal tracing debug information in this log file. + This file is opened with ``'w'`` and truncated as a result, care advised. + Default file name if not specified: ``pytestdebug.log``. + +.. option:: --trace-config + + Trace considerations of conftest.py files. + +Output and Reporting +~~~~~~~~~~~~~~~~~~~~ + +.. option:: -v, --verbose + + Increase verbosity. + Can be specified multiple times (e.g., ``-vv``) for even more verbose output. + + See :ref:`pytest.fine_grained_verbosity` for fine-grained control over verbosity. + +.. option:: -q, --quiet + + Decrease verbosity. + +.. option:: --verbosity=NUM + + Set verbosity level explicitly. Default: 0. + +.. option:: -r CHARS + + Show extra test summary info as specified by chars: + + * ``f``: failed + * ``E``: error + * ``s``: skipped + * ``x``: xfailed + * ``X``: xpassed + * ``p``: passed + * ``P``: passed with output + * ``a``: all except passed (p/P) + * ``A``: all + * ``w``: warnings (enabled by default) + * ``N``: resets the list + + Default: ``'fE'`` + + Examples:: + + pytest -rA # show all outcomes + pytest -rfE # show only failed and errors (default) + pytest -rfs # show failed and skipped + + See :ref:`pytest.detailed_failed_tests_usage` for more information. + +.. option:: --no-header + + Disable header. + +.. option:: --no-summary + + Disable summary. + +.. option:: --no-fold-skipped + + Do not fold skipped tests in short summary. + +.. option:: --force-short-summary + + Force condensed summary output regardless of verbosity level. + +.. option:: -l, --showlocals + + Show locals in tracebacks (disabled by default). + +.. option:: --no-showlocals + + Hide locals in tracebacks (negate :option:`--showlocals` passed through addopts). + +.. option:: --tb=STYLE + + Traceback print mode: + + * ``auto``: intelligent traceback formatting (default) + * ``long``: exhaustive, informative traceback formatting + * ``short``: shorter traceback format + * ``line``: only the failing line + * ``native``: Python's standard traceback + * ``no``: no traceback + + See :ref:`how-to-modifying-python-tb-printing` for examples. + +.. option:: --xfail-tb + + Show tracebacks for xfail (as long as :option:`--tb` != ``no``). + +.. option:: --show-capture + + Controls how captured stdout/stderr/log is shown on failed tests. + + * ``no``: don't show captured output + * ``stdout``: show captured stdout + * ``stderr``: show captured stderr + * ``log``: show captured logging + * ``all`` (default): show all captured output + +.. option:: --color=WHEN + + Color terminal output: + + * ``yes``: always use color + * ``no``: never use color + * ``auto`` (default): use color if terminal supports it + +.. option:: --code-highlight={yes,no} + + Whether code should be highlighted (only if :option:`--color` is also enabled). + Default: ``yes``. + +.. option:: --pastebin=MODE + + Send failed|all info to bpaste.net pastebin service. + +.. option:: --durations=NUM + + Show N slowest setup/test durations (N=0 for all). + See :ref:`durations`. + +.. option:: --durations-min=NUM + + Minimal duration in seconds for inclusion in slowest list. + Default: 0.005 (or 0.0 if ``-vv`` is given). + +Output Capture +~~~~~~~~~~~~~~ + +.. option:: --capture=METHOD + + Per-test capturing method: + + * ``fd``: capture at file descriptor level (default) + * ``sys``: capture at sys level + * ``no``: don't capture output + * ``tee-sys``: capture but also show output on terminal + + See :ref:`captures`. + +.. option:: -s + + Shortcut for :option:`--capture=no`. + +JUnit XML +~~~~~~~~~ + +.. option:: --junit-xml=PATH, --junitxml=PATH + + Create junit-xml style report file at given path. + +.. option:: --junit-prefix=STR, --junitprefix=STR + + Prepend prefix to classnames in junit-xml output. + +Cache +~~~~~ + +.. option:: --cache-show[=PATTERN] + + Show cache contents, don't perform collection or tests. + Default glob pattern: ``'*'``. + +.. option:: --cache-clear + + Remove all cache contents at start of test run. + See :ref:`cache`. + +Warnings +~~~~~~~~ + +.. option:: --disable-pytest-warnings, --disable-warnings + + Disable warnings summary. + +.. option:: -W WARNING, --pythonwarnings=WARNING + + Set which warnings to report, see ``-W`` option of Python itself. + Can be specified multiple times. + +Doctest +~~~~~~~ + +.. option:: --doctest-modules + + Run doctests in all .py modules. + + See :ref:`doctest` for more information on using doctests with pytest. + +.. option:: --doctest-report + + Choose another output format for diffs on doctest failure: + + * ``none`` + * ``cdiff`` + * ``ndiff`` + * ``udiff`` + * ``only_first_failure`` + +.. option:: --doctest-glob=PATTERN + + Doctests file matching pattern. + Default: ``test*.txt``. + +.. option:: --doctest-ignore-import-errors + + Ignore doctest collection errors. + +.. option:: --doctest-continue-on-failure + + For a given doctest, continue to run after the first failure. + +Configuration +~~~~~~~~~~~~~ + +.. option:: -c FILE, --config-file=FILE + + Load configuration from ``FILE`` instead of trying to locate one of the implicit configuration files. + +.. option:: --rootdir=ROOTDIR + + Define root directory for tests. + Can be relative path: ``'root_dir'``, ``'./root_dir'``, ``'root_dir/another_dir/'``; absolute path: ``'/home/user/root_dir'``; path with variables: ``'$HOME/root_dir'``. + +.. option:: --basetemp=DIR + + Base temporary directory for this test run. + Warning: this directory is removed if it exists. + + See :ref:`temporary directory location and retention` for more information. + +.. option:: -o OPTION=VALUE, --override-ini=OPTION=VALUE + + Override configuration option with ``option=value`` style. + Can be specified multiple times. + + Example:: + + pytest -o strict_xfail=true -o cache_dir=cache + +.. option:: --strict-config + + Enables the :confval:`strict_config` option. + +.. option:: --strict-markers + + Enables the :confval:`strict_markers` option. + +.. option:: --strict + + Enables the :confval:`strict` option (which enables all strictness options). + +.. option:: --assert=MODE + + Control assertion debugging tools: + + * ``plain``: performs no assertion debugging + * ``rewrite`` (default): rewrites assert statements in test modules on import to provide assert expression information + +Logging +~~~~~~~ + +See :ref:`logging` for a guide on using these flags. + +.. option:: --log-level=LEVEL + + Level of messages to catch/display. + Not set by default, so it depends on the root/parent log handler's effective level, where it is ``WARNING`` by default. + +.. option:: --log-format=FORMAT + + Log format used by the logging module. + +.. option:: --log-date-format=FORMAT + + Log date format used by the logging module. + +.. option:: --log-cli-level=LEVEL + + CLI logging level. See :ref:`live_logs`. + +.. option:: --log-cli-format=FORMAT + + Log format used by the logging module for CLI output. + +.. option:: --log-cli-date-format=FORMAT + + Log date format used by the logging module for CLI output. + +.. option:: --log-file=PATH + + Path to a file logging will be written to. + +.. option:: --log-file-mode + + Log file open mode: + + * ``w`` (default): recreate the file + * ``a``: append to the file + +.. option:: --log-file-level=LEVEL + + Log file logging level. + +.. option:: --log-file-format=FORMAT + + Log format used by the logging module for the log file. + +.. option:: --log-file-date-format=FORMAT + + Log date format used by the logging module for the log file. + +.. option:: --log-auto-indent=VALUE + + Auto-indent multiline messages passed to the logging module. + Accepts ``true|on``, ``false|off`` or an integer. + +.. option:: --log-disable=LOGGER + + Disable a logger by name. Can be passed multiple times. + +Plugin and Extension Management +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. option:: -p NAME + + Early-load given plugin module name or entry point (multi-allowed). + To avoid loading of plugins, use the ``no:`` prefix, e.g. ``no:doctest``. + See also :option:`--disable-plugin-autoload`. + +.. option:: --disable-plugin-autoload + + Disable plugin auto-loading through entry point packaging metadata. + Only plugins explicitly specified in :option:`-p` or env var :envvar:`PYTEST_PLUGINS` will be loaded. + +Version and Help +~~~~~~~~~~~~~~~~ + +.. option:: -V, --version + + Display pytest version and information about plugins. When given twice, also display information about plugins. + +.. option:: -h, --help + + Show help message and configuration info. + +Complete Help Output +~~~~~~~~~~~~~~~~~~~~ + +All the command-line flags can also be obtained by running ``pytest --help``:: $ pytest --help usage: pytest [options] [file_or_dir] [file_or_dir] [...] @@ -1939,22 +3252,27 @@ All the command-line flags can be obtained by running ``pytest --help``:: general: -k EXPRESSION Only run tests which match the given substring expression. An expression is a Python evaluable - expression where all names are substring-matched against - test names and their parent classes. Example: -k - 'test_method or test_other' matches all test functions - and classes whose name contains 'test_method' or - 'test_other', while -k 'not test_method' matches those - that don't contain 'test_method' in their names. -k 'not - test_method and not test_other' will eliminate the - matches. Additionally keywords are matched to classes - and functions containing extra names in their - 'extra_keyword_matches' set, as well as functions which - have names assigned directly to them. The matching is - case-insensitive. + expression where all names are substring-matched + against test names and their parent classes. + Example: -k 'test_method or test_other' matches all + test functions and classes whose name contains + 'test_method' or 'test_other', while -k 'not + test_method' matches those that don't contain + 'test_method' in their names. -k 'not test_method + and not test_other' will eliminate the matches. + Additionally keywords are matched to classes and + functions containing extra names in their + 'extra_keyword_matches' set, as well as functions + which have names assigned directly to them. The + matching is case-insensitive. -m MARKEXPR Only run tests matching given mark expression. For example: -m 'mark1 and not mark2'. --markers show markers (builtin, plugin and per-project ones). -x, --exitfirst Exit instantly on first error or failed test + --maxfail=num Exit after first num failures or errors + --strict-config Enables the strict_config option + --strict-markers Enables the strict_markers option + --strict Enables the strict option --fixtures, --funcargs Show available fixtures, sorted by plugin appearance (fixtures with leading '_' are only shown with '-v') @@ -1968,53 +3286,61 @@ All the command-line flags can be obtained by running ``pytest --help``:: --trace Immediately break when running each test --capture=method Per-test capturing method: one of fd|sys|no|tee-sys -s Shortcut for --capture=no - --runxfail Report the results of xfail tests as if they were not - marked - --lf, --last-failed Rerun only the tests that failed at the last run (or all - if none failed) - --ff, --failed-first Run all tests, but run the last failures first. This may - re-order tests and thus lead to repeated fixture + --runxfail Report the results of xfail tests as if they were + not marked + --lf, --last-failed Rerun only the tests that failed at the last run (or + all if none failed) + --ff, --failed-first Run all tests, but run the last failures first. This + may re-order tests and thus lead to repeated fixture setup/teardown. --nf, --new-first Run tests from new files first, then the rest of the tests sorted by file mtime --cache-show=[CACHESHOW] - Show cache contents, don't perform collection or tests. - Optional argument: glob (default: '*'). + Show cache contents, don't perform collection or + tests. Optional argument: glob (default: '*'). --cache-clear Remove all cache contents at start of test run - --lfnf={all,none}, --last-failed-no-failures={all,none} - With ``--lf``, determines whether to execute tests when - there are no previously (known) failures or when no - cached ``lastfailed`` data was found. ``all`` (the - default) runs the full test suite again. ``none`` just - emits a message about no known failures and exits - successfully. - --sw, --stepwise Exit on test failure and continue from last failing test - next time + --lfnf, --last-failed-no-failures={all,none} + With ``--lf``, determines whether to execute tests + when there are no previously (known) failures or + when no cached ``lastfailed`` data was found. + ``all`` (the default) runs the full test suite + again. ``none`` just emits a message about no known + failures and exits successfully. + --sw, --stepwise Exit on test failure and continue from last failing + test next time --sw-skip, --stepwise-skip Ignore the first failing test but stop on the next failing test. Implicitly enables --stepwise. + --sw-reset, --stepwise-reset + Resets stepwise state, restarting the stepwise + workflow. Implicitly enables --stepwise. Reporting: --durations=N Show N slowest setup/test durations (N=0 for all) --durations-min=N Minimal duration in seconds for inclusion in slowest - list. Default: 0.005. + list. Default: 0.005 (or 0.0 if -vv is given). -v, --verbose Increase verbosity --no-header Disable header --no-summary Disable summary + --no-fold-skipped Do not fold skipped tests in short summary. + --force-short-summary + Force condensed summary output regardless of + verbosity level. -q, --quiet Decrease verbosity --verbosity=VERBOSE Set verbosity. Default: 0. -r chars Show extra test summary info as specified by chars: (f)ailed, (E)rror, (s)kipped, (x)failed, (X)passed, (p)assed, (P)assed with output, (a)ll except passed - (p/P), or (A)ll. (w)arnings are enabled by default (see - --disable-warnings), 'N' can be used to reset the list. - (default: 'fE'). + (p/P), or (A)ll. (w)arnings are enabled by default + (see --disable-warnings), 'N' can be used to reset + the list. (default: 'fE'). --disable-warnings, --disable-pytest-warnings Disable warnings summary -l, --showlocals Show locals in tracebacks (disabled by default) - --no-showlocals Hide locals in tracebacks (negate --showlocals passed - through addopts) - --tb=style Traceback print mode (auto/long/short/line/native/no) + --no-showlocals Hide locals in tracebacks (negate --showlocals + passed through addopts) + --tb=style Traceback print mode + (auto/long/short/line/native/no) --xfail-tb Show tracebacks for xfail (as long as --tb != no) --show-capture={no,stdout,stderr,log,all} Controls how captured stdout/stderr/log is shown on @@ -2022,37 +3348,25 @@ All the command-line flags can be obtained by running ``pytest --help``:: --full-trace Don't cut any tracebacks (default is to cut) --color=color Color terminal output (yes/no/auto) --code-highlight={yes,no} - Whether code should be highlighted (only if --color is - also enabled). Default: yes. + Whether code should be highlighted (only if --color + is also enabled). Default: yes. --pastebin=mode Send failed|all info to bpaste.net pastebin service - --junit-xml=path Create junit-xml style report file at given path - --junit-prefix=str Prepend prefix to classnames in junit-xml output + --junitxml, --junit-xml=path + Create junit-xml style report file at given path + --junitprefix, --junit-prefix=str + Prepend prefix to classnames in junit-xml output pytest-warnings: - -W PYTHONWARNINGS, --pythonwarnings=PYTHONWARNINGS - Set which warnings to report, see -W option of Python - itself - --maxfail=num Exit after first num failures or errors - --strict-config Any warnings encountered while parsing the `pytest` - section of the configuration file raise errors - --strict-markers Markers not registered in the `markers` section of the - configuration file raise errors - --strict (Deprecated) alias to --strict-markers - -c FILE, --config-file=FILE - Load configuration from `FILE` instead of trying to - locate one of the implicit configuration files. - --continue-on-collection-errors - Force test execution even if collection errors occur - --rootdir=ROOTDIR Define root directory for tests. Can be relative path: - 'root_dir', './root_dir', 'root_dir/another_dir/'; - absolute path: '/home/user/root_dir'; path with - variables: '$HOME/root_dir'. + -W, --pythonwarnings PYTHONWARNINGS + Set which warnings to report, see -W option of + Python itself collection: --collect-only, --co Only collect tests, don't execute them --pyargs Try to interpret all arguments as Python packages --ignore=path Ignore path during collection (multi-allowed) - --ignore-glob=path Ignore path pattern during collection (multi-allowed) + --ignore-glob=path Ignore path pattern during collection (multi- + allowed) --deselect=nodeid_prefix Deselect item (via node id prefix) during collection (multi-allowed) @@ -2061,9 +3375,11 @@ All the command-line flags can be obtained by running ``pytest --help``:: --keep-duplicates Keep duplicate tests --collect-in-virtualenv Don't ignore tests in a local virtualenv directory + --continue-on-collection-errors + Force test execution even if collection errors occur --import-mode={prepend,append,importlib} - Prepend/append to sys.path when importing test modules - and conftest files. Default: prepend. + Prepend/append to sys.path when importing test + modules and conftest files. Default: prepend. --doctest-modules Run doctests in all .py modules --doctest-report={none,cdiff,ndiff,udiff,only_first_failure} Choose another output format for diffs on doctest @@ -2076,37 +3392,53 @@ All the command-line flags can be obtained by running ``pytest --help``:: failure test session debugging and configuration: - --basetemp=dir Base temporary directory for this test run. (Warning: - this directory is removed if it exists.) - -V, --version Display pytest version and information about plugins. - When given twice, also display information about - plugins. + -c, --config-file FILE + Load configuration from `FILE` instead of trying to + locate one of the implicit configuration files. + --rootdir=ROOTDIR Define root directory for tests. Can be relative + path: 'root_dir', './root_dir', + 'root_dir/another_dir/'; absolute path: + '/home/user/root_dir'; path with variables: + '$HOME/root_dir'. + --basetemp=dir Base temporary directory for this test run. + (Warning: this directory is removed if it exists.) + -V, --version Display pytest version and information about + plugins. When given twice, also display information + about plugins. -h, --help Show help message and configuration info -p name Early-load given plugin module name or entry point - (multi-allowed). To avoid loading of plugins, use the - `no:` prefix, e.g. `no:doctest`. + (multi-allowed). To avoid loading of plugins, use + the `no:` prefix, e.g. `no:doctest`. See also + --disable-plugin-autoload. + --disable-plugin-autoload + Disable plugin auto-loading through entry point + packaging metadata. Only plugins explicitly + specified in -p or env var PYTEST_PLUGINS will be + loaded. --trace-config Trace considerations of conftest.py files --debug=[DEBUG_FILE_NAME] Store internal tracing debug information in this log - file. This file is opened with 'w' and truncated as a - result, care advised. Default: pytestdebug.log. - -o OVERRIDE_INI, --override-ini=OVERRIDE_INI - Override ini option with "option=value" style, e.g. `-o - xfail_strict=True -o cache_dir=cache`. + file. This file is opened with 'w' and truncated as + a result, care advised. Default: pytestdebug.log. + -o, --override-ini OVERRIDE_INI + Override configuration option with "option=value" + style, e.g. `-o strict_xfail=True -o + cache_dir=cache`. --assert=MODE Control assertion debugging tools. 'plain' performs no assertion debugging. - 'rewrite' (the default) rewrites assert statements in - test modules on import to provide assert expression - information. + 'rewrite' (the default) rewrites assert statements + in test modules on import to provide assert + expression information. --setup-only Only setup fixtures, do not execute tests --setup-show Show setup of fixtures while executing tests - --setup-plan Show what fixtures and tests would be executed but don't - execute anything + --setup-plan Show what fixtures and tests would be executed but + don't execute anything logging: - --log-level=LEVEL Level of messages to catch/display. Not set by default, - so it depends on the root/parent log handler's effective - level, where it is "WARNING" by default. + --log-level=LEVEL Level of messages to catch/display. Not set by + default, so it depends on the root/parent log + handler's effective level, where it is "WARNING" by + default. --log-format=LOG_FORMAT Log format used by the logging module --log-date-format=LOG_DATE_FORMAT @@ -2130,54 +3462,68 @@ All the command-line flags can be obtained by running ``pytest --help``:: Auto-indent multiline messages passed to the logging module. Accepts true|on, false|off or an integer. --log-disable=LOGGER_DISABLE - Disable a logger by name. Can be passed multiple times. + Disable a logger by name. Can be passed multiple + times. - Custom options: - --lsof Run FD checks if lsof is available - --runpytest={inprocess,subprocess} - Run pytest sub runs in tests using an 'inprocess' or - 'subprocess' (python -m main) method - - [pytest] ini-options in the first pytest.ini|tox.ini|setup.cfg|pyproject.toml file found: + [pytest] configuration options in the first pytest.toml|pytest.ini|tox.ini|setup.cfg|pyproject.toml file found: markers (linelist): Register new markers for test functions empty_parameter_set_mark (string): Default marker for empty parametersets - norecursedirs (args): Directory patterns to avoid for recursion - testpaths (args): Directories to search for tests when no files or - directories are given on the command line + strict_config (bool): Any warnings encountered while parsing the `pytest` + section of the configuration file raise errors + strict_markers (bool): + Markers not registered in the `markers` section of + the configuration file raise errors + strict (bool): Enables all strictness options, currently: + strict_config, strict_markers, strict_xfail, + strict_parametrization_ids filterwarnings (linelist): Each line specifies a pattern for warnings.filterwarnings. Processed after -W/--pythonwarnings. + norecursedirs (args): Directory patterns to avoid for recursion + testpaths (args): Directories to search for tests when no files or + directories are given on the command line + collect_imported_tests (bool): + Whether to collect tests in imported modules outside + `testpaths` consider_namespace_packages (bool): - Consider namespace packages when resolving module names - during import - usefixtures (args): List of default fixtures to be used with this project + Consider namespace packages when resolving module + names during import + usefixtures (args): List of default fixtures to be used with this + project python_files (args): Glob-style file patterns for Python test module discovery python_classes (args): - Prefixes or glob names for Python test class discovery + Prefixes or glob names for Python test class + discovery python_functions (args): Prefixes or glob names for Python test function and method discovery disable_test_id_escaping_and_forfeit_all_rights_to_community_support (bool): - Disable string escape non-ASCII characters, might cause - unwanted side effects(use at your own risk) + Disable string escape non-ASCII characters, might + cause unwanted side effects(use at your own risk) + strict_parametrization_ids (bool): + Emit an error if non-unique parameter set IDs are + detected console_output_style (string): - Console output: "classic", or with additional progress - information ("progress" (percentage) | "count" | - "progress-even-when-capture-no" (forces progress even - when capture=no) + Console output: "classic", or with additional + progress information ("progress" (percentage) | + "count" | "progress-even-when-capture-no" (forces + progress even when capture=no) verbosity_test_cases (string): Specify a verbosity level for test case execution, - overriding the main level. Higher levels will provide - more detailed information about each test case executed. - xfail_strict (bool): Default for the strict parameter of xfail markers when - not given explicitly (default: False) + overriding the main level. Higher levels will + provide more detailed information about each test + case executed. + strict_xfail (bool): Default for the strict parameter of xfail markers + when not given explicitly (default: False) (alias: + xfail_strict) tmp_path_retention_count (string): How many sessions should we keep the `tmp_path` - directories, according to `tmp_path_retention_policy`. + directories, according to + `tmp_path_retention_policy`. tmp_path_retention_policy (string): Controls which directories created by the `tmp_path` fixture are kept around, based on test outcome. @@ -2185,10 +3531,16 @@ All the command-line flags can be obtained by running ``pytest --help``:: enable_assertion_pass_hook (bool): Enables the pytest_assertion_pass hook. Make sure to delete any previously generated pyc cache files. + truncation_limit_lines (string): + Set threshold of LINES after which truncation will + take effect + truncation_limit_chars (string): + Set threshold of CHARS after which truncation will + take effect verbosity_assertions (string): - Specify a verbosity level for assertions, overriding the - main level. Higher levels will provide more detailed - explanation when an assertion fails. + Specify a verbosity level for assertions, overriding + the main level. Higher levels will provide more + detailed explanation when an assertion fails. junit_suite_name (string): Test suite name for JUnit report junit_logging (string): @@ -2210,8 +3562,8 @@ All the command-line flags can be obtained by running ``pytest --help``:: log_format (string): Default value for --log-format log_date_format (string): Default value for --log-date-format - log_cli (bool): Enable log display during test run (also known as "live - logging") + log_cli (bool): Enable log display during test run (also known as + "live logging") log_cli_level (string): Default value for --log-cli-level log_cli_format (string): @@ -2229,24 +3581,32 @@ All the command-line flags can be obtained by running ``pytest --help``:: Default value for --log-file-date-format log_auto_indent (string): Default value for --log-auto-indent - pythonpath (paths): Add paths to sys.path faulthandler_timeout (string): - Dump the traceback of all threads if a test takes more - than TIMEOUT seconds to finish + Dump the traceback of all threads if a test takes + more than TIMEOUT seconds to finish + faulthandler_exit_on_timeout (bool): + Exit the test process if a test takes more than + faulthandler_timeout seconds to finish + verbosity_subtests (string): + Specify verbosity level for subtests. Higher levels + will generate output for passed subtests. Failed + subtests are always reported. addopts (args): Extra command line options minversion (string): Minimally required pytest version + pythonpath (paths): Add paths to sys.path required_plugins (args): Plugins that must be present for pytest to run - pytester_example_dir (string): - Directory to take the pytester example files from Environment variables: - CI When set (regardless of value), pytest knows it is running in a CI process and does not truncate summary info - BUILD_NUMBER equivalent to CI + CI When set to a non-empty value, pytest knows it is running in a CI process and does not truncate summary info + BUILD_NUMBER Equivalent to CI PYTEST_ADDOPTS Extra command line options PYTEST_PLUGINS Comma-separated plugins to load during startup PYTEST_DISABLE_PLUGIN_AUTOLOAD Set to disable plugin auto-loading PYTEST_DEBUG Set to enable debug tracing of pytest's internals + PYTEST_DEBUG_TEMPROOT Override the system temporary directory + PYTEST_THEME The Pygments style to use for code output + PYTEST_THEME_MODE Set the PYTEST_THEME to be either 'dark' or 'light' to see available markers type: pytest --markers diff --git a/doc/en/requirements.txt b/doc/en/requirements.txt index 0637c967b8a..d672a9d7e15 100644 --- a/doc/en/requirements.txt +++ b/doc/en/requirements.txt @@ -1,13 +1,12 @@ +-c broken-dep-constraints.txt pluggy>=1.5.0 -pygments-pytest>=2.3.0 +pygments-pytest>=2.5.0 sphinx-removed-in>=0.2.0 -sphinx>=7 +# Pinning to <9.0 due to https://github.com/python-trio/sphinxcontrib-trio/issues/399. +sphinx>=7,<9.0 sphinxcontrib-trio sphinxcontrib-svg2pdfconverter -# Pin packaging because it no longer handles 'latest' version, which -# is the version that is assigned to the docs. -# See https://github.com/pytest-dev/pytest/pull/10578#issuecomment-1348249045. -packaging furo sphinxcontrib-towncrier sphinx-issues +sphinx-inline-tabs diff --git a/doc/en/sponsor.rst b/doc/en/sponsor.rst index 8362a7f0a3a..6ad722be94c 100644 --- a/doc/en/sponsor.rst +++ b/doc/en/sponsor.rst @@ -2,7 +2,7 @@ Sponsor ======= pytest is maintained by a team of volunteers from all around the world in their free time. While -we work on pytest because we love the project and use it daily at our daily jobs, monetary +we work on pytest because we love the project and use it daily in our jobs, monetary compensation when possible is welcome to justify time away from friends, family and personal time. Money is also used to fund local sprints, merchandising (stickers to distribute in conferences for example) @@ -12,7 +12,7 @@ OpenCollective -------------- `Open Collective`_ is an online funding platform for open and transparent communities. -It provide tools to raise money and share your finances in full transparency. +It provides tools to raise money and share your finances in full transparency. It is the platform of choice for individuals and companies that want to make one-time or monthly donations directly to the project. diff --git a/doc/en/talks.rst b/doc/en/talks.rst index b9b153a792e..a45c05c6f2f 100644 --- a/doc/en/talks.rst +++ b/doc/en/talks.rst @@ -17,19 +17,19 @@ Books Talks and blog postings --------------------------------------------- -- Training: `pytest - simple, rapid and fun testing with Python `_, Florian Bruhin, PyConDE 2022 +- Training: `pytest - simple, rapid and fun testing with Python `_, Freya Bruhin, PyConDE 2022 -- `pytest: Simple, rapid and fun testing with Python, `_ (@ 4:22:32), Florian Bruhin, WeAreDevelopers World Congress 2021 +- `pytest: Simple, rapid and fun testing with Python, `_ (@ 4:22:32), Freya Bruhin, WeAreDevelopers World Congress 2021 -- Webinar: `pytest: Test Driven Development für Python (German) `_, Florian Bruhin, via mylearning.ch, 2020 +- Webinar: `pytest: Test Driven Development für Python (German) `_, Freya Bruhin, via mylearning.ch, 2020 - Webinar: `Simplify Your Tests with Fixtures `_, Oliver Bestwalter, via JetBrains, 2020 -- Training: `Introduction to pytest - simple, rapid and fun testing with Python `_, Florian Bruhin, PyConDE 2019 +- Training: `Introduction to pytest - simple, rapid and fun testing with Python `_, Freya Bruhin, PyConDE 2019 - Abridged metaprogramming classics - this episode: pytest, Oliver Bestwalter, PyConDE 2019 (`repository `__, `recording `__) -- Testing PySide/PyQt code easily using the pytest framework, Florian Bruhin, Qt World Summit 2019 (`slides `__, `recording `__) +- Testing PySide/PyQt code easily using the pytest framework, Freya Bruhin, Qt World Summit 2019 (`slides `__, `recording `__) - `pytest: recommendations, basic packages for testing in Python and Django, Andreu Vallbona, PyBCN June 2019 `_. @@ -41,7 +41,7 @@ Talks and blog postings - `Pythonic testing, Igor Starikov (Russian, PyNsk, November 2016) `_. -- `pytest - Rapid Simple Testing, Florian Bruhin, Swiss Python Summit 2016 +- `pytest - Rapid Simple Testing, Freya Bruhin, Swiss Python Summit 2016 `_. - `Improve your testing with Pytest and Mock, Gabe Hollombe, PyCon SG 2015 diff --git a/pyproject.toml b/pyproject.toml index f3eba4a08a8..31b8a029ec5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] build-backend = "setuptools.build_meta" requires = [ - "setuptools>=61", + "setuptools>=77", "setuptools-scm[toml]>=6.2.3", ] @@ -13,31 +13,31 @@ keywords = [ "test", "unittest", ] -license = { text = "MIT" } +license = "MIT" +license-files = [ "LICENSE" ] authors = [ { name = "Holger Krekel" }, { name = "Bruno Oliveira" }, { name = "Ronny Pfannschmidt" }, { name = "Floris Bruynooghe" }, { name = "Brianna Laugher" }, - { name = "Florian Bruhin" }, + { name = "Freya Bruhin" }, { name = "Others (See AUTHORS)" }, ] -requires-python = ">=3.8" +requires-python = ">=3.10" classifiers = [ "Development Status :: 6 - Mature", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Operating System :: Unix", "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Topic :: Software Development :: Libraries", "Topic :: Software Development :: Testing", "Topic :: Utilities", @@ -46,11 +46,12 @@ dynamic = [ "version", ] dependencies = [ - "colorama; sys_platform=='win32'", - "exceptiongroup>=1.0.0rc8; python_version<'3.11'", - "iniconfig", - "packaging", - "pluggy<2,>=1.5", + "colorama>=0.4; sys_platform=='win32'", + "exceptiongroup>=1; python_version<'3.11'", + "iniconfig>=1.0.1", + "packaging>=22", + "pluggy>=1.5,<2", + "pygments>=2.7.2", "tomli>=1; python_version<'3.11'", ] optional-dependencies.dev = [ @@ -58,16 +59,16 @@ optional-dependencies.dev = [ "attrs>=19.2", "hypothesis>=3.56", "mock", - "pygments>=2.7.2", "requests", "setuptools", "xmlschema", ] urls.Changelog = "https://docs.pytest.org/en/stable/changelog.html" +urls.Contact = "https://docs.pytest.org/en/stable/contact.html" +urls.Funding = "https://docs.pytest.org/en/stable/sponsor.html" urls.Homepage = "https://docs.pytest.org/en/latest/" urls.Source = "https://github.com/pytest-dev/pytest" urls.Tracker = "https://github.com/pytest-dev/pytest/issues" -urls.Twitter = "https://twitter.com/pytestdotorg" scripts."py.test" = "pytest:console_main" scripts.pytest = "pytest:console_main" @@ -83,11 +84,11 @@ scripts.pytest = "pytest:console_main" write_to = "src/_pytest/_version.py" [tool.black] -target-version = [ - 'py38', -] +# See https://black.readthedocs.io/en/stable/usage_and_configuration/the_basics.html#t-target-version +target-version = [ "py310", "py311", "py312", "py313" ] [tool.ruff] +target-version = "py310" line-length = 88 src = [ "src", @@ -143,6 +144,7 @@ lint.ignore = [ # pylint ignore "PLC0105", # `TypeVar` name "E" does not reflect its covariance; "PLC0414", # Import alias does not rename original package + "PLC0415", # import should be at top level of package "PLR0124", # Name compared with itself "PLR0133", # Two constants compared in a comparison (lots of those in tests) "PLR0402", # Use `from x.y import z` in lieu of alias @@ -155,6 +157,7 @@ lint.ignore = [ "PLR5501", # Use `elif` instead of `else` then `if` "PLW0120", # remove the else and dedent its contents "PLW0603", # Using the global statement + "PLW1641", # Does not implement the __hash__ method "PLW2901", # for loop variable overwritten by assignment target # ruff ignore "RUF012", # Mutable class attributes should be annotated with `typing.ClassVar` @@ -166,6 +169,10 @@ lint.per-file-ignores."src/_pytest/_py/**/*.py" = [ lint.per-file-ignores."src/_pytest/_version.py" = [ "I001", ] +# can't be disabled on a line-by-line basis in file +lint.per-file-ignores."testing/code/test_source.py" = [ + "F841", +] lint.per-file-ignores."testing/python/approx.py" = [ "B015", ] @@ -199,7 +206,9 @@ disable = [ "arguments-renamed", "assigning-non-slot", "attribute-defined-outside-init", + "bad-builtin", "bad-classmethod-argument", + "bad-dunder-name", "bad-mcs-method-argument", "broad-exception-caught", "broad-exception-raised", @@ -208,25 +217,38 @@ disable = [ "comparison-with-callable", "comparison-with-itself", # PLR0124 from ruff "condition-evals-to-constant", + "consider-alternative-union-syntax", + "confusing-consecutive-elif", + "consider-using-assignment-expr", "consider-using-dict-items", "consider-using-from-import", "consider-using-f-string", "consider-using-in", + "consider-using-namedtuple-or-dataclass", "consider-using-ternary", + "consider-using-tuple", "consider-using-with", "consider-using-from-import", # not activated by default, PLR0402 disabled in ruff + "consider-ternary-expression", "cyclic-import", + "differing-param-doc", + "docstring-first-line-empty", + "deprecated-argument", + "deprecated-attribute", + "deprecated-class", "disallowed-name", # foo / bar are used often in tests "duplicate-code", "else-if-used", # not activated by default, PLR5501 disabled in ruff "empty-comment", # not activated by default, PLR2044 disabled in ruff "eval-used", + "eq-without-hash", # PLW1641 disabled in ruff "exec-used", "expression-not-assigned", "fixme", "global-statement", # PLW0603 disabled in ruff "import-error", - "import-outside-toplevel", + "import-outside-toplevel", # PLC0415 disabled in ruff + "import-private-name", "inconsistent-return-statements", "invalid-bool-returned", "invalid-name", @@ -237,8 +259,12 @@ disable = [ "magic-value-comparison", # not activated by default, PLR2004 disabled in ruff "method-hidden", "missing-docstring", + "missing-param-doc", + "missing-raises-doc", "missing-timeout", + "missing-type-doc", "misplaced-bare-raise", # PLE0704 from ruff + "misplaced-comparison-constant", "multiple-statements", # multiple-statements-on-one-line-colon (E701) from ruff "no-else-break", "no-else-continue", @@ -247,6 +273,7 @@ disable = [ "no-member", "no-name-in-module", "no-self-argument", + "no-self-use", "not-an-iterable", "not-callable", "pointless-exception-statement", # https://github.com/pytest-dev/pytest/pull/12379 @@ -259,12 +286,14 @@ disable = [ "redefined-builtin", "redefined-loop-name", # PLW2901 disabled in ruff "redefined-outer-name", + "redefined-variable-type", "reimported", "simplifiable-condition", "simplifiable-if-expression", "singleton-comparison", "superfluous-parens", "super-init-not-called", + "too-complex", "too-few-public-methods", "too-many-ancestors", "too-many-arguments", # disabled in ruff @@ -274,9 +303,11 @@ disable = [ "too-many-lines", "too-many-locals", "too-many-nested-blocks", + "too-many-positional-arguments", "too-many-public-methods", "too-many-return-statements", # disabled in ruff "too-many-statements", # disabled in ruff + "too-many-try-statements", "try-except-raise", "typevar-name-incorrect-variance", # PLC0105 disabled in ruff "unbalanced-tuple-unpacking", @@ -298,14 +329,21 @@ disable = [ "use-dict-literal", "use-implicit-booleaness-not-comparison", "use-implicit-booleaness-not-len", + "use-set-for-membership", "useless-else-on-loop", # PLC0414 disabled in ruff "useless-import-alias", "useless-return", "using-constant-test", + "while-used", "wrong-import-order", # handled by isort / ruff "wrong-import-position", # handled by isort / ruff ] +[tool.codespell] +ignore-words-list = "afile,asend,asser,assertio,feld,hove,ned,noes,notin,paramete,parth,tesults,varius,wil" +skip = "AUTHORS,*/plugin_list.rst" +write-changes = true + [tool.check-wheel-contents] # check-wheel-contents is executed by the build-and-inspect-python-package action. # W009: Wheel contains multiple toplevel library entries @@ -313,10 +351,11 @@ ignore = "W009" [tool.pyproject-fmt] indent = 4 +max_supported_python = "3.14" -[tool.pytest.ini_options] +[tool.pytest] minversion = "2.0" -addopts = "-rfEX -p pytester --strict-markers" +addopts = [ "-rfEX", "-p", "pytester" ] python_files = [ "test_*.py", "*_test.py", @@ -339,7 +378,7 @@ norecursedirs = [ "build", "dist", ] -xfail_strict = true +strict = true filterwarnings = [ "error", "default:Using or importing the ABCs:DeprecationWarning:unittest2.*", @@ -371,6 +410,9 @@ filterwarnings = [ "ignore:VendorImporter\\.find_spec\\(\\) not found; falling back to find_module\\(\\):ImportWarning", # https://github.com/pytest-dev/execnet/pull/127 "ignore:isSet\\(\\) is deprecated, use is_set\\(\\) instead:DeprecationWarning", + # https://github.com/pytest-dev/pytest/issues/2366 + # https://github.com/pytest-dev/pytest/pull/13057 + "default::pytest.PytestFDWarning", ] pytester_example_dir = "testing/example_scripts" markers = [ @@ -385,6 +427,9 @@ markers = [ "slow", # experimental mark for all tests using pexpect "uses_pexpect", + # Disables the `remove_ci_env_var` autouse fixture on a given test that + # actually inspects whether the CI environment variable is set. + "keep_ci_var", ] [tool.towncrier] @@ -474,6 +519,7 @@ files = [ mypy_path = [ "src", ] +python_version = "3.10" check_untyped_defs = true disallow_any_generics = true disallow_untyped_defs = true @@ -486,3 +532,17 @@ warn_unreachable = true warn_unused_configs = true no_implicit_reexport = true warn_unused_ignores = true + +[tool.pyright] +include = [ + "src", + "testing", + "scripts", +] +extraPaths = [ + "src", +] +pythonVersion = "3.10" +typeCheckingMode = "basic" +reportMissingImports = "none" +reportMissingModuleSource = "none" diff --git a/scripts/generate-gh-release-notes.py b/scripts/generate-gh-release-notes.py index 7f195ba1e0a..d293a3bb695 100644 --- a/scripts/generate-gh-release-notes.py +++ b/scripts/generate-gh-release-notes.py @@ -11,10 +11,10 @@ from __future__ import annotations +from collections.abc import Sequence from pathlib import Path import re import sys -from typing import Sequence import pypandoc @@ -43,7 +43,7 @@ def extract_changelog_entries_for(version: str) -> str: def convert_rst_to_md(text: str) -> str: result = pypandoc.convert_text( - text, "md", format="rst", extra_args=["--wrap=preserve"] + text, "gfm", format="rst", extra_args=["--wrap=preserve"] ) assert isinstance(result, str), repr(result) return result diff --git a/scripts/prepare-release-pr.py b/scripts/prepare-release-pr.py index 49cb2110639..eb4f19f8386 100644 --- a/scripts/prepare-release-pr.py +++ b/scripts/prepare-release-pr.py @@ -10,8 +10,7 @@ After that, it will create a release using the `release` tox environment, and push a new PR. -**Token**: currently the token from the GitHub Actions is used, pushed with -`pytest bot ` commit author. +Note: the script uses the `gh` command-line tool, so `GH_TOKEN` must be set in the environment. """ from __future__ import annotations @@ -25,7 +24,6 @@ from colorama import Fore from colorama import init -from github3.repos import Repository class InvalidFeatureRelease(Exception): @@ -54,17 +52,7 @@ class InvalidFeatureRelease(Exception): """ -def login(token: str) -> Repository: - import github3 - - github = github3.login(token=token) - owner, repo = SLUG.split("/") - return github.repository(owner, repo) - - -def prepare_release_pr( - base_branch: str, is_major: bool, token: str, prerelease: str -) -> None: +def prepare_release_pr(base_branch: str, is_major: bool, prerelease: str) -> None: print() print(f"Processing release for branch {Fore.CYAN}{base_branch}") @@ -131,22 +119,26 @@ def prepare_release_pr( check=True, ) - oauth_url = f"https://{token}:x-oauth-basic@github.com/{SLUG}.git" run( - ["git", "push", oauth_url, f"HEAD:{release_branch}", "--force"], + ["git", "push", "origin", f"HEAD:{release_branch}", "--force"], check=True, ) print(f"Branch {Fore.CYAN}{release_branch}{Fore.RESET} pushed.") body = PR_BODY.format(version=version) - repo = login(token) - pr = repo.create_pull( - f"Prepare release {version}", - base=base_branch, - head=release_branch, - body=body, + run( + [ + "gh", + "pr", + "create", + f"--base={base_branch}", + f"--head={release_branch}", + f"--title=Release {version}", + f"--body={body}", + "--draft", + ], + check=True, ) - print(f"Pull request {Fore.CYAN}{pr.url}{Fore.RESET} created.") def find_next_version( @@ -163,7 +155,7 @@ def find_next_version( last_version = valid_versions[-1] if is_major: - return f"{last_version[0]+1}.0.0{prerelease}" + return f"{last_version[0] + 1}.0.0{prerelease}" elif is_feature_release: return f"{last_version[0]}.{last_version[1] + 1}.0{prerelease}" else: @@ -174,14 +166,12 @@ def main() -> None: init(autoreset=True) parser = argparse.ArgumentParser() parser.add_argument("base_branch") - parser.add_argument("token") parser.add_argument("--major", action="store_true", default=False) parser.add_argument("--prerelease", default="") options = parser.parse_args() prepare_release_pr( base_branch=options.base_branch, is_major=options.major, - token=options.token, prerelease=options.prerelease, ) diff --git a/scripts/release.patch.rst b/scripts/release.patch.rst index 59fbe50ce0e..120cae51702 100644 --- a/scripts/release.patch.rst +++ b/scripts/release.patch.rst @@ -3,9 +3,7 @@ pytest-{version} pytest {version} has just been released to PyPI. -This is a bug-fix release, being a drop-in replacement. To upgrade:: - - pip install --upgrade pytest +This is a bug-fix release, being a drop-in replacement. The full changelog is available at https://docs.pytest.org/en/stable/changelog.html. diff --git a/scripts/release.py b/scripts/release.py index 545919cd60b..6549cd00a3d 100644 --- a/scripts/release.py +++ b/scripts/release.py @@ -6,6 +6,7 @@ import argparse import os from pathlib import Path +import re from subprocess import call from subprocess import check_call from subprocess import check_output @@ -16,17 +17,27 @@ def announce(version: str, template_name: str, doc_version: str) -> None: """Generates a new release announcement entry in the docs.""" - # Get our list of authors + # Get our list of authors and co-authors. stdout = check_output(["git", "describe", "--abbrev=0", "--tags"], encoding="UTF-8") last_version = stdout.strip() + rev_range = f"{last_version}..HEAD" - stdout = check_output( - ["git", "log", f"{last_version}..HEAD", "--format=%aN"], encoding="UTF-8" + authors = check_output( + ["git", "log", rev_range, "--format=%aN"], encoding="UTF-8" + ).splitlines() + + co_authors_output = check_output( + ["git", "log", rev_range, "--format=%(trailers:key=Co-authored-by) "], + encoding="UTF-8", ) + co_authors: list[str] = [] + for co_author_line in co_authors_output.splitlines(): + if m := re.search(r"Co-authored-by: (.+?)<", co_author_line): + co_authors.append(m.group(1).strip()) contributors = { name - for name in stdout.splitlines() + for name in authors + co_authors if not name.endswith("[bot]") and name != "pytest bot" } @@ -110,7 +121,7 @@ def pre_release( def changelog(version: str, write_out: bool = False) -> None: addopts = [] if write_out else ["--draft"] - check_call(["towncrier", "--yes", "--version", version, *addopts]) + check_call(["towncrier", "build", "--yes", "--version", version, *addopts]) def main() -> None: diff --git a/scripts/update-plugin-list.py b/scripts/update-plugin-list.py index 75df0ddba40..be57d436966 100644 --- a/scripts/update-plugin-list.py +++ b/scripts/update-plugin-list.py @@ -1,14 +1,14 @@ # mypy: disallow-untyped-defs from __future__ import annotations +from collections.abc import Iterable +from collections.abc import Iterator import datetime import pathlib import re from textwrap import dedent from textwrap import indent from typing import Any -from typing import Iterable -from typing import Iterator from typing import TypedDict import packaging.version @@ -30,7 +30,7 @@ Pytest Plugin List ================== -Below is an automated compilation of ``pytest``` plugins available on `PyPI `_. +Below is an automated compilation of ``pytest`` plugins available on `PyPI `_. It includes PyPI projects whose names begin with ``pytest-`` or ``pytest_`` and a handful of manually selected projects. Packages classified as inactive are excluded. @@ -66,6 +66,8 @@ "logot", "nuts", "flask_fixture", + "databricks-labs-pytester", + "tursu", } @@ -175,7 +177,7 @@ def version_sort_key(version_string: str) -> Any: ) last_release = release_date.strftime("%b %d, %Y") break - name = f':pypi:`{info["name"]}`' + name = f":pypi:`{info['name']}`" summary = "" if info["summary"]: summary = escape_rst(info["summary"].replace("\n", "")) @@ -193,7 +195,7 @@ def plugin_definitions(plugins: Iterable[PluginInfo]) -> Iterator[str]: for plugin in plugins: yield dedent( f""" - {plugin['name']} + {plugin["name"]} *last release*: {plugin["last_release"]}, *status*: {plugin["status"]}, *requires*: {plugin["requires"]} diff --git a/src/_pytest/_code/__init__.py b/src/_pytest/_code/__init__.py index 0bfde42604d..7f67a2e3e0a 100644 --- a/src/_pytest/_code/__init__.py +++ b/src/_pytest/_code/__init__.py @@ -16,11 +16,11 @@ __all__ = [ "Code", "ExceptionInfo", - "filter_traceback", "Frame", - "getfslineno", - "getrawcode", + "Source", "Traceback", "TracebackEntry", - "Source", + "filter_traceback", + "getfslineno", + "getrawcode", ] diff --git a/src/_pytest/_code/code.py b/src/_pytest/_code/code.py index e7452825756..4cf99a77340 100644 --- a/src/_pytest/_code/code.py +++ b/src/_pytest/_code/code.py @@ -2,6 +2,10 @@ from __future__ import annotations import ast +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Mapping +from collections.abc import Sequence import dataclasses import inspect from inspect import CO_VARARGS @@ -11,29 +15,23 @@ from pathlib import Path import re import sys -import traceback +from traceback import extract_tb +from traceback import format_exception from traceback import format_exception_only +from traceback import FrameSummary from types import CodeType from types import FrameType from types import TracebackType from typing import Any -from typing import Callable from typing import ClassVar from typing import Final from typing import final from typing import Generic -from typing import Iterable -from typing import List from typing import Literal -from typing import Mapping from typing import overload -from typing import Pattern -from typing import Sequence from typing import SupportsIndex -from typing import Tuple -from typing import Type +from typing import TypeAlias from typing import TypeVar -from typing import Union import pluggy @@ -56,7 +54,7 @@ TracebackStyle = Literal["long", "short", "line", "no", "native", "value", "auto"] -EXCEPTION_OR_MORE = Union[Type[Exception], Tuple[Type[Exception], ...]] +EXCEPTION_OR_MORE = type[BaseException] | tuple[type[BaseException], ...] class Code: @@ -212,6 +210,45 @@ def with_repr_style( def lineno(self) -> int: return self._rawentry.tb_lineno - 1 + def get_python_framesummary(self) -> FrameSummary: + # Python's built-in traceback module implements all the nitty gritty + # details to get column numbers of out frames. + stack_summary = extract_tb(self._rawentry, limit=1) + return stack_summary[0] + + # Column and end line numbers introduced in python 3.11 + if sys.version_info < (3, 11): + + @property + def end_lineno_relative(self) -> int | None: + return None + + @property + def colno(self) -> int | None: + return None + + @property + def end_colno(self) -> int | None: + return None + else: + + @property + def end_lineno_relative(self) -> int | None: + frame_summary = self.get_python_framesummary() + if frame_summary.end_lineno is None: # pragma: no cover + return None + return frame_summary.end_lineno - 1 - self.frame.code.firstlineno + + @property + def colno(self) -> int | None: + """Starting byte offset of the expression in the traceback entry.""" + return self.get_python_framesummary().colno + + @property + def end_colno(self) -> int | None: + """Ending byte offset of the expression in the traceback entry.""" + return self.get_python_framesummary().end_colno + @property def frame(self) -> Frame: return Frame(self._rawentry.tb_frame) @@ -221,7 +258,7 @@ def relline(self) -> int: return self.lineno - self.frame.code.firstlineno def __repr__(self) -> str: - return "" % (self.frame.code.path, self.lineno + 1) + return f"" @property def statement(self) -> Source: @@ -307,12 +344,7 @@ def __str__(self) -> str: # This output does not quite match Python's repr for traceback entries, # but changing it to do so would break certain plugins. See # https://github.com/pytest-dev/pytest/pull/7535/ for details. - return " File %r:%d in %s\n %s\n" % ( - str(self.path), - self.lineno + 1, - name, - line, - ) + return f" File '{self.path}':{self.lineno + 1} in {name}\n {line}\n" @property def name(self) -> str: @@ -320,7 +352,7 @@ def name(self) -> str: return self.frame.code.raw.co_name -class Traceback(List[TracebackEntry]): +class Traceback(list[TracebackEntry]): """Traceback objects encapsulate and offer higher level access to Traceback entries.""" def __init__( @@ -429,6 +461,33 @@ def recursionindex(self) -> int | None: return None +def stringify_exception( + exc: BaseException, include_subexception_msg: bool = True +) -> str: + try: + notes = getattr(exc, "__notes__", []) + except KeyError: + # Workaround for https://github.com/python/cpython/issues/98778 on + # some 3.10 and 3.11 patch versions. + HTTPError = getattr(sys.modules.get("urllib.error", None), "HTTPError", ()) + if sys.version_info < (3, 12) and isinstance(exc, HTTPError): + notes = [] + else: # pragma: no cover + # exception not related to above bug, reraise + raise + if not include_subexception_msg and isinstance(exc, BaseExceptionGroup): + message = exc.message + else: + message = str(exc) + + return "\n".join( + [ + message, + *notes, + ] + ) + + E = TypeVar("E", bound=BaseException, covariant=True) @@ -536,33 +595,33 @@ def fill_unfilled(self, exc_info: tuple[type[E], E, TracebackType]) -> None: @property def type(self) -> type[E]: """The exception class.""" - assert ( - self._excinfo is not None - ), ".type can only be used after the context manager exits" + assert self._excinfo is not None, ( + ".type can only be used after the context manager exits" + ) return self._excinfo[0] @property def value(self) -> E: """The exception value.""" - assert ( - self._excinfo is not None - ), ".value can only be used after the context manager exits" + assert self._excinfo is not None, ( + ".value can only be used after the context manager exits" + ) return self._excinfo[1] @property def tb(self) -> TracebackType: """The exception raw traceback.""" - assert ( - self._excinfo is not None - ), ".tb can only be used after the context manager exits" + assert self._excinfo is not None, ( + ".tb can only be used after the context manager exits" + ) return self._excinfo[2] @property def typename(self) -> str: """The type name of the exception.""" - assert ( - self._excinfo is not None - ), ".typename can only be used after the context manager exits" + assert self._excinfo is not None, ( + ".typename can only be used after the context manager exits" + ) return self.type.__name__ @property @@ -589,6 +648,23 @@ def exconly(self, tryshort: bool = False) -> str: representation is returned (so 'AssertionError: ' is removed from the beginning). """ + + def _get_single_subexc( + eg: BaseExceptionGroup[BaseException], + ) -> BaseException | None: + if len(eg.exceptions) != 1: + return None + if isinstance(e := eg.exceptions[0], BaseExceptionGroup): + return _get_single_subexc(e) + return e + + if ( + tryshort + and isinstance(self.value, BaseExceptionGroup) + and (subexc := _get_single_subexc(self.value)) is not None + ): + return f"{subexc!r} [single exception in {type(self.value).__name__}]" + lines = format_exception_only(self.type, self.value) text = "".join(lines) text = text.rstrip() @@ -620,8 +696,7 @@ def getrepr( showlocals: bool = False, style: TracebackStyle = "long", abspath: bool = False, - tbfilter: bool - | Callable[[ExceptionInfo[BaseException]], _pytest._code.code.Traceback] = True, + tbfilter: bool | Callable[[ExceptionInfo[BaseException]], Traceback] = True, funcargs: bool = False, truncate_locals: bool = True, truncate_args: bool = True, @@ -668,7 +743,7 @@ def getrepr( if style == "native": return ReprExceptionInfo( reprtraceback=ReprTracebackNative( - traceback.format_exception( + format_exception( self.type, self.value, self.traceback[0]._rawentry if self.traceback else None, @@ -689,34 +764,19 @@ def getrepr( ) return fmt.repr_excinfo(self) - def _stringify_exception(self, exc: BaseException) -> str: - try: - notes = getattr(exc, "__notes__", []) - except KeyError: - # Workaround for https://github.com/python/cpython/issues/98778 on - # Python <= 3.9, and some 3.10 and 3.11 patch versions. - HTTPError = getattr(sys.modules.get("urllib.error", None), "HTTPError", ()) - if sys.version_info < (3, 12) and isinstance(exc, HTTPError): - notes = [] - else: - raise - - return "\n".join( - [ - str(exc), - *notes, - ] - ) - - def match(self, regexp: str | Pattern[str]) -> Literal[True]: + def match(self, regexp: str | re.Pattern[str]) -> Literal[True]: """Check whether the regular expression `regexp` matches the string representation of the exception using :func:`python:re.search`. If it matches `True` is returned, otherwise an `AssertionError` is raised. """ __tracebackhide__ = True - value = self._stringify_exception(self.value) - msg = f"Regex pattern did not match.\n Regex: {regexp!r}\n Input: {value!r}" + value = stringify_exception(self.value) + msg = ( + f"Regex pattern did not match.\n" + f" Expected regex: {regexp!r}\n" + f" Actual message: {value!r}" + ) if regexp == value: msg += "\n Did you mean to `re.escape()` the regex?" assert re.search(regexp, value), msg @@ -727,7 +787,7 @@ def _group_contains( self, exc_group: BaseExceptionGroup[BaseException], expected_exception: EXCEPTION_OR_MORE, - match: str | Pattern[str] | None, + match: str | re.Pattern[str] | None, target_depth: int | None = None, current_depth: int = 1, ) -> bool: @@ -747,7 +807,7 @@ def _group_contains( if not isinstance(exc, expected_exception): continue if match is not None: - value = self._stringify_exception(exc) + value = stringify_exception(exc) if not re.search(match, value): continue return True @@ -757,7 +817,7 @@ def group_contains( self, expected_exception: EXCEPTION_OR_MORE, *, - match: str | Pattern[str] | None = None, + match: str | re.Pattern[str] | None = None, depth: int | None = None, ) -> bool: """Check whether a captured exception group contains a matching exception. @@ -766,7 +826,7 @@ def group_contains( The expected exception type, or a tuple if one of multiple possible exception types are expected. - :param str | Pattern[str] | None match: + :param str | re.Pattern[str] | None match: If specified, a string containing a regular expression, or a regular expression object, that is tested against the string representation of the exception and its `PEP-678 ` `__notes__` @@ -781,6 +841,13 @@ def group_contains( the exceptions contained within the topmost exception group). .. versionadded:: 8.0 + + .. warning:: + This helper makes it easy to check for the presence of specific exceptions, + but it is very bad for checking that the group does *not* contain + *any other exceptions*. + You should instead consider using :class:`pytest.RaisesGroup` + """ msg = "Captured exception is not an instance of `BaseExceptionGroup`" assert isinstance(self.value, BaseExceptionGroup), msg @@ -789,6 +856,12 @@ def group_contains( return self._group_contains(self.value, expected_exception, match, depth) +# Type alias for the `tbfilter` setting: +# bool: If True, it should be filtered using Traceback.filter() +# callable: A callable that takes an ExceptionInfo and returns the filtered traceback. +TracebackFilter: TypeAlias = bool | Callable[[ExceptionInfo[BaseException]], Traceback] + + @dataclasses.dataclass class FormattedExcinfo: """Presenting information about failing Functions and Generators.""" @@ -800,7 +873,7 @@ class FormattedExcinfo: showlocals: bool = False style: TracebackStyle = "long" abspath: bool = True - tbfilter: bool | Callable[[ExceptionInfo[BaseException]], Traceback] = True + tbfilter: TracebackFilter = True funcargs: bool = False truncate_locals: bool = True truncate_args: bool = True @@ -848,6 +921,9 @@ def get_source( line_index: int = -1, excinfo: ExceptionInfo[BaseException] | None = None, short: bool = False, + end_line_index: int | None = None, + colno: int | None = None, + end_colno: int | None = None, ) -> list[str]: """Return formatted and marked up source lines.""" lines = [] @@ -861,10 +937,30 @@ def get_source( space_prefix = " " if short: lines.append(space_prefix + source.lines[line_index].strip()) + lines.extend( + self.get_highlight_arrows_for_line( + raw_line=source.raw_lines[line_index], + line=source.lines[line_index].strip(), + lineno=line_index, + end_lineno=end_line_index, + colno=colno, + end_colno=end_colno, + ) + ) else: for line in source.lines[:line_index]: lines.append(space_prefix + line) lines.append(self.flow_marker + " " + source.lines[line_index]) + lines.extend( + self.get_highlight_arrows_for_line( + raw_line=source.raw_lines[line_index], + line=source.lines[line_index], + lineno=line_index, + end_lineno=end_line_index, + colno=colno, + end_colno=end_colno, + ) + ) for line in source.lines[line_index + 1 :]: lines.append(space_prefix + line) if excinfo is not None: @@ -872,6 +968,43 @@ def get_source( lines.extend(self.get_exconly(excinfo, indent=indent, markall=True)) return lines + def get_highlight_arrows_for_line( + self, + line: str, + raw_line: str, + lineno: int | None, + end_lineno: int | None, + colno: int | None, + end_colno: int | None, + ) -> list[str]: + """Return characters highlighting a source line. + + Example with colno and end_colno pointing to the bar expression: + "foo() + bar()" + returns " ^^^^^" + """ + if lineno != end_lineno: + # Don't handle expressions that span multiple lines. + return [] + if colno is None or end_colno is None: + # Can't do anything without column information. + return [] + + num_stripped_chars = len(raw_line) - len(line) + + start_char_offset = _byte_offset_to_character_offset(raw_line, colno) + end_char_offset = _byte_offset_to_character_offset(raw_line, end_colno) + num_carets = end_char_offset - start_char_offset + # If the highlight would span the whole line, it is redundant, don't + # show it. + if num_carets >= len(line.strip()): + return [] + + highlights = " " + highlights += " " * (start_char_offset - num_stripped_chars + 1) + highlights += "^" * num_carets + return [highlights] + def get_exconly( self, excinfo: ExceptionInfo[BaseException], @@ -931,16 +1064,28 @@ def repr_traceback_entry( if source is None: source = Source("???") line_index = 0 + end_line_index, colno, end_colno = None, None, None else: - line_index = entry.lineno - entry.getfirstlinesource() + line_index = entry.relline + end_line_index = entry.end_lineno_relative + colno = entry.colno + end_colno = entry.end_colno short = style == "short" reprargs = self.repr_args(entry) if not short else None - s = self.get_source(source, line_index, excinfo, short=short) + s = self.get_source( + source=source, + line_index=line_index, + excinfo=excinfo, + short=short, + end_line_index=end_line_index, + colno=colno, + end_colno=end_colno, + ) lines.extend(s) if short: message = f"in {entry.name}" else: - message = excinfo and excinfo.typename or "" + message = (excinfo and excinfo.typename) or "" entry_path = entry.path path = self._makepath(entry_path) reprfileloc = ReprFileLocation(path, entry.lineno + 1, message) @@ -966,11 +1111,7 @@ def _makepath(self, path: Path | str) -> str: return str(path) def repr_traceback(self, excinfo: ExceptionInfo[BaseException]) -> ReprTraceback: - traceback = excinfo.traceback - if callable(self.tbfilter): - traceback = self.tbfilter(excinfo) - elif self.tbfilter: - traceback = traceback.filter(excinfo) + traceback = filter_excinfo_traceback(self.tbfilter, excinfo) if isinstance(excinfo.value, RecursionError): traceback, extraline = self._truncate_recursive_traceback(traceback) @@ -1044,25 +1185,30 @@ def repr_excinfo(self, excinfo: ExceptionInfo[BaseException]) -> ExceptionChainR # Fall back to native traceback as a temporary workaround until # full support for exception groups added to ExceptionInfo. # See https://github.com/pytest-dev/pytest/issues/9159 + reprtraceback: ReprTraceback | ReprTracebackNative if isinstance(e, BaseExceptionGroup): - reprtraceback: ReprTracebackNative | ReprTraceback = ( - ReprTracebackNative( - traceback.format_exception( - type(excinfo_.value), - excinfo_.value, - excinfo_.traceback[0]._rawentry, - ) + # don't filter any sub-exceptions since they shouldn't have any internal frames + traceback = filter_excinfo_traceback(self.tbfilter, excinfo) + reprtraceback = ReprTracebackNative( + format_exception( + type(excinfo.value), + excinfo.value, + traceback[0]._rawentry if traceback else None, ) ) + if not traceback: + reprtraceback.extraline = ( + "All traceback entries are hidden. " + "Pass `--full-trace` to see hidden and internal frames." + ) + else: reprtraceback = self.repr_traceback(excinfo_) reprcrash = excinfo_._getreprcrash() else: # Fallback to native repr if the exception doesn't have a traceback: # ExceptionInfo objects require a full traceback to work. - reprtraceback = ReprTracebackNative( - traceback.format_exception(type(e), e, None) - ) + reprtraceback = ReprTracebackNative(format_exception(type(e), e, None)) reprcrash = None repr_chain += [(reprtraceback, reprcrash, descr)] @@ -1169,10 +1315,8 @@ def toterminal(self, tw: TerminalWriter) -> None: entry.toterminal(tw) if i < len(self.reprentries) - 1: next_entry = self.reprentries[i + 1] - if ( - entry.style == "long" - or entry.style == "short" - and next_entry.style == "long" + if entry.style == "long" or ( + entry.style == "short" and next_entry.style == "long" ): tw.sep(self.entrysep) @@ -1221,6 +1365,15 @@ def _write_entry_lines(self, tw: TerminalWriter) -> None: if not self.lines: return + if self.style == "value": + # Using tw.write instead of tw.line for testing purposes due to TWMock implementation; + # lines written with TWMock.line and TWMock._write_source cannot be distinguished + # from each other, whereas lines written with TWMock.write are marked with TWMock.WRITE + for line in self.lines: + tw.write(line) + tw.write("\n") + return + # separate indents and source lines that are not failures: we want to # highlight the code but not the indentation, which may contain markers # such as "> assert 0" @@ -1236,11 +1389,8 @@ def _write_entry_lines(self, tw: TerminalWriter) -> None: failure_lines.extend(self.lines[index:]) break else: - if self.style == "value": - source_lines.append(line) - else: - indents.append(line[:indent_size]) - source_lines.append(line[indent_size:]) + indents.append(line[:indent_size]) + source_lines.append(line[indent_size:]) tw._write_source(source_lines, indents) @@ -1350,7 +1500,7 @@ def getfslineno(obj: object) -> tuple[str | Path, int]: except TypeError: return "", -1 - fspath = fn and absolutepath(fn) or "" + fspath = (fn and absolutepath(fn)) or "" lineno = -1 if fspath: try: @@ -1362,6 +1512,12 @@ def getfslineno(obj: object) -> tuple[str | Path, int]: return code.path, code.firstlineno +def _byte_offset_to_character_offset(str, offset): + """Converts a byte based offset in a string to a code-point.""" + as_utf8 = str.encode("utf-8") + return len(as_utf8[:offset].decode("utf-8", errors="replace")) + + # Relative paths that we use to filter traceback entries from appearing to the user; # see filter_traceback. # note: if we need to add more paths than what we have now we should probably use a list @@ -1401,3 +1557,15 @@ def filter_traceback(entry: TracebackEntry) -> bool: return False return True + + +def filter_excinfo_traceback( + tbfilter: TracebackFilter, excinfo: ExceptionInfo[BaseException] +) -> Traceback: + """Filter the exception traceback in ``excinfo`` according to ``tbfilter``.""" + if callable(tbfilter): + return tbfilter(excinfo) + elif tbfilter: + return excinfo.traceback.filter(excinfo) + else: + return excinfo.traceback diff --git a/src/_pytest/_code/source.py b/src/_pytest/_code/source.py index 604aff8ba19..99c242dd98e 100644 --- a/src/_pytest/_code/source.py +++ b/src/_pytest/_code/source.py @@ -3,12 +3,12 @@ import ast from bisect import bisect_right +from collections.abc import Iterable +from collections.abc import Iterator import inspect import textwrap import tokenize import types -from typing import Iterable -from typing import Iterator from typing import overload import warnings @@ -22,12 +22,16 @@ class Source: def __init__(self, obj: object = None) -> None: if not obj: self.lines: list[str] = [] + self.raw_lines: list[str] = [] elif isinstance(obj, Source): self.lines = obj.lines - elif isinstance(obj, (tuple, list)): + self.raw_lines = obj.raw_lines + elif isinstance(obj, tuple | list): self.lines = deindent(x.rstrip("\n") for x in obj) + self.raw_lines = list(x.rstrip("\n") for x in obj) elif isinstance(obj, str): self.lines = deindent(obj.split("\n")) + self.raw_lines = obj.split("\n") else: try: rawcode = getrawcode(obj) @@ -35,6 +39,7 @@ def __init__(self, obj: object = None) -> None: except TypeError: src = inspect.getsource(obj) # type: ignore[arg-type] self.lines = deindent(src.split("\n")) + self.raw_lines = src.split("\n") def __eq__(self, other: object) -> bool: if not isinstance(other, Source): @@ -58,6 +63,7 @@ def __getitem__(self, key: int | slice) -> str | Source: raise IndexError("cannot slice a Source with a step") newsource = Source() newsource.lines = self.lines[key.start : key.stop] + newsource.raw_lines = self.raw_lines[key.start : key.stop] return newsource def __iter__(self) -> Iterator[str]: @@ -74,6 +80,7 @@ def strip(self) -> Source: while end > start and not self.lines[end - 1].strip(): end -= 1 source = Source() + source.raw_lines = self.raw_lines source.lines[:] = self.lines[start:end] return source @@ -81,6 +88,7 @@ def indent(self, indent: str = " " * 4) -> Source: """Return a copy of the source object with all lines indented by the given indent-string.""" newsource = Source() + newsource.raw_lines = self.raw_lines newsource.lines = [(indent + line) for line in self.lines] return newsource @@ -95,13 +103,14 @@ def getstatementrange(self, lineno: int) -> tuple[int, int]: which containing the given lineno.""" if not (0 <= lineno < len(self)): raise IndexError("lineno out of range") - ast, start, end = getstatementrange_ast(lineno, self) + _ast, start, end = getstatementrange_ast(lineno, self) return start, end def deindent(self) -> Source: """Return a new Source object deindented.""" newsource = Source() newsource.lines[:] = deindent(self.lines) + newsource.raw_lines = self.raw_lines return newsource def __str__(self) -> str: @@ -120,6 +129,7 @@ def findsource(obj) -> tuple[Source | None, int]: return None, -1 source = Source() source.lines = [line.rstrip() for line in sourcelines] + source.raw_lines = sourcelines return source, lineno @@ -145,9 +155,9 @@ def get_statement_startend2(lineno: int, node: ast.AST) -> tuple[int, int | None # AST's line numbers start indexing at 1. values: list[int] = [] for x in ast.walk(node): - if isinstance(x, (ast.stmt, ast.ExceptHandler)): + if isinstance(x, ast.stmt | ast.ExceptHandler): # The lineno points to the class/def, so need to include the decorators. - if isinstance(x, (ast.ClassDef, ast.FunctionDef, ast.AsyncFunctionDef)): + if isinstance(x, ast.ClassDef | ast.FunctionDef | ast.AsyncFunctionDef): for d in x.decorator_list: values.append(d.lineno - 1) values.append(x.lineno - 1) diff --git a/src/_pytest/_io/pprint.py b/src/_pytest/_io/pprint.py index 7213be7ba9b..28f06909206 100644 --- a/src/_pytest/_io/pprint.py +++ b/src/_pytest/_io/pprint.py @@ -16,14 +16,14 @@ from __future__ import annotations import collections as _collections +from collections.abc import Callable +from collections.abc import Iterator import dataclasses as _dataclasses from io import StringIO as _StringIO import re import types as _types from typing import Any -from typing import Callable from typing import IO -from typing import Iterator class _safe_key: @@ -113,7 +113,7 @@ def _format( elif ( _dataclasses.is_dataclass(object) and not isinstance(object, type) - and object.__dataclass_params__.repr + and object.__dataclass_params__.repr # type:ignore[attr-defined] and # Check dataclass has generated repr method. hasattr(object.__repr__, "__wrapped__") @@ -540,7 +540,7 @@ def _pprint_deque( ) -> None: stream.write(object.__class__.__name__ + "(") if object.maxlen is not None: - stream.write("maxlen=%d, " % object.maxlen) + stream.write(f"maxlen={object.maxlen}, ") stream.write("[") self._format_items(object, stream, indent, allowance + 1, context, level) diff --git a/src/_pytest/_io/terminalwriter.py b/src/_pytest/_io/terminalwriter.py index 70ebd3d061b..9191b4edace 100644 --- a/src/_pytest/_io/terminalwriter.py +++ b/src/_pytest/_io/terminalwriter.py @@ -2,24 +2,24 @@ from __future__ import annotations +from collections.abc import Sequence import os import shutil import sys from typing import final from typing import Literal -from typing import Sequence from typing import TextIO -from typing import TYPE_CHECKING + +import pygments +from pygments.formatters.terminal import TerminalFormatter +from pygments.lexer import Lexer +from pygments.lexers.diff import DiffLexer +from pygments.lexers.python import PythonLexer from ..compat import assert_never from .wcwidth import wcswidth -if TYPE_CHECKING: - from pygments.formatter import Formatter - from pygments.lexer import Lexer - - # This code was initially copied from py 1.8.1, file _io/terminalwriter.py. @@ -161,20 +161,23 @@ def write(self, msg: str, *, flush: bool = False, **markup: bool) -> None: msg = self.markup(msg, **markup) - try: - self._file.write(msg) - except UnicodeEncodeError: - # Some environments don't support printing general Unicode - # strings, due to misconfiguration or otherwise; in that case, - # print the string escaped to ASCII. - # When the Unicode situation improves we should consider - # letting the error propagate instead of masking it (see #7475 - # for one brief attempt). - msg = msg.encode("unicode-escape").decode("ascii") - self._file.write(msg) - - if flush: - self.flush() + self.write_raw(msg, flush=flush) + + def write_raw(self, msg: str, *, flush: bool = False) -> None: + try: + self._file.write(msg) + except UnicodeEncodeError: + # Some environments don't support printing general Unicode + # strings, due to misconfiguration or otherwise; in that case, + # print the string escaped to ASCII. + # When the Unicode situation improves we should consider + # letting the error propagate instead of masking it (see #7475 + # for one brief attempt). + msg = msg.encode("unicode-escape").decode("ascii") + self._file.write(msg) + + if flush: + self.flush() def line(self, s: str = "", **markup: bool) -> None: self.write(s, **markup) @@ -198,40 +201,26 @@ def _write_source(self, lines: Sequence[str], indents: Sequence[str] = ()) -> No indents = [""] * len(lines) source = "\n".join(lines) new_lines = self._highlight(source).splitlines() - for indent, new_line in zip(indents, new_lines): + # Would be better to strict=True but that fails some CI jobs. + for indent, new_line in zip(indents, new_lines, strict=False): self.line(indent + new_line) - def _get_pygments_lexer(self, lexer: Literal["python", "diff"]) -> Lexer | None: - try: - if lexer == "python": - from pygments.lexers.python import PythonLexer - - return PythonLexer() - elif lexer == "diff": - from pygments.lexers.diff import DiffLexer - - return DiffLexer() - else: - assert_never(lexer) - except ModuleNotFoundError: - return None - - def _get_pygments_formatter(self) -> Formatter | None: - try: - import pygments.util - except ModuleNotFoundError: - return None + def _get_pygments_lexer(self, lexer: Literal["python", "diff"]) -> Lexer: + if lexer == "python": + return PythonLexer() + elif lexer == "diff": + return DiffLexer() + else: + assert_never(lexer) + def _get_pygments_formatter(self) -> TerminalFormatter: from _pytest.config.exceptions import UsageError theme = os.getenv("PYTEST_THEME") theme_mode = os.getenv("PYTEST_THEME_MODE", "dark") try: - from pygments.formatters.terminal import TerminalFormatter - return TerminalFormatter(bg=theme_mode, style=theme) - except pygments.util.ClassNotFound as e: raise UsageError( f"PYTEST_THEME environment variable has an invalid value: '{theme}'. " @@ -251,16 +240,11 @@ def _highlight( return source pygments_lexer = self._get_pygments_lexer(lexer) - if pygments_lexer is None: - return source - pygments_formatter = self._get_pygments_formatter() - if pygments_formatter is None: - return source - - from pygments import highlight - highlighted: str = highlight(source, pygments_lexer, pygments_formatter) + highlighted: str = pygments.highlight( + source, pygments_lexer, pygments_formatter + ) # pygments terminal formatter may add a newline when there wasn't one. # We don't want this, remove. if highlighted[-1] == "\n" and source[-1] != "\n": diff --git a/src/_pytest/_py/error.py b/src/_pytest/_py/error.py index ab3a4ed318e..dace23764ff 100644 --- a/src/_pytest/_py/error.py +++ b/src/_pytest/_py/error.py @@ -2,10 +2,10 @@ from __future__ import annotations +from collections.abc import Callable import errno import os import sys -from typing import Callable from typing import TYPE_CHECKING from typing import TypeVar @@ -69,7 +69,7 @@ def _geterrnoclass(self, eno: int) -> type[Error]: try: return self._errno2class[eno] except KeyError: - clsname = errno.errorcode.get(eno, "UnknownErrno%d" % (eno,)) + clsname = errno.errorcode.get(eno, f"UnknownErrno{eno}") errorcls = type( clsname, (Error,), @@ -90,15 +90,23 @@ def checked_call( except OSError as value: if not hasattr(value, "errno"): raise - errno = value.errno if sys.platform == "win32": try: - cls = self._geterrnoclass(_winerrnomap[errno]) + # error: Invalid index type "Optional[int]" for "dict[int, int]"; expected type "int" [index] + # OK to ignore because we catch the KeyError below. + cls = self._geterrnoclass(_winerrnomap[value.errno]) # type:ignore[index] except KeyError: raise value else: # we are not on Windows, or we got a proper OSError - cls = self._geterrnoclass(errno) + if value.errno is None: + cls = type( + "UnknownErrnoNone", + (Error,), + {"__module__": "py.error", "__doc__": None}, + ) + else: + cls = self._geterrnoclass(value.errno) raise cls(f"{func.__name__}{args!r}") diff --git a/src/_pytest/_py/path.py b/src/_pytest/_py/path.py index c7ab1182f4a..998a7819972 100644 --- a/src/_pytest/_py/path.py +++ b/src/_pytest/_py/path.py @@ -4,6 +4,7 @@ from __future__ import annotations import atexit +from collections.abc import Callable from contextlib import contextmanager import fnmatch import importlib.util @@ -23,7 +24,6 @@ from stat import S_ISREG import sys from typing import Any -from typing import Callable from typing import cast from typing import Literal from typing import overload @@ -137,7 +137,7 @@ class NeverRaised(Exception): class Visitor: def __init__(self, fil, rec, ignore, bf, sort): - if isinstance(fil, str): + if isinstance(fil, (str, bytes)): fil = FNMatcher(fil) if isinstance(rec, str): self.rec: Callable[[LocalPath], bool] = FNMatcher(rec) @@ -432,7 +432,7 @@ def relto(self, relpath): """Return a string which is the relative part of the path to the given 'relpath'. """ - if not isinstance(relpath, (str, LocalPath)): + if not isinstance(relpath, str | LocalPath): raise TypeError(f"{relpath!r}: not a string or path object") strrelpath = str(relpath) if strrelpath and strrelpath[-1] != self.sep: @@ -652,7 +652,7 @@ def new(self, **kw): if not kw: obj.strpath = self.strpath return obj - drive, dirname, basename, purebasename, ext = self._getbyspec( + drive, dirname, _basename, purebasename, ext = self._getbyspec( "drive,dirname,basename,purebasename,ext" ) if "basename" in kw: diff --git a/src/_pytest/assertion/__init__.py b/src/_pytest/assertion/__init__.py index f2f1d029b4c..22f3ca8e258 100644 --- a/src/_pytest/assertion/__init__.py +++ b/src/_pytest/assertion/__init__.py @@ -3,9 +3,10 @@ from __future__ import annotations +from collections.abc import Generator import sys from typing import Any -from typing import Generator +from typing import Protocol from typing import TYPE_CHECKING from _pytest.assertion import rewrite @@ -45,6 +46,18 @@ def pytest_addoption(parser: Parser) -> None: help="Enables the pytest_assertion_pass hook. " "Make sure to delete any previously generated pyc cache files.", ) + + parser.addini( + "truncation_limit_lines", + default=None, + help="Set threshold of LINES after which truncation will take effect", + ) + parser.addini( + "truncation_limit_chars", + default=None, + help=("Set threshold of CHARS after which truncation will take effect"), + ) + Config._add_verbosity_ini( parser, Config.VERBOSITY_ASSERTIONS, @@ -70,15 +83,18 @@ def register_assert_rewrite(*names: str) -> None: if not isinstance(name, str): msg = "expected module names as *args, got {0} instead" # type: ignore[unreachable] raise TypeError(msg.format(repr(names))) + rewrite_hook: RewriteHook for hook in sys.meta_path: if isinstance(hook, rewrite.AssertionRewritingHook): - importhook = hook + rewrite_hook = hook break else: - # TODO(typing): Add a protocol for mark_rewrite() and use it - # for importhook and for PytestPluginManager.rewrite_hook. - importhook = DummyRewriteHook() # type: ignore - importhook.mark_rewrite(*names) + rewrite_hook = DummyRewriteHook() + rewrite_hook.mark_rewrite(*names) + + +class RewriteHook(Protocol): + def mark_rewrite(self, *names: str) -> None: ... class DummyRewriteHook: diff --git a/src/_pytest/assertion/rewrite.py b/src/_pytest/assertion/rewrite.py index bfcbcbd3f8d..566549d66f2 100644 --- a/src/_pytest/assertion/rewrite.py +++ b/src/_pytest/assertion/rewrite.py @@ -4,6 +4,10 @@ import ast from collections import defaultdict +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Sequence import errno import functools import importlib.abc @@ -19,18 +23,27 @@ import sys import tokenize import types -from typing import Callable from typing import IO -from typing import Iterable -from typing import Iterator -from typing import Sequence from typing import TYPE_CHECKING + +if sys.version_info >= (3, 12): + from importlib.resources.abc import TraversableResources +else: + from importlib.abc import TraversableResources +if sys.version_info < (3, 11): + from importlib.readers import FileReader +else: + from importlib.resources.readers import FileReader + + from _pytest._io.saferepr import DEFAULT_REPR_MAX_SIZE from _pytest._io.saferepr import saferepr +from _pytest._io.saferepr import saferepr_unlimited from _pytest._version import version from _pytest.assertion import util from _pytest.config import Config +from _pytest.fixtures import FixtureFunctionDefinition from _pytest.main import Session from _pytest.pathlib import absolutepath from _pytest.pathlib import fnmatch_ex @@ -53,7 +66,7 @@ class Sentinel: # pytest caches rewritten pycs in pycache dirs PYTEST_TAG = f"{sys.implementation.cache_tag}-pytest-{version}" -PYC_EXT = ".py" + (__debug__ and "c" or "o") +PYC_EXT = ".py" + ((__debug__ and "c") or "o") PYC_TAIL = "." + PYTEST_TAG + PYC_EXT # Special marker that denotes we have just left a scope definition @@ -101,6 +114,16 @@ def find_spec( # Type ignored because mypy is confused about the `self` binding here. spec = self._find_spec(name, path) # type: ignore + + if spec is None and path is not None: + # With --import-mode=importlib, PathFinder cannot find spec without modifying `sys.path`, + # causing inability to assert rewriting (#12659). + # At this point, try using the file path to find the module spec. + for _path_str in path: + spec = importlib.util.spec_from_file_location(name, _path_str) + if spec is not None: + break + if ( # the import machinery could not find a file to import spec is None @@ -269,7 +292,7 @@ def _warn_already_imported(self, name: str) -> None: self.config.issue_config_time_warning( PytestAssertRewriteWarning( - f"Module already imported so cannot be rewritten: {name}" + f"Module already imported so cannot be rewritten; {name}" ), stacklevel=5, ) @@ -279,19 +302,8 @@ def get_data(self, pathname: str | bytes) -> bytes: with open(pathname, "rb") as f: return f.read() - if sys.version_info >= (3, 10): - if sys.version_info >= (3, 12): - from importlib.resources.abc import TraversableResources - else: - from importlib.abc import TraversableResources - - def get_resource_reader(self, name: str) -> TraversableResources: - if sys.version_info < (3, 11): - from importlib.readers import FileReader - else: - from importlib.resources.readers import FileReader - - return FileReader(types.SimpleNamespace(path=self._rewritten_names[name])) + def get_resource_reader(self, name: str) -> TraversableResources: + return FileReader(types.SimpleNamespace(path=self._rewritten_names[name])) # type: ignore[arg-type] def _write_pyc_fp( @@ -422,6 +434,8 @@ def _saferepr(obj: object) -> str: return obj.__name__ maxsize = _get_maxsize_for_saferepr(util._config) + if not maxsize: + return saferepr_unlimited(obj).replace("\n", "\\n") return saferepr(obj, maxsize=maxsize).replace("\n", "\\n") @@ -451,7 +465,7 @@ def _format_assertmsg(obj: object) -> str: # However in either case we want to preserve the newline. replaces = [("\n", "\n~"), ("%", "%%")] if not isinstance(obj, str): - obj = saferepr(obj) + obj = saferepr(obj, _get_maxsize_for_saferepr(util._config)) replaces.append(("\\n", "\n~")) for r1, r2 in replaces: @@ -462,7 +476,8 @@ def _format_assertmsg(obj: object) -> str: def _should_repr_global_name(obj: object) -> bool: if callable(obj): - return False + # For pytest fixtures the __repr__ method provides more information than the function name. + return isinstance(obj, FixtureFunctionDefinition) try: return not hasattr(obj, "__name__") @@ -471,7 +486,7 @@ def _should_repr_global_name(obj: object) -> bool: def _format_boolop(explanations: Iterable[str], is_or: bool) -> str: - explanation = "(" + (is_or and " or " or " and ").join(explanations) + ")" + explanation = "(" + ((is_or and " or ") or " and ").join(explanations) + ")" return explanation.replace("%", "%%") @@ -481,7 +496,7 @@ def _call_reprcompare( expls: Sequence[str], each_obj: Sequence[object], ) -> str: - for i, res, expl in zip(range(len(ops)), results, expls): + for i, res, expl in zip(range(len(ops)), results, expls, strict=True): try: done = not res except Exception: @@ -687,26 +702,18 @@ def run(self, mod: ast.Module) -> None: if doc is not None and self.is_rewrite_disabled(doc): return pos = 0 - item = None for item in mod.body: - if ( - expect_docstring - and isinstance(item, ast.Expr) - and isinstance(item.value, ast.Constant) - and isinstance(item.value.value, str) - ): - doc = item.value.value - if self.is_rewrite_disabled(doc): - return - expect_docstring = False - elif ( - isinstance(item, ast.ImportFrom) - and item.level == 0 - and item.module == "__future__" - ): - pass - else: - break + match item: + case ast.Expr(value=ast.Constant(value=str() as doc)) if ( + expect_docstring + ): + if self.is_rewrite_disabled(doc): + return + expect_docstring = False + case ast.ImportFrom(level=0, module="__future__"): + pass + case _: + break pos += 1 # Special case: for a decorated function, set the lineno to that of the # first decorator, not the `def`. Issue #4984. @@ -715,21 +722,15 @@ def run(self, mod: ast.Module) -> None: else: lineno = item.lineno # Now actually insert the special imports. - if sys.version_info >= (3, 10): - aliases = [ - ast.alias("builtins", "@py_builtins", lineno=lineno, col_offset=0), - ast.alias( - "_pytest.assertion.rewrite", - "@pytest_ar", - lineno=lineno, - col_offset=0, - ), - ] - else: - aliases = [ - ast.alias("builtins", "@py_builtins"), - ast.alias("_pytest.assertion.rewrite", "@pytest_ar"), - ] + aliases = [ + ast.alias("builtins", "@py_builtins", lineno=lineno, col_offset=0), + ast.alias( + "_pytest.assertion.rewrite", + "@pytest_ar", + lineno=lineno, + col_offset=0, + ), + ] imports = [ ast.Import([alias], lineno=lineno, col_offset=0) for alias in aliases ] @@ -740,7 +741,7 @@ def run(self, mod: ast.Module) -> None: nodes: list[ast.AST | Sentinel] = [mod] while nodes: node = nodes.pop() - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef): self.scope = tuple((*self.scope, node)) nodes.append(_SCOPE_END_MARKER) if node == _SCOPE_END_MARKER: @@ -782,7 +783,7 @@ def assign(self, expr: ast.expr) -> ast.Name: """Give *expr* a name.""" name = self.variable() self.statements.append(ast.Assign([ast.Name(name, ast.Store())], expr)) - return ast.Name(name, ast.Load()) + return ast.copy_location(ast.Name(name, ast.Load()), expr) def display(self, expr: ast.expr) -> ast.expr: """Call saferepr on the expression.""" @@ -965,7 +966,10 @@ def visit_Assert(self, assert_: ast.Assert) -> list[ast.stmt]: # Fix locations (line numbers/column offsets). for stmt in self.statements: for node in traverse_node(stmt): - ast.copy_location(node, assert_) + if getattr(node, "lineno", None) is None: + # apply the assertion location to all generated ast nodes without source location + # and preserve the location of existing nodes or generated nodes with an correct location. + ast.copy_location(node, assert_) return self.statements def visit_NamedExpr(self, name: ast.NamedExpr) -> tuple[ast.NamedExpr, str]: @@ -1006,20 +1010,17 @@ def visit_BoolOp(self, boolop: ast.BoolOp) -> tuple[ast.Name, str]: # cond is set in a prior loop iteration below self.expl_stmts.append(ast.If(cond, fail_inner, [])) # noqa: F821 self.expl_stmts = fail_inner - # Check if the left operand is a ast.NamedExpr and the value has already been visited - if ( - isinstance(v, ast.Compare) - and isinstance(v.left, ast.NamedExpr) - and v.left.target.id - in [ - ast_expr.id - for ast_expr in boolop.values[:i] - if hasattr(ast_expr, "id") - ] - ): - pytest_temp = self.variable() - self.variables_overwrite[self.scope][v.left.target.id] = v.left # type:ignore[assignment] - v.left.target.id = pytest_temp + match v: + # Check if the left operand is an ast.NamedExpr and the value has already been visited + case ast.Compare( + left=ast.NamedExpr(target=ast.Name(id=target_id)) + ) if target_id in [ + e.id for e in boolop.values[:i] if hasattr(e, "id") + ]: + pytest_temp = self.variable() + self.variables_overwrite[self.scope][target_id] = v.left # type:ignore[assignment] + # mypy's false positive, we're checking that the 'target' attribute exists. + v.left.target.id = pytest_temp # type:ignore[attr-defined] self.push_format_context() res, expl = self.visit(v) body.append(ast.Assign([ast.Name(res_var, ast.Store())], res)) @@ -1042,7 +1043,7 @@ def visit_BoolOp(self, boolop: ast.BoolOp) -> tuple[ast.Name, str]: def visit_UnaryOp(self, unary: ast.UnaryOp) -> tuple[ast.Name, str]: pattern = UNARY_MAP[unary.op.__class__] operand_res, operand_expl = self.visit(unary.operand) - res = self.assign(ast.UnaryOp(unary.op, operand_res)) + res = self.assign(ast.copy_location(ast.UnaryOp(unary.op, operand_res), unary)) return res, pattern % (operand_expl,) def visit_BinOp(self, binop: ast.BinOp) -> tuple[ast.Name, str]: @@ -1050,7 +1051,9 @@ def visit_BinOp(self, binop: ast.BinOp) -> tuple[ast.Name, str]: left_expr, left_expl = self.visit(binop.left) right_expr, right_expl = self.visit(binop.right) explanation = f"({left_expl} {symbol} {right_expl})" - res = self.assign(ast.BinOp(left_expr, binop.op, right_expr)) + res = self.assign( + ast.copy_location(ast.BinOp(left_expr, binop.op, right_expr), binop) + ) return res, explanation def visit_Call(self, call: ast.Call) -> tuple[ast.Name, str]: @@ -1067,10 +1070,11 @@ def visit_Call(self, call: ast.Call) -> tuple[ast.Name, str]: arg_expls.append(expl) new_args.append(res) for keyword in call.keywords: - if isinstance( - keyword.value, ast.Name - ) and keyword.value.id in self.variables_overwrite.get(self.scope, {}): - keyword.value = self.variables_overwrite[self.scope][keyword.value.id] # type:ignore[assignment] + match keyword.value: + case ast.Name(id=id) if id in self.variables_overwrite.get( + self.scope, {} + ): + keyword.value = self.variables_overwrite[self.scope][id] # type:ignore[assignment] res, expl = self.visit(keyword.value) new_kwargs.append(ast.keyword(keyword.arg, res)) if keyword.arg: @@ -1079,7 +1083,7 @@ def visit_Call(self, call: ast.Call) -> tuple[ast.Name, str]: arg_expls.append("**" + expl) expl = "{}({})".format(func_expl, ", ".join(arg_expls)) - new_call = ast.Call(new_func, new_args, new_kwargs) + new_call = ast.copy_location(ast.Call(new_func, new_args, new_kwargs), call) res = self.assign(new_call) res_expl = self.explanation_param(self.display(res)) outer_expl = f"{res_expl}\n{{{res_expl} = {expl}\n}}" @@ -1095,7 +1099,9 @@ def visit_Attribute(self, attr: ast.Attribute) -> tuple[ast.Name, str]: if not isinstance(attr.ctx, ast.Load): return self.generic_visit(attr) value, value_expl = self.visit(attr.value) - res = self.assign(ast.Attribute(value, attr.attr, ast.Load())) + res = self.assign( + ast.copy_location(ast.Attribute(value, attr.attr, ast.Load()), attr) + ) res_expl = self.explanation_param(self.display(res)) pat = "%s\n{%s = %s.%s\n}" expl = pat % (res_expl, res_expl, value_expl, attr.attr) @@ -1104,39 +1110,41 @@ def visit_Attribute(self, attr: ast.Attribute) -> tuple[ast.Name, str]: def visit_Compare(self, comp: ast.Compare) -> tuple[ast.expr, str]: self.push_format_context() # We first check if we have overwritten a variable in the previous assert - if isinstance( - comp.left, ast.Name - ) and comp.left.id in self.variables_overwrite.get(self.scope, {}): - comp.left = self.variables_overwrite[self.scope][comp.left.id] # type:ignore[assignment] - if isinstance(comp.left, ast.NamedExpr): - self.variables_overwrite[self.scope][comp.left.target.id] = comp.left # type:ignore[assignment] + match comp.left: + case ast.Name(id=name_id) if name_id in self.variables_overwrite.get( + self.scope, {} + ): + comp.left = self.variables_overwrite[self.scope][name_id] # type: ignore[assignment] + case ast.NamedExpr(target=ast.Name(id=target_id)): + self.variables_overwrite[self.scope][target_id] = comp.left # type: ignore[assignment] left_res, left_expl = self.visit(comp.left) - if isinstance(comp.left, (ast.Compare, ast.BoolOp)): + if isinstance(comp.left, ast.Compare | ast.BoolOp): left_expl = f"({left_expl})" res_variables = [self.variable() for i in range(len(comp.ops))] load_names: list[ast.expr] = [ast.Name(v, ast.Load()) for v in res_variables] store_names = [ast.Name(v, ast.Store()) for v in res_variables] - it = zip(range(len(comp.ops)), comp.ops, comp.comparators) + it = zip(range(len(comp.ops)), comp.ops, comp.comparators, strict=True) expls: list[ast.expr] = [] syms: list[ast.expr] = [] results = [left_res] for i, op, next_operand in it: - if ( - isinstance(next_operand, ast.NamedExpr) - and isinstance(left_res, ast.Name) - and next_operand.target.id == left_res.id - ): - next_operand.target.id = self.variable() - self.variables_overwrite[self.scope][left_res.id] = next_operand # type:ignore[assignment] + match (next_operand, left_res): + case ( + ast.NamedExpr(target=ast.Name(id=target_id)), + ast.Name(id=name_id), + ) if target_id == name_id: + next_operand.target.id = self.variable() + self.variables_overwrite[self.scope][name_id] = next_operand # type: ignore[assignment] + next_res, next_expl = self.visit(next_operand) - if isinstance(next_operand, (ast.Compare, ast.BoolOp)): + if isinstance(next_operand, ast.Compare | ast.BoolOp): next_expl = f"({next_expl})" results.append(next_res) sym = BINOP_MAP[op.__class__] syms.append(ast.Constant(sym)) expl = f"{left_expl} {sym} {next_expl}" expls.append(ast.Constant(expl)) - res_expr = ast.Compare(left_res, [op], [next_res]) + res_expr = ast.copy_location(ast.Compare(left_res, [op], [next_res]), comp) self.statements.append(ast.Assign([store_names[i]], res_expr)) left_res, left_expl = next_res, next_expl # Use pytest.assertion.util._reprcompare if that's available. diff --git a/src/_pytest/assertion/truncate.py b/src/_pytest/assertion/truncate.py index b67f02ccaf8..5820e6e8a80 100644 --- a/src/_pytest/assertion/truncate.py +++ b/src/_pytest/assertion/truncate.py @@ -6,47 +6,60 @@ from __future__ import annotations -from _pytest.assertion import util +from _pytest.compat import running_on_ci from _pytest.config import Config from _pytest.nodes import Item DEFAULT_MAX_LINES = 8 -DEFAULT_MAX_CHARS = 8 * 80 +DEFAULT_MAX_CHARS = DEFAULT_MAX_LINES * 80 USAGE_MSG = "use '-vv' to show" -def truncate_if_required( - explanation: list[str], item: Item, max_length: int | None = None -) -> list[str]: +def truncate_if_required(explanation: list[str], item: Item) -> list[str]: """Truncate this assertion explanation if the given test item is eligible.""" - if _should_truncate_item(item): - return _truncate_explanation(explanation) + should_truncate, max_lines, max_chars = _get_truncation_parameters(item) + if should_truncate: + return _truncate_explanation( + explanation, + max_lines=max_lines, + max_chars=max_chars, + ) return explanation -def _should_truncate_item(item: Item) -> bool: - """Whether or not this test item is eligible for truncation.""" +def _get_truncation_parameters(item: Item) -> tuple[bool, int, int]: + """Return the truncation parameters related to the given item, as (should truncate, max lines, max chars).""" + # We do not need to truncate if one of conditions is met: + # 1. Verbosity level is 2 or more; + # 2. Test is being run in CI environment; + # 3. Both truncation_limit_lines and truncation_limit_chars + # .ini parameters are set to 0 explicitly. + max_lines = item.config.getini("truncation_limit_lines") + max_lines = int(max_lines if max_lines is not None else DEFAULT_MAX_LINES) + + max_chars = item.config.getini("truncation_limit_chars") + max_chars = int(max_chars if max_chars is not None else DEFAULT_MAX_CHARS) + verbose = item.config.get_verbosity(Config.VERBOSITY_ASSERTIONS) - return verbose < 2 and not util.running_on_ci() + + should_truncate = verbose < 2 and not running_on_ci() + should_truncate = should_truncate and (max_lines > 0 or max_chars > 0) + + return should_truncate, max_lines, max_chars def _truncate_explanation( input_lines: list[str], - max_lines: int | None = None, - max_chars: int | None = None, + max_lines: int, + max_chars: int, ) -> list[str]: """Truncate given list of strings that makes up the assertion explanation. - Truncates to either 8 lines, or 640 characters - whichever the input reaches + Truncates to either max_lines, or max_chars - whichever the input reaches first, taking the truncation explanation into account. The remaining lines will be replaced by a usage message. """ - if max_lines is None: - max_lines = DEFAULT_MAX_LINES - if max_chars is None: - max_chars = DEFAULT_MAX_CHARS - # Check if truncation required input_char_count = len("".join(input_lines)) # The length of the truncation explanation depends on the number of lines @@ -71,16 +84,23 @@ def _truncate_explanation( ): return input_lines # Truncate first to max_lines, and then truncate to max_chars if necessary - truncated_explanation = input_lines[:max_lines] + if max_lines > 0: + truncated_explanation = input_lines[:max_lines] + else: + truncated_explanation = input_lines truncated_char = True # We reevaluate the need to truncate chars following removal of some lines - if len("".join(truncated_explanation)) > tolerable_max_chars: + if len("".join(truncated_explanation)) > tolerable_max_chars and max_chars > 0: truncated_explanation = _truncate_by_char_count( truncated_explanation, max_chars ) else: truncated_char = False + if truncated_explanation == input_lines: + # No truncation happened, so we do not need to add any explanations + return truncated_explanation + truncated_line_count = len(input_lines) - len(truncated_explanation) if truncated_explanation[-1]: # Add ellipsis and take into account part-truncated final line diff --git a/src/_pytest/assertion/util.py b/src/_pytest/assertion/util.py index 4dc1af4af03..f35d83a6fe4 100644 --- a/src/_pytest/assertion/util.py +++ b/src/_pytest/assertion/util.py @@ -4,16 +4,15 @@ from __future__ import annotations import collections.abc -import os +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Mapping +from collections.abc import Sequence +from collections.abc import Set as AbstractSet import pprint -from typing import AbstractSet from typing import Any -from typing import Callable -from typing import Iterable from typing import Literal -from typing import Mapping from typing import Protocol -from typing import Sequence from unicodedata import normalize from _pytest import outcomes @@ -21,6 +20,7 @@ from _pytest._io.pprint import PrettyPrinter from _pytest._io.saferepr import saferepr from _pytest._io.saferepr import saferepr_unlimited +from _pytest.compat import running_on_ci from _pytest.config import Config @@ -43,6 +43,14 @@ def __call__(self, source: str, lexer: Literal["diff", "python"] = "python") -> """Apply highlighting to the given source.""" +def dummy_highlighter(source: str, lexer: Literal["diff", "python"] = "python") -> str: + """Dummy highlighter that returns the text unprocessed. + + Needed for _notin_text, as the diff gets post-processed to only show the "+" part. + """ + return source + + def format_explanation(explanation: str) -> str: r"""Format an explanation. @@ -123,7 +131,7 @@ def isdict(x: Any) -> bool: def isset(x: Any) -> bool: - return isinstance(x, (set, frozenset)) + return isinstance(x, set | frozenset) def isnamedtuple(obj: Any) -> bool: @@ -161,7 +169,7 @@ def has_default_eq( code_filename = obj.__eq__.__code__.co_filename if isattrs(obj): - return "attrs generated eq" in code_filename + return "attrs generated " in code_filename return code_filename == "" # data class return True @@ -242,7 +250,7 @@ def _compare_eq_any( ) -> list[str]: explanation = [] if istext(left) and istext(right): - explanation = _diff_text(left, right, verbose) + explanation = _diff_text(left, right, highlighter, verbose) else: from _pytest.python_api import ApproxBase @@ -274,7 +282,9 @@ def _compare_eq_any( return explanation -def _diff_text(left: str, right: str, verbose: int = 0) -> list[str]: +def _diff_text( + left: str, right: str, highlighter: _HighlightFunc, verbose: int = 0 +) -> list[str]: """Return the explanation for the diff between text. Unless --verbose is used this will skip leading and trailing @@ -315,10 +325,15 @@ def _diff_text(left: str, right: str, verbose: int = 0) -> list[str]: explanation += ["Strings contain only whitespace, escaping them using repr()"] # "right" is the expected base against which we compare "left", # see https://github.com/pytest-dev/pytest/issues/3333 - explanation += [ - line.strip("\n") - for line in ndiff(right.splitlines(keepends), left.splitlines(keepends)) - ] + explanation.extend( + highlighter( + "\n".join( + line.strip("\n") + for line in ndiff(right.splitlines(keepends), left.splitlines(keepends)) + ), + lexer="diff", + ).splitlines() + ) return explanation @@ -406,8 +421,7 @@ def _compare_eq_sequence( ] else: explanation += [ - "%s contains %d more items, first extra item: %s" - % (dir_with_more, len_diff, highlighter(extra)) + f"{dir_with_more} contains {len_diff} more items, first extra item: {highlighter(extra)}" ] return explanation @@ -510,8 +524,7 @@ def _compare_eq_dict( len_extra_left = len(extra_left) if len_extra_left: explanation.append( - "Left contains %d more item%s:" - % (len_extra_left, "" if len_extra_left == 1 else "s") + f"Left contains {len_extra_left} more item{'' if len_extra_left == 1 else 's'}:" ) explanation.extend( highlighter(pprint.pformat({k: left[k] for k in extra_left})).splitlines() @@ -520,8 +533,7 @@ def _compare_eq_dict( len_extra_right = len(extra_right) if len_extra_right: explanation.append( - "Right contains %d more item%s:" - % (len_extra_right, "" if len_extra_right == 1 else "s") + f"Right contains {len_extra_right} more item{'' if len_extra_right == 1 else 's'}:" ) explanation.extend( highlighter(pprint.pformat({k: right[k] for k in extra_right})).splitlines() @@ -589,7 +601,7 @@ def _notin_text(term: str, text: str, verbose: int = 0) -> list[str]: head = text[:index] tail = text[index + len(term) :] correct_text = head + tail - diff = _diff_text(text, correct_text, verbose) + diff = _diff_text(text, correct_text, dummy_highlighter, verbose) newdiff = [f"{saferepr(term, maxsize=42)} is contained here:"] for line in diff: if line.startswith("Skipping"): @@ -601,9 +613,3 @@ def _notin_text(term: str, text: str, verbose: int = 0) -> list[str]: else: newdiff.append(line) return newdiff - - -def running_on_ci() -> bool: - """Check if we're currently running on a CI system.""" - env_vars = ["CI", "BUILD_NUMBER"] - return any(var in os.environ for var in env_vars) diff --git a/src/_pytest/cacheprovider.py b/src/_pytest/cacheprovider.py old mode 100755 new mode 100644 index 20bb262e05d..4383f105af6 --- a/src/_pytest/cacheprovider.py +++ b/src/_pytest/cacheprovider.py @@ -5,6 +5,8 @@ # pytest-cache version. from __future__ import annotations +from collections.abc import Generator +from collections.abc import Iterable import dataclasses import errno import json @@ -12,8 +14,6 @@ from pathlib import Path import tempfile from typing import final -from typing import Generator -from typing import Iterable from .pathlib import resolve_from_str from .pathlib import rm_rf @@ -256,7 +256,7 @@ def pytest_make_collect_report( self, collector: nodes.Collector ) -> Generator[None, CollectReport, CollectReport]: res = yield - if isinstance(collector, (Session, Directory)): + if isinstance(collector, Session | Directory): # Sort any lf-paths to the beginning. lf_paths = self.lfplugin._last_failed_paths @@ -347,7 +347,7 @@ def get_last_failed_paths(self) -> set[Path]: return {x for x in result if x.exists()} def pytest_report_collectionfinish(self) -> str | None: - if self.active and self.config.getoption("verbose") >= 0: + if self.active and self.config.get_verbosity() >= 0: return f"run-last-failure: {self._report_status}" return None @@ -369,7 +369,7 @@ def pytest_collectreport(self, report: CollectReport) -> None: @hookimpl(wrapper=True, tryfirst=True) def pytest_collection_modifyitems( self, config: Config, items: list[nodes.Item] - ) -> Generator[None, None, None]: + ) -> Generator[None]: res = yield if not self.active: @@ -388,8 +388,8 @@ def pytest_collection_modifyitems( if not previously_failed: # Running a subset of all tests with recorded failures # only outside of it. - self._report_status = "%d known failures not in selected tests" % ( - len(self.lastfailed), + self._report_status = ( + f"{len(self.lastfailed)} known failures not in selected tests" ) else: if self.config.getoption("lf"): @@ -439,9 +439,7 @@ def __init__(self, config: Config) -> None: self.cached_nodeids = set(config.cache.get("cache/nodeids", [])) @hookimpl(wrapper=True, tryfirst=True) - def pytest_collection_modifyitems( - self, items: list[nodes.Item] - ) -> Generator[None, None, None]: + def pytest_collection_modifyitems(self, items: list[nodes.Item]) -> Generator[None]: res = yield if self.active: @@ -478,14 +476,17 @@ def pytest_sessionfinish(self) -> None: def pytest_addoption(parser: Parser) -> None: + """Add command-line options for cache functionality. + + :param parser: Parser object to add command-line options to. + """ group = parser.getgroup("general") group.addoption( "--lf", "--last-failed", action="store_true", dest="lf", - help="Rerun only the tests that failed " - "at the last run (or all if none failed)", + help="Rerun only the tests that failed at the last run (or all if none failed)", ) group.addoption( "--ff", @@ -549,6 +550,13 @@ def pytest_cmdline_main(config: Config) -> int | ExitCode | None: @hookimpl(tryfirst=True) def pytest_configure(config: Config) -> None: + """Configure cache system and register related plugins. + + Creates the Cache instance and registers the last-failed (LFPlugin) + and new-first (NFPlugin) plugins with the plugin manager. + + :param config: pytest configuration object. + """ config.cache = Cache.for_config(config, _ispytest=True) config.pluginmanager.register(LFPlugin(config), "lfplugin") config.pluginmanager.register(NFPlugin(config), "nfplugin") @@ -587,6 +595,16 @@ def pytest_report_header(config: Config) -> str | None: def cacheshow(config: Config, session: Session) -> int: + """Display cache contents when --cache-show is used. + + Shows cached values and directories matching the specified glob pattern + (default: '*'). Displays cache location, cached test results, and + any cached directories created by plugins. + + :param config: pytest configuration object. + :param session: pytest session object. + :returns: Exit code (0 for success). + """ from pprint import pformat assert config.cache is not None @@ -624,5 +642,5 @@ def cacheshow(config: Config, session: Session) -> int: # print("%s/" % p.relative_to(basedir)) if p.is_file(): key = str(p.relative_to(basedir)) - tw.line(f"{key} is a file of length {p.stat().st_size:d}") + tw.line(f"{key} is a file of length {p.stat().st_size}") return 0 diff --git a/src/_pytest/capture.py b/src/_pytest/capture.py index c4dfcc27552..6d98676be5f 100644 --- a/src/_pytest/capture.py +++ b/src/_pytest/capture.py @@ -5,6 +5,9 @@ import abc import collections +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator import contextlib import io from io import UnsupportedOperation @@ -15,12 +18,10 @@ from typing import Any from typing import AnyStr from typing import BinaryIO +from typing import cast from typing import Final from typing import final -from typing import Generator from typing import Generic -from typing import Iterable -from typing import Iterator from typing import Literal from typing import NamedTuple from typing import TextIO @@ -47,7 +48,7 @@ def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("general") - group._addoption( + group.addoption( "--capture", action="store", default="fd", @@ -55,7 +56,7 @@ def pytest_addoption(parser: Parser) -> None: choices=["fd", "sys", "no", "tee-sys"], help="Per-test capturing method: one of fd|sys|no|tee-sys", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-s", action="store_const", const="no", @@ -79,6 +80,23 @@ def _colorama_workaround() -> None: pass +def _readline_workaround() -> None: + """Ensure readline is imported early so it attaches to the correct stdio handles. + + This isn't a problem with the default GNU readline implementation, but in + some configurations, Python uses libedit instead (on macOS, and for prebuilt + binaries such as used by uv). + + In theory this is only needed if readline.backend == "libedit", but the + workaround consists of importing readline here, so we already worked around + the issue by the time we could check if we need to. + """ + try: + import readline # noqa: F401 + except ImportError: + pass + + def _windowsconsoleio_workaround(stream: TextIO) -> None: """Workaround for Windows Unicode console handling. @@ -135,11 +153,12 @@ def _reopen_stdio(f, mode): @hookimpl(wrapper=True) -def pytest_load_initial_conftests(early_config: Config) -> Generator[None, None, None]: +def pytest_load_initial_conftests(early_config: Config) -> Generator[None]: ns = early_config.known_args_namespace if ns.capture == "fd": _windowsconsoleio_workaround(sys.stdout) _colorama_workaround() + _readline_workaround() pluginmanager = early_config.pluginmanager capman = CaptureManager(ns.capture) pluginmanager.register(capman, "capturemanager") @@ -177,7 +196,8 @@ def name(self) -> str: def mode(self) -> str: # TextIOWrapper doesn't expose a mode, but at least some of our # tests check it. - return self.buffer.mode.replace("b", "") + assert hasattr(self.buffer, "mode") + return cast(str, self.buffer.mode.replace("b", "")) class CaptureIO(io.TextIOWrapper): @@ -202,6 +222,7 @@ def write(self, s: str) -> int: class DontReadFromInput(TextIO): @property def encoding(self) -> str: + assert sys.__stdin__ is not None return sys.__stdin__.encoding def read(self, size: int = -1) -> str: @@ -357,7 +378,7 @@ def repr(self, class_name: str) -> str: return "<{} {} _old={} _state={!r} tmpfile={!r}>".format( class_name, self.name, - hasattr(self, "_old") and repr(self._old) or "", + (hasattr(self, "_old") and repr(self._old)) or "", self._state, self.tmpfile, ) @@ -366,16 +387,16 @@ def __repr__(self) -> str: return "<{} {} _old={} _state={!r} tmpfile={!r}>".format( self.__class__.__name__, self.name, - hasattr(self, "_old") and repr(self._old) or "", + (hasattr(self, "_old") and repr(self._old)) or "", self._state, self.tmpfile, ) def _assert_state(self, op: str, states: tuple[str, ...]) -> None: - assert ( - self._state in states - ), "cannot {} in state {!r}: expected one of {}".format( - op, self._state, ", ".join(states) + assert self._state in states, ( + "cannot {} in state {!r}: expected one of {}".format( + op, self._state, ", ".join(states) + ) ) def start(self) -> None: @@ -489,10 +510,10 @@ def __repr__(self) -> str: ) def _assert_state(self, op: str, states: tuple[str, ...]) -> None: - assert ( - self._state in states - ), "cannot {} in state {!r}: expected one of {}".format( - op, self._state, ", ".join(states) + assert self._state in states, ( + "cannot {} in state {!r}: expected one of {}".format( + op, self._state, ", ".join(states) + ) ) def start(self) -> None: @@ -549,7 +570,7 @@ def snap(self) -> bytes: res = self.tmpfile.buffer.read() self.tmpfile.seek(0) self.tmpfile.truncate() - return res + return res # type: ignore[return-value] def writeorg(self, data: bytes) -> None: """Write to original file descriptor.""" @@ -817,7 +838,7 @@ def resume_fixture(self) -> None: # Helper context managers @contextlib.contextmanager - def global_and_fixture_disabled(self) -> Generator[None, None, None]: + def global_and_fixture_disabled(self) -> Generator[None]: """Context manager to temporarily disable global and current fixture capturing.""" do_fixture = self._capture_fixture and self._capture_fixture._is_started() if do_fixture: @@ -834,7 +855,7 @@ def global_and_fixture_disabled(self) -> Generator[None, None, None]: self.resume_fixture() @contextlib.contextmanager - def item_capture(self, when: str, item: Item) -> Generator[None, None, None]: + def item_capture(self, when: str, item: Item) -> Generator[None]: self.resume_global_capture() self.activate_fixture() try: @@ -869,17 +890,17 @@ def pytest_make_collect_report( return rep @hookimpl(wrapper=True) - def pytest_runtest_setup(self, item: Item) -> Generator[None, None, None]: + def pytest_runtest_setup(self, item: Item) -> Generator[None]: with self.item_capture("setup", item): return (yield) @hookimpl(wrapper=True) - def pytest_runtest_call(self, item: Item) -> Generator[None, None, None]: + def pytest_runtest_call(self, item: Item) -> Generator[None]: with self.item_capture("call", item): return (yield) @hookimpl(wrapper=True) - def pytest_runtest_teardown(self, item: Item) -> Generator[None, None, None]: + def pytest_runtest_teardown(self, item: Item) -> Generator[None]: with self.item_capture("teardown", item): return (yield) @@ -901,11 +922,13 @@ def __init__( captureclass: type[CaptureBase[AnyStr]], request: SubRequest, *, + config: dict[str, Any] | None = None, _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) self.captureclass: type[CaptureBase[AnyStr]] = captureclass self.request = request + self._config = config if config else {} self._capture: MultiCapture[AnyStr] | None = None self._captured_out: AnyStr = self.captureclass.EMPTY_BUFFER self._captured_err: AnyStr = self.captureclass.EMPTY_BUFFER @@ -914,8 +937,8 @@ def _start(self) -> None: if self._capture is None: self._capture = MultiCapture( in_=None, - out=self.captureclass(1), - err=self.captureclass(2), + out=self.captureclass(1, **self._config), + err=self.captureclass(2, **self._config), ) self._capture.start_capturing() @@ -961,7 +984,7 @@ def _is_started(self) -> bool: return False @contextlib.contextmanager - def disabled(self) -> Generator[None, None, None]: + def disabled(self) -> Generator[None]: """Temporarily disable capturing while inside the ``with`` block.""" capmanager: CaptureManager = self.request.config.pluginmanager.getplugin( "capturemanager" @@ -974,7 +997,7 @@ def disabled(self) -> Generator[None, None, None]: @fixture -def capsys(request: SubRequest) -> Generator[CaptureFixture[str], None, None]: +def capsys(request: SubRequest) -> Generator[CaptureFixture[str]]: r"""Enable text capturing of writes to ``sys.stdout`` and ``sys.stderr``. The captured output is made available via ``capsys.readouterr()`` method @@ -1002,7 +1025,42 @@ def test_output(capsys): @fixture -def capsysbinary(request: SubRequest) -> Generator[CaptureFixture[bytes], None, None]: +def capteesys(request: SubRequest) -> Generator[CaptureFixture[str]]: + r"""Enable simultaneous text capturing and pass-through of writes + to ``sys.stdout`` and ``sys.stderr`` as defined by ``--capture=``. + + + The captured output is made available via ``capteesys.readouterr()`` method + calls, which return a ``(out, err)`` namedtuple. + ``out`` and ``err`` will be ``text`` objects. + + The output is also passed-through, allowing it to be "live-printed", + reported, or both as defined by ``--capture=``. + + Returns an instance of :class:`CaptureFixture[str] `. + + Example: + + .. code-block:: python + + def test_output(capteesys): + print("hello") + captured = capteesys.readouterr() + assert captured.out == "hello\n" + """ + capman: CaptureManager = request.config.pluginmanager.getplugin("capturemanager") + capture_fixture = CaptureFixture( + SysCapture, request, config=dict(tee=True), _ispytest=True + ) + capman.set_fixture(capture_fixture) + capture_fixture._start() + yield capture_fixture + capture_fixture.close() + capman.unset_fixture() + + +@fixture +def capsysbinary(request: SubRequest) -> Generator[CaptureFixture[bytes]]: r"""Enable bytes capturing of writes to ``sys.stdout`` and ``sys.stderr``. The captured output is made available via ``capsysbinary.readouterr()`` @@ -1030,7 +1088,7 @@ def test_output(capsysbinary): @fixture -def capfd(request: SubRequest) -> Generator[CaptureFixture[str], None, None]: +def capfd(request: SubRequest) -> Generator[CaptureFixture[str]]: r"""Enable text capturing of writes to file descriptors ``1`` and ``2``. The captured output is made available via ``capfd.readouterr()`` method @@ -1058,7 +1116,7 @@ def test_system_echo(capfd): @fixture -def capfdbinary(request: SubRequest) -> Generator[CaptureFixture[bytes], None, None]: +def capfdbinary(request: SubRequest) -> Generator[CaptureFixture[bytes]]: r"""Enable bytes capturing of writes to file descriptors ``1`` and ``2``. The captured output is made available via ``capfd.readouterr()`` method diff --git a/src/_pytest/compat.py b/src/_pytest/compat.py index 614848e0dba..72c3d0918fb 100644 --- a/src/_pytest/compat.py +++ b/src/_pytest/compat.py @@ -1,25 +1,28 @@ # mypy: allow-untyped-defs -"""Python version compatibility code.""" +"""Python version compatibility code and random general utilities.""" from __future__ import annotations -import dataclasses +from collections.abc import Callable import enum import functools import inspect from inspect import Parameter -from inspect import signature +from inspect import Signature import os from pathlib import Path import sys from typing import Any -from typing import Callable from typing import Final from typing import NoReturn import py +if sys.version_info >= (3, 14): + from annotationlib import Format + + #: constant to prepare valuing pylib path replacements/lazy proxies later on # intended for removal in pytest 8.0 or 9.0 @@ -43,11 +46,6 @@ class NotSetType(enum.Enum): # fmt: on -def is_generator(func: object) -> bool: - genfunc = inspect.isgeneratorfunction(func) - return genfunc and not iscoroutinefunction(func) - - def iscoroutinefunction(func: object) -> bool: """Return True if func is a coroutine function (a function defined with async def syntax, and doesn't contain yield), or a function decorated with @@ -66,6 +64,13 @@ def is_async_function(func: object) -> bool: return iscoroutinefunction(func) or inspect.isasyncgenfunction(func) +def signature(obj: Callable[..., Any]) -> Signature: + """Return signature without evaluating annotations.""" + if sys.version_info >= (3, 14): + return inspect.signature(obj, annotation_format=Format.STRING) + return inspect.signature(obj) + + def getlocation(function, curdir: str | os.PathLike[str] | None = None) -> str: function = get_real_func(function) fn = Path(inspect.getfile(function)) @@ -76,8 +81,8 @@ def getlocation(function, curdir: str | os.PathLike[str] | None = None) -> str: except ValueError: pass else: - return "%s:%d" % (relfn, lineno + 1) - return "%s:%d" % (fn, lineno + 1) + return f"{relfn}:{lineno + 1}" + return f"{fn}:{lineno + 1}" def num_mock_patch_args(function) -> int: @@ -128,7 +133,7 @@ def getfuncargnames( # creates a tuple of the names of the parameters that don't have # defaults. try: - parameters = signature(function).parameters + parameters = signature(function).parameters.values() except (ValueError, TypeError) as e: from _pytest.outcomes import fail @@ -139,7 +144,7 @@ def getfuncargnames( arg_names = tuple( p.name - for p in parameters.values() + for p in parameters if ( p.kind is Parameter.POSITIONAL_OR_KEYWORD or p.kind is Parameter.KEYWORD_ONLY @@ -150,9 +155,9 @@ def getfuncargnames( name = function.__name__ # If this function should be treated as a bound method even though - # it's passed as an unbound method or function, remove the first - # parameter name. - if ( + # it's passed as an unbound method or function, and its first parameter + # wasn't defined as positional only, remove the first parameter name. + if not any(p.kind is Parameter.POSITIONAL_ONLY for p in parameters) and ( # Not using `getattr` because we don't want to resolve the staticmethod. # Not using `cls.__dict__` because we want to check the entire MRO. cls @@ -210,59 +215,16 @@ def ascii_escaped(val: bytes | str) -> str: return ret.translate(_non_printable_ascii_translate_table) -@dataclasses.dataclass -class _PytestWrapper: - """Dummy wrapper around a function object for internal use only. - - Used to correctly unwrap the underlying function object when we are - creating fixtures, because we wrap the function object ourselves with a - decorator to issue warnings when the fixture function is called directly. - """ - - obj: Any - - def get_real_func(obj): """Get the real function object of the (possibly) wrapped object by - functools.wraps or functools.partial.""" - start_obj = obj - for i in range(100): - # __pytest_wrapped__ is set by @pytest.fixture when wrapping the fixture function - # to trigger a warning if it gets called directly instead of by pytest: we don't - # want to unwrap further than this otherwise we lose useful wrappings like @mock.patch (#3774) - new_obj = getattr(obj, "__pytest_wrapped__", None) - if isinstance(new_obj, _PytestWrapper): - obj = new_obj.obj - break - new_obj = getattr(obj, "__wrapped__", None) - if new_obj is None: - break - obj = new_obj - else: - from _pytest._io.saferepr import saferepr + :func:`functools.wraps`, or :func:`functools.partial`.""" + obj = inspect.unwrap(obj) - raise ValueError( - f"could not find real function of {saferepr(start_obj)}\nstopped at {saferepr(obj)}" - ) if isinstance(obj, functools.partial): obj = obj.func return obj -def get_real_method(obj, holder): - """Attempt to obtain the real function object that might be wrapping - ``obj``, while at the same time returning a bound method to ``holder`` if - the original object was a bound method.""" - try: - is_method = hasattr(obj, "__func__") - obj = get_real_func(obj) - except Exception: # pragma: no cover - return obj - if is_method and hasattr(obj, "__get__") and callable(obj.__get__): - obj = obj.__get__(holder) - return obj - - def getimfunc(func): try: return func.__func__ @@ -316,36 +278,37 @@ def get_user_id() -> int | None: return uid if uid != ERROR else None -# Perform exhaustiveness checking. -# -# Consider this example: -# -# MyUnion = Union[int, str] -# -# def handle(x: MyUnion) -> int { -# if isinstance(x, int): -# return 1 -# elif isinstance(x, str): -# return 2 -# else: -# raise Exception('unreachable') -# -# Now suppose we add a new variant: -# -# MyUnion = Union[int, str, bytes] -# -# After doing this, we must remember ourselves to go and update the handle -# function to handle the new variant. -# -# With `assert_never` we can do better: -# -# // raise Exception('unreachable') -# return assert_never(x) -# -# Now, if we forget to handle the new variant, the type-checker will emit a -# compile-time error, instead of the runtime error we would have gotten -# previously. -# -# This also work for Enums (if you use `is` to compare) and Literals. -def assert_never(value: NoReturn) -> NoReturn: - assert False, f"Unhandled value: {value} ({type(value).__name__})" +if sys.version_info >= (3, 11): + from typing import assert_never +else: + + def assert_never(value: NoReturn) -> NoReturn: + assert False, f"Unhandled value: {value} ({type(value).__name__})" + + +class CallableBool: + """ + A bool-like object that can also be called, returning its true/false value. + + Used for backwards compatibility in cases where something was supposed to be a method + but was implemented as a simple attribute by mistake (see `TerminalReporter.isatty`). + + Do not use in new code. + """ + + def __init__(self, value: bool) -> None: + self._value = value + + def __bool__(self) -> bool: + return self._value + + def __call__(self) -> bool: + return self._value + + +def running_on_ci() -> bool: + """Check if we're currently running on a CI system.""" + # Only enable CI mode if one of these env variables is defined and non-empty. + # Note: review `regendoc` tox env in case this list is changed. + env_vars = ["CI", "BUILD_NUMBER"] + return any(os.environ.get(var) for var in env_vars) diff --git a/src/_pytest/config/__init__.py b/src/_pytest/config/__init__.py index 0c1850df503..a027dbc02a4 100644 --- a/src/_pytest/config/__init__.py +++ b/src/_pytest/config/__init__.py @@ -1,10 +1,19 @@ # mypy: allow-untyped-defs -"""Command line options, ini-file and conftest.py processing.""" +"""Command line options, config-file and conftest.py processing.""" from __future__ import annotations import argparse +import builtins import collections.abc +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import MutableMapping +from collections.abc import Sequence +import contextlib import copy import dataclasses import enum @@ -21,17 +30,11 @@ import types from types import FunctionType from typing import Any -from typing import Callable from typing import cast from typing import Final from typing import final -from typing import Generator from typing import IO -from typing import Iterable -from typing import Iterator -from typing import Sequence from typing import TextIO -from typing import Type from typing import TYPE_CHECKING import warnings @@ -45,6 +48,7 @@ from .compat import PathAwareHookProxy from .exceptions import PrintHelp as PrintHelp from .exceptions import UsageError as UsageError +from .findpaths import ConfigValue from .findpaths import determine_setup from _pytest import __version__ import _pytest._code @@ -52,7 +56,9 @@ from _pytest._code import filter_traceback from _pytest._code.code import TracebackStyle from _pytest._io import TerminalWriter +from _pytest.compat import assert_never from _pytest.config.argparsing import Argument +from _pytest.config.argparsing import FILE_OR_DIR from _pytest.config.argparsing import Parser import _pytest.deprecated import _pytest.hookspec @@ -70,10 +76,10 @@ if TYPE_CHECKING: + from _pytest.assertion.rewrite import AssertionRewritingHook from _pytest.cacheprovider import Cache from _pytest.terminal import TerminalReporter - _PluggyPlugin = object """A type to represent plugin objects. @@ -110,6 +116,8 @@ class ExitCode(enum.IntEnum): #: pytest couldn't find tests. NO_TESTS_COLLECTED = 5 + __module__ = "pytest" + class ConftestImportFailure(Exception): def __init__( @@ -136,6 +144,29 @@ def filter_traceback_for_conftest_import_failure( return filter_traceback(entry) and "importlib" not in str(entry.path).split(os.sep) +def print_conftest_import_error(e: ConftestImportFailure, file: TextIO) -> None: + exc_info = ExceptionInfo.from_exception(e.cause) + tw = TerminalWriter(file) + tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) + exc_info.traceback = exc_info.traceback.filter( + filter_traceback_for_conftest_import_failure + ) + exc_repr = ( + exc_info.getrepr(style="short", chain=False) + if exc_info.traceback + else exc_info.exconly() + ) + formatted_tb = str(exc_repr) + for line in formatted_tb.splitlines(): + tw.line(line.rstrip(), red=True) + + +def print_usage_error(e: UsageError, file: TextIO) -> None: + tw = TerminalWriter(file) + for msg in e.args: + tw.line(f"ERROR: {msg}\n", red=True) + + def main( args: list[str] | os.PathLike[str] | None = None, plugins: Sequence[str | _PluggyPlugin] | None = None, @@ -149,40 +180,31 @@ def main( :returns: An exit code. """ + # Handle a single `--version` argument early to avoid starting up the entire pytest infrastructure. + new_args = sys.argv[1:] if args is None else args + if isinstance(new_args, Sequence) and new_args.count("--version") == 1: + sys.stdout.write(f"pytest {__version__}\n") + return ExitCode.OK + old_pytest_version = os.environ.get("PYTEST_VERSION") try: os.environ["PYTEST_VERSION"] = __version__ try: - config = _prepareconfig(args, plugins) + config = _prepareconfig(new_args, plugins) except ConftestImportFailure as e: - exc_info = ExceptionInfo.from_exception(e.cause) - tw = TerminalWriter(sys.stderr) - tw.line(f"ImportError while loading conftest '{e.path}'.", red=True) - exc_info.traceback = exc_info.traceback.filter( - filter_traceback_for_conftest_import_failure - ) - exc_repr = ( - exc_info.getrepr(style="short", chain=False) - if exc_info.traceback - else exc_info.exconly() - ) - formatted_tb = str(exc_repr) - for line in formatted_tb.splitlines(): - tw.line(line.rstrip(), red=True) + print_conftest_import_error(e, file=sys.stderr) return ExitCode.USAGE_ERROR - else: + + try: + ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config) try: - ret: ExitCode | int = config.hook.pytest_cmdline_main(config=config) - try: - return ExitCode(ret) - except ValueError: - return ret - finally: - config._ensure_unconfigure() + return ExitCode(ret) + except ValueError: + return ret + finally: + config._ensure_unconfigure() except UsageError as e: - tw = TerminalWriter(sys.stderr) - for msg in e.args: - tw.line(f"ERROR: {msg}\n", red=True) + print_usage_error(e, file=sys.stderr) return ExitCode.USAGE_ERROR finally: if old_pytest_version is None: @@ -261,42 +283,42 @@ def directory_arg(path: str, optname: str) -> str: "junitxml", "doctest", "cacheprovider", - "freeze_support", "setuponly", "setupplan", "stepwise", + "unraisableexception", + "threadexception", "warnings", "logging", "reports", - "python_path", - "unraisableexception", - "threadexception", "faulthandler", + "subtests", ) -builtin_plugins = set(default_plugins) -builtin_plugins.add("pytester") -builtin_plugins.add("pytester_assertions") +builtin_plugins = { + *default_plugins, + "pytester", + "pytester_assertions", + "terminalprogress", +} def get_config( - args: list[str] | None = None, + args: Iterable[str] | None = None, plugins: Sequence[str | _PluggyPlugin] | None = None, ) -> Config: - # subsequent calls to main will create a fresh instance + # Subsequent calls to main will create a fresh instance. pluginmanager = PytestPluginManager() - config = Config( - pluginmanager, - invocation_params=Config.InvocationParams( - args=args or (), - plugins=plugins, - dir=pathlib.Path.cwd(), - ), + invocation_params = Config.InvocationParams( + args=args or (), + plugins=plugins, + dir=pathlib.Path.cwd(), ) + config = Config(pluginmanager, invocation_params=invocation_params) - if args is not None: + if invocation_params.args: # Handle any "-p no:plugin" args. - pluginmanager.consider_preparse(args, exclude_only=True) + pluginmanager.consider_preparse(invocation_params.args, exclude_only=True) for spec in default_plugins: pluginmanager.import_plugin(spec) @@ -316,12 +338,10 @@ def get_plugin_manager() -> PytestPluginManager: def _prepareconfig( - args: list[str] | os.PathLike[str] | None = None, + args: list[str] | os.PathLike[str], plugins: Sequence[str | _PluggyPlugin] | None = None, ) -> Config: - if args is None: - args = sys.argv[1:] - elif isinstance(args, os.PathLike): + if isinstance(args, os.PathLike): args = [os.fspath(args)] elif not isinstance(args, list): msg = ( # type:ignore[unreachable] @@ -329,8 +349,8 @@ def _prepareconfig( ) raise TypeError(msg.format(args, type(args))) - config = get_config(args, plugins) - pluginmanager = config.pluginmanager + initial_config = get_config(args, plugins) + pluginmanager = initial_config.pluginmanager try: if plugins: for plugin in plugins: @@ -338,12 +358,12 @@ def _prepareconfig( pluginmanager.consider_pluginarg(plugin) else: pluginmanager.register(plugin) - config = pluginmanager.hook.pytest_cmdline_parse( + config: Config = pluginmanager.hook.pytest_cmdline_parse( pluginmanager=pluginmanager, args=args ) return config except BaseException: - config._ensure_unconfigure() + initial_config._ensure_unconfigure() raise @@ -361,7 +381,7 @@ def _get_legacy_hook_marks( opt_names: tuple[str, ...], ) -> dict[str, bool]: if TYPE_CHECKING: - # abuse typeguard from importlib to avoid massive method type union thats lacking a alias + # abuse typeguard from importlib to avoid massive method type union that's lacking an alias assert inspect.isroutine(method) known_marks: set[str] = {m.name for m in getattr(method, "pytestmark", [])} must_warn: list[str] = [] @@ -398,7 +418,8 @@ class PytestPluginManager(PluginManager): """ def __init__(self) -> None: - import _pytest.assertion + from _pytest.assertion import DummyRewriteHook + from _pytest.assertion import RewriteHook super().__init__("pytest") @@ -444,7 +465,7 @@ def __init__(self) -> None: self.enable_tracing() # Config._consider_importhook will set a real object if required. - self.rewrite_hook = _pytest.assertion.DummyRewriteHook() + self.rewrite_hook: RewriteHook = DummyRewriteHook() # Used to know when we are importing conftests after the pytest_configure stage. self._configured = False @@ -470,9 +491,10 @@ def parse_hookimpl_opts( if not inspect.isroutine(method): return None # Collect unmarked hooks as long as they have the `pytest_' prefix. - return _get_legacy_hook_marks( # type: ignore[return-value] + legacy = _get_legacy_hook_marks( method, "impl", ("tryfirst", "trylast", "optionalhook", "hookwrapper") ) + return cast(HookimplOpts, legacy) def parse_hookspec_opts(self, module_or_class, name: str) -> HookspecOpts | None: """:meta private:""" @@ -480,11 +502,10 @@ def parse_hookspec_opts(self, module_or_class, name: str) -> HookspecOpts | None if opts is None: method = getattr(module_or_class, name) if name.startswith("pytest_"): - opts = _get_legacy_hook_marks( # type: ignore[assignment] - method, - "spec", - ("firstresult", "historic"), + legacy = _get_legacy_hook_marks( + method, "spec", ("firstresult", "historic") ) + opts = cast(HookspecOpts, legacy) return opts def register(self, plugin: _PluggyPlugin, name: str | None = None) -> str | None: @@ -793,6 +814,12 @@ def consider_pluginarg(self, arg: str) -> None: if name in essential_plugins: raise UsageError(f"plugin {name} cannot be disabled") + if name.endswith("conftest.py"): + raise UsageError( + f"Blocking conftest files using -p is not supported: -p no:{name}\n" + "conftest.py files are not plugins and cannot be disabled via -p.\n" + ) + # PR #4304: remove stepwise if cacheprovider is blocked. if name == "cacheprovider": self.set_blocked("stepwise") @@ -840,9 +867,9 @@ def import_plugin(self, modname: str, consider_entry_points: bool = False) -> No # "terminal" or "capture". Those plugins are registered under their # basename for historic purposes but must be imported with the # _pytest prefix. - assert isinstance( - modname, str - ), f"module name as text required, got {modname!r}" + assert isinstance(modname, str), ( + f"module name as text required, got {modname!r}" + ) if self.is_blocked(modname) or self.get_plugin(modname) is not None: return @@ -962,6 +989,30 @@ def _iter_rewritable_modules(package_files: Iterable[str]) -> Iterator[str]: yield from _iter_rewritable_modules(new_package_files) +class _DeprecatedInicfgProxy(MutableMapping[str, Any]): + """Compatibility proxy for the deprecated Config.inicfg.""" + + __slots__ = ("_config",) + + def __init__(self, config: Config) -> None: + self._config = config + + def __getitem__(self, key: str) -> Any: + return self._config._inicfg[key].value + + def __setitem__(self, key: str, value: Any) -> None: + self._config._inicfg[key] = ConfigValue(value, origin="override", mode="toml") + + def __delitem__(self, key: str) -> None: + del self._config._inicfg[key] + + def __iter__(self) -> Iterator[str]: + return iter(self._config._inicfg) + + def __len__(self) -> int: + return len(self._config._inicfg) + + @final class Config: """Access to configuration values, pluginmanager and plugin hooks. @@ -986,7 +1037,7 @@ class InvocationParams: .. note:: Note that the environment variable ``PYTEST_ADDOPTS`` and the ``addopts`` - ini option are handled by pytest, not being included in the ``args`` attribute. + configuration option are handled by pytest, not being included in the ``args`` attribute. Plugins accessing ``InvocationParams`` must be aware of that. """ @@ -996,7 +1047,7 @@ class InvocationParams: plugins: Sequence[str | _PluggyPlugin] | None """Extra plugins, might be `None`.""" dir: pathlib.Path - """The directory from which :func:`pytest.main` was invoked. :type: pathlib.Path""" + """The directory from which :func:`pytest.main` was invoked.""" def __init__( self, @@ -1032,9 +1083,6 @@ def __init__( *, invocation_params: InvocationParams | None = None, ) -> None: - from .argparsing import FILE_OR_DIR - from .argparsing import Parser - if invocation_params is None: invocation_params = self.InvocationParams( args=(), plugins=None, dir=pathlib.Path.cwd() @@ -1052,9 +1100,8 @@ def __init__( :type: InvocationParams """ - _a = FILE_OR_DIR self._parser = Parser( - usage=f"%(prog)s [options] [{_a}] [{_a}] [...]", + usage=f"%(prog)s [options] [{FILE_OR_DIR}] [{FILE_OR_DIR}] [...]", processopt=self._processopt, _ispytest=True, ) @@ -1076,9 +1123,8 @@ def __init__( self.trace = self.pluginmanager.trace.root.get("config") self.hook: pluggy.HookRelay = PathAwareHookProxy(self.pluginmanager.hook) # type: ignore[assignment] self._inicache: dict[str, Any] = {} - self._override_ini: Sequence[str] = () self._opt2dest: dict[str, str] = {} - self._cleanup: list[Callable[[], None]] = [] + self._cleanup_stack = contextlib.ExitStack() self.pluginmanager.register(self, "pytestconfig") self._configured = False self.hook.pytest_addoption.call_historic( @@ -1087,12 +1133,14 @@ def __init__( self.args_source = Config.ArgsSource.ARGS self.args: list[str] = [] + @property + def inicfg(self) -> _DeprecatedInicfgProxy: + return _DeprecatedInicfgProxy(self) + @property def rootpath(self) -> pathlib.Path: """The path to the :ref:`rootdir `. - :type: pathlib.Path - .. versionadded:: 6.1 """ return self._rootpath @@ -1107,24 +1155,28 @@ def inipath(self) -> pathlib.Path | None: def add_cleanup(self, func: Callable[[], None]) -> None: """Add a function to be called when the config object gets out of - use (usually coinciding with pytest_unconfigure).""" - self._cleanup.append(func) + use (usually coinciding with pytest_unconfigure). + """ + self._cleanup_stack.callback(func) def _do_configure(self) -> None: assert not self._configured self._configured = True - with warnings.catch_warnings(): - warnings.simplefilter("default") - self.hook.pytest_configure.call_historic(kwargs=dict(config=self)) + self.hook.pytest_configure.call_historic(kwargs=dict(config=self)) def _ensure_unconfigure(self) -> None: - if self._configured: - self._configured = False - self.hook.pytest_unconfigure(config=self) - self.hook.pytest_configure._call_history = [] - while self._cleanup: - fin = self._cleanup.pop() - fin() + try: + if self._configured: + self._configured = False + try: + self.hook.pytest_unconfigure(config=self) + finally: + self.hook.pytest_configure._call_history = [] + finally: + try: + self._cleanup_stack.close() + finally: + self._cleanup_stack = contextlib.ExitStack() def get_terminal_writer(self) -> TerminalWriter: terminalreporter: TerminalReporter | None = self.pluginmanager.get_plugin( @@ -1139,17 +1191,19 @@ def pytest_cmdline_parse( try: self.parse(args) except UsageError: - # Handle --version and --help here in a minimal fashion. + # Handle `--version --version` and `--help` here in a minimal fashion. # This gets done via helpconfig normally, but its # pytest_cmdline_main is not called in case of errors. if getattr(self.option, "version", False) or "--version" in args: - from _pytest.helpconfig import showversion + from _pytest.helpconfig import show_version_verbose - showversion(self) + # Note that `--version` (single argument) is handled early by `Config.main()`, so the only + # way we are reaching this point is via `--version --version`. + show_version_verbose(self) elif ( getattr(self.option, "help", False) or "--help" in args or "-h" in args ): - self._parser._getparser().print_help() + self._parser.optparser.print_help() sys.stdout.write( "\nNOTE: displaying only minimal help due to UsageError.\n\n" ) @@ -1179,12 +1233,16 @@ def notify_exception( def cwd_relative_nodeid(self, nodeid: str) -> str: # nodeid's are relative to the rootpath, compute relative to cwd. if self.invocation_params.dir != self.rootpath: - fullpath = self.rootpath / nodeid - nodeid = bestrelpath(self.invocation_params.dir, fullpath) + base_path_part, *nodeid_part = nodeid.split("::") + # Only process path part + fullpath = self.rootpath / base_path_part + relative_path = bestrelpath(self.invocation_params.dir, fullpath) + + nodeid = "::".join([relative_path, *nodeid_part]) return nodeid @classmethod - def fromdictargs(cls, option_dict, args) -> Config: + def fromdictargs(cls, option_dict: Mapping[str, Any], args: list[str]) -> Config: """Constructor usable for subprocesses.""" config = get_config(args) config.option.__dict__.update(option_dict) @@ -1207,7 +1265,7 @@ def pytest_load_initial_conftests(self, early_config: Config) -> None: # early_config.args it not set yet. But we need it for # discovering the initial conftests. So "pre-run" the logic here. # It will be done for real in `parse()`. - args, args_source = early_config._decide_args( + args, _args_source = early_config._decide_args( args=early_config.known_args_namespace.file_or_dir, pyargs=early_config.known_args_namespace.pyargs, testpaths=early_config.getini("testpaths"), @@ -1228,40 +1286,18 @@ def pytest_load_initial_conftests(self, early_config: Config) -> None: ), ) - def _initini(self, args: Sequence[str]) -> None: - ns, unknown_args = self._parser.parse_known_and_unknown_args( - args, namespace=copy.copy(self.option) - ) - rootpath, inipath, inicfg = determine_setup( - inifile=ns.inifilename, - args=ns.file_or_dir + unknown_args, - rootdir_cmd_arg=ns.rootdir or None, - invocation_dir=self.invocation_params.dir, - ) - self._rootpath = rootpath - self._inipath = inipath - self.inicfg = inicfg - self._parser.extra_info["rootdir"] = str(self.rootpath) - self._parser.extra_info["inifile"] = str(self.inipath) - self._parser.addini("addopts", "Extra command line options", "args") - self._parser.addini("minversion", "Minimally required pytest version") - self._parser.addini( - "required_plugins", - "Plugins that must be present for pytest to run", - type="args", - default=[], - ) - self._override_ini = ns.override_ini or () - - def _consider_importhook(self, args: Sequence[str]) -> None: + def _consider_importhook(self) -> None: """Install the PEP 302 import hook if using assertion rewriting. Needs to parse the --assert= option from the commandline and find all the installed plugins to mark them for rewriting by the importhook. """ - ns, unknown_args = self._parser.parse_known_and_unknown_args(args) - mode = getattr(ns, "assertmode", "plain") + mode = getattr(self.known_args_namespace, "assertmode", "plain") + + disable_autoload = getattr( + self.known_args_namespace, "disable_plugin_autoload", False + ) or bool(os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD")) if mode == "rewrite": import _pytest.assertion @@ -1270,16 +1306,18 @@ def _consider_importhook(self, args: Sequence[str]) -> None: except SystemError: mode = "plain" else: - self._mark_plugins_for_rewrite(hook) + self._mark_plugins_for_rewrite(hook, disable_autoload) self._warn_about_missing_assertion(mode) - def _mark_plugins_for_rewrite(self, hook) -> None: + def _mark_plugins_for_rewrite( + self, hook: AssertionRewritingHook, disable_autoload: bool + ) -> None: """Given an importhook, mark for rewrite any top-level modules or packages in the distribution package for all pytest plugins.""" self.pluginmanager.rewrite_hook = hook - if os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): + if disable_autoload: # We don't autoload from distribution package entry points, # no need to continue. return @@ -1294,15 +1332,27 @@ def _mark_plugins_for_rewrite(self, hook) -> None: for name in _iter_rewritable_modules(package_files): hook.mark_rewrite(name) + def _configure_python_path(self) -> None: + # `pythonpath = a b` will set `sys.path` to `[a, b, x, y, z, ...]` + for path in reversed(self.getini("pythonpath")): + sys.path.insert(0, str(path)) + self.add_cleanup(self._unconfigure_python_path) + + def _unconfigure_python_path(self) -> None: + for path in self.getini("pythonpath"): + path_str = str(path) + if path_str in sys.path: + sys.path.remove(path_str) + def _validate_args(self, args: list[str], via: str) -> list[str]: """Validate known args.""" - self._parser._config_source_hint = via # type: ignore + self._parser.extra_info["config source"] = via try: self._parser.parse_known_and_unknown_args( args, namespace=copy.copy(self.option) ) finally: - del self._parser._config_source_hint # type: ignore + self._parser.extra_info.pop("config source", None) return args @@ -1351,64 +1401,10 @@ def _decide_args( result = [str(invocation_dir)] return result, source - def _preparse(self, args: list[str], addopts: bool = True) -> None: - if addopts: - env_addopts = os.environ.get("PYTEST_ADDOPTS", "") - if len(env_addopts): - args[:] = ( - self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") - + args - ) - self._initini(args) - if addopts: - args[:] = ( - self._validate_args(self.getini("addopts"), "via addopts config") + args - ) - - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.option) - ) - self._checkversion() - self._consider_importhook(args) - self.pluginmanager.consider_preparse(args, exclude_only=False) - if not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD"): - # Don't autoload from distribution package entry point. Only - # explicitly specified plugins are going to be loaded. - self.pluginmanager.load_setuptools_entrypoints("pytest11") - self.pluginmanager.consider_env() - - self.known_args_namespace = self._parser.parse_known_args( - args, namespace=copy.copy(self.known_args_namespace) - ) - - self._validate_plugins() - self._warn_about_skipped_plugins() - - if self.known_args_namespace.confcutdir is None: - if self.inipath is not None: - confcutdir = str(self.inipath.parent) - else: - confcutdir = str(self.rootpath) - self.known_args_namespace.confcutdir = confcutdir - try: - self.hook.pytest_load_initial_conftests( - early_config=self, args=args, parser=self._parser - ) - except ConftestImportFailure as e: - if self.known_args_namespace.help or self.known_args_namespace.version: - # we don't want to prevent --help/--version to work - # so just let is pass and print a warning at the end - self.issue_config_time_warning( - PytestConfigWarning(f"could not load initial conftests: {e.path}"), - stacklevel=2, - ) - else: - raise - @hookimpl(wrapper=True) def pytest_collection(self) -> Generator[None, object, object]: - # Validate invalid ini keys after collection is done so we take in account - # options added by late-loading conftest files. + # Validate invalid configuration keys after collection is done so we + # take in account options added by late-loading conftest files. try: return (yield) finally: @@ -1417,7 +1413,8 @@ def pytest_collection(self) -> Generator[None, object, object]: def _checkversion(self) -> None: import pytest - minver = self.inicfg.get("minversion", None) + minver_ini_value = self._inicfg.get("minversion", None) + minver = minver_ini_value.value if minver_ini_value is not None else None if minver: # Imported lazily to improve start-up time. from packaging.version import Version @@ -1470,39 +1467,125 @@ def _validate_plugins(self) -> None: ) def _warn_or_fail_if_strict(self, message: str) -> None: - if self.known_args_namespace.strict_config: + strict_config = self.getini("strict_config") + if strict_config is None: + strict_config = self.getini("strict") + if strict_config: raise UsageError(message) self.issue_config_time_warning(PytestConfigWarning(message), stacklevel=3) - def _get_unknown_ini_keys(self) -> list[str]: - parser_inicfg = self._parser._inidict - return [name for name in self.inicfg if name not in parser_inicfg] + def _get_unknown_ini_keys(self) -> set[str]: + known_keys = self._parser._inidict.keys() | self._parser._ini_aliases.keys() + return self._inicfg.keys() - known_keys def parse(self, args: list[str], addopts: bool = True) -> None: # Parse given cmdline arguments into this config object. - assert ( - self.args == [] - ), "can only parse cmdline args at most once per Config object" + assert self.args == [], ( + "can only parse cmdline args at most once per Config object" + ) + self.hook.pytest_addhooks.call_historic( kwargs=dict(pluginmanager=self.pluginmanager) ) - self._preparse(args, addopts=addopts) - self._parser.after_preparse = True # type: ignore - try: - args = self._parser.parse_setoption( - args, self.option, namespace=self.option + + if addopts: + env_addopts = os.environ.get("PYTEST_ADDOPTS", "") + if len(env_addopts): + args[:] = ( + self._validate_args(shlex.split(env_addopts), "via PYTEST_ADDOPTS") + + args + ) + + ns = self._parser.parse_known_args(args, namespace=copy.copy(self.option)) + rootpath, inipath, inicfg, ignored_config_files = determine_setup( + inifile=ns.inifilename, + override_ini=ns.override_ini, + args=ns.file_or_dir, + rootdir_cmd_arg=ns.rootdir or None, + invocation_dir=self.invocation_params.dir, + ) + self._rootpath = rootpath + self._inipath = inipath + self._ignored_config_files = ignored_config_files + self._inicfg = inicfg + self._parser.extra_info["rootdir"] = str(self.rootpath) + self._parser.extra_info["inifile"] = str(self.inipath) + + self._parser.addini("addopts", "Extra command line options", "args") + self._parser.addini("minversion", "Minimally required pytest version") + self._parser.addini( + "pythonpath", type="paths", help="Add paths to sys.path", default=[] + ) + self._parser.addini( + "required_plugins", + "Plugins that must be present for pytest to run", + type="args", + default=[], + ) + + if addopts: + args[:] = ( + self._validate_args(self.getini("addopts"), "via addopts config") + args ) - self.args, self.args_source = self._decide_args( - args=args, - pyargs=self.known_args_namespace.pyargs, - testpaths=self.getini("testpaths"), - invocation_dir=self.invocation_params.dir, - rootpath=self.rootpath, - warn=True, + + self.known_args_namespace = self._parser.parse_known_args( + args, namespace=copy.copy(self.option) + ) + self._checkversion() + self._consider_importhook() + self._configure_python_path() + self.pluginmanager.consider_preparse(args, exclude_only=False) + if ( + not os.environ.get("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + and not self.known_args_namespace.disable_plugin_autoload + ): + # Autoloading from distribution package entry point has + # not been disabled. + self.pluginmanager.load_setuptools_entrypoints("pytest11") + # Otherwise only plugins explicitly specified in PYTEST_PLUGINS + # are going to be loaded. + self.pluginmanager.consider_env() + + self._parser.parse_known_args(args, namespace=self.known_args_namespace) + + self._validate_plugins() + self._warn_about_skipped_plugins() + + if self.known_args_namespace.confcutdir is None: + if self.inipath is not None: + confcutdir = str(self.inipath.parent) + else: + confcutdir = str(self.rootpath) + self.known_args_namespace.confcutdir = confcutdir + try: + self.hook.pytest_load_initial_conftests( + early_config=self, args=args, parser=self._parser ) + except ConftestImportFailure as e: + if self.known_args_namespace.help or self.known_args_namespace.version: + # we don't want to prevent --help/--version to work + # so just let it pass and print a warning at the end + self.issue_config_time_warning( + PytestConfigWarning(f"could not load initial conftests: {e.path}"), + stacklevel=2, + ) + else: + raise + + try: + self._parser.parse(args, namespace=self.option) except PrintHelp: - pass + return + + self.args, self.args_source = self._decide_args( + args=getattr(self.option, FILE_OR_DIR), + pyargs=self.option.pyargs, + testpaths=self.getini("testpaths"), + invocation_dir=self.invocation_params.dir, + rootpath=self.rootpath, + warn=True, + ) def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None: """Issue and handle a warning during the "configure" stage. @@ -1540,19 +1623,19 @@ def issue_config_time_warning(self, warning: Warning, stacklevel: int) -> None: ) def addinivalue_line(self, name: str, line: str) -> None: - """Add a line to an ini-file option. The option must have been + """Add a line to a configuration option. The option must have been declared but might not yet be set in which case the line becomes the first line in its value.""" x = self.getini(name) assert isinstance(x, list) x.append(line) # modifies the cached list inline - def getini(self, name: str): - """Return configuration value from an :ref:`ini file `. + def getini(self, name: str) -> Any: + """Return configuration value the an :ref:`configuration file `. - If a configuration value is not defined in an - :ref:`ini file `, then the ``default`` value provided while - registering the configuration through + If a configuration value is not defined in a + :ref:`configuration file `, then the ``default`` value + provided while registering the configuration through :func:`parser.addini ` will be returned. Please note that you can even provide ``None`` as a valid default value. @@ -1565,6 +1648,8 @@ def getini(self, name: str): ``paths``, ``pathlist``, ``args`` and ``linelist`` : empty list ``[]`` ``bool`` : ``False`` ``string`` : empty string ``""`` + ``int`` : ``0`` + ``float`` : ``0.0`` If neither the ``default`` nor the ``type`` parameter is passed while registering the configuration through @@ -1575,46 +1660,86 @@ def getini(self, name: str): :func:`parser.addini ` call (usually from a plugin), a ValueError is raised. """ + canonical_name = self._parser._ini_aliases.get(name, name) try: - return self._inicache[name] + return self._inicache[canonical_name] except KeyError: - self._inicache[name] = val = self._getini(name) - return val + pass + self._inicache[canonical_name] = val = self._getini(canonical_name) + return val # Meant for easy monkeypatching by legacypath plugin. # Can be inlined back (with no cover removed) once legacypath is gone. - def _getini_unknown_type(self, name: str, type: str, value: str | list[str]): - msg = f"unknown configuration type: {type}" - raise ValueError(msg, value) # pragma: no cover + def _getini_unknown_type(self, name: str, type: str, value: object): + msg = ( + f"Option {name} has unknown configuration type {type} with value {value!r}" + ) + raise ValueError(msg) # pragma: no cover def _getini(self, name: str): + # If this is an alias, resolve to canonical name. + canonical_name = self._parser._ini_aliases.get(name, name) + try: - description, type, default = self._parser._inidict[name] + _description, type, default = self._parser._inidict[canonical_name] except KeyError as e: raise ValueError(f"unknown configuration value: {name!r}") from e - override_value = self._get_override_ini_value(name) - if override_value is None: - try: - value = self.inicfg[name] - except KeyError: - return default + + # Collect all possible values (canonical name + aliases) from _inicfg. + # Each candidate is (ConfigValue, is_canonical). + candidates = [] + if canonical_name in self._inicfg: + candidates.append((self._inicfg[canonical_name], True)) + for alias, target in self._parser._ini_aliases.items(): + if target == canonical_name and alias in self._inicfg: + candidates.append((self._inicfg[alias], False)) + + if not candidates: + return default + + # Pick the best candidate based on precedence: + # 1. CLI override takes precedence over file, then + # 2. Canonical name takes precedence over alias. + selected = max(candidates, key=lambda x: (x[0].origin == "override", x[1]))[0] + value = selected.value + mode = selected.mode + + if mode == "ini": + # In ini mode, values are always str | list[str]. + assert isinstance(value, (str, list)) + return self._getini_ini(name, canonical_name, type, value, default) + elif mode == "toml": + return self._getini_toml(name, canonical_name, type, value, default) else: - value = override_value - # Coerce the values based on types. - # - # Note: some coercions are only required if we are reading from .ini files, because - # the file format doesn't contain type information, but when reading from toml we will - # get either str or list of str values (see _parse_ini_config_from_pyproject_toml). - # For example: + assert_never(mode) + + def _getini_ini( + self, + name: str, + canonical_name: str, + type: str, + value: str | list[str], + default: Any, + ): + """Handle config values read in INI mode. + + In INI mode, values are stored as str or list[str] only, and coerced + from string based on the registered type. + """ + # Note: some coercions are only required if we are reading from .ini + # files, because the file format doesn't contain type information, but + # when reading from toml (in ini mode) we will get either str or list of + # str values (see load_config_dict_from_file). For example: # # ini: # a_line_list = "tests acceptance" - # in this case, we need to split the string to obtain a list of strings. # - # toml: + # in this case, we need to split the string to obtain a list of strings. + # + # toml (ini mode): # a_line_list = ["tests", "acceptance"] - # in this case, we already have a list ready to use. # + # in this case, we already have a list ready to use. if type == "paths": dp = ( self.inipath.parent @@ -1634,7 +1759,101 @@ def _getini(self, name: str): return _strtobool(str(value).strip()) elif type == "string": return value - elif type is None: + elif type == "int": + if not isinstance(value, str): + raise TypeError( + f"Expected an int string for option {name} of type integer, but got: {value!r}" + ) from None + return int(value) + elif type == "float": + if not isinstance(value, str): + raise TypeError( + f"Expected a float string for option {name} of type float, but got: {value!r}" + ) from None + return float(value) + else: + return self._getini_unknown_type(name, type, value) + + def _getini_toml( + self, + name: str, + canonical_name: str, + type: str, + value: object, + default: Any, + ): + """Handle TOML config values with strict type validation and no coercion. + + In TOML mode, values already have native types from TOML parsing. + We validate types match expectations exactly, including list items. + """ + value_type = builtins.type(value).__name__ + if type == "paths": + # Expect a list of strings. + if not isinstance(value, list): + raise TypeError( + f"{self.inipath}: config option '{name}' expects a list for type 'paths', " + f"got {value_type}: {value!r}" + ) + for i, item in enumerate(value): + if not isinstance(item, str): + item_type = builtins.type(item).__name__ + raise TypeError( + f"{self.inipath}: config option '{name}' expects a list of strings, " + f"but item at index {i} is {item_type}: {item!r}" + ) + dp = ( + self.inipath.parent + if self.inipath is not None + else self.invocation_params.dir + ) + return [dp / x for x in value] + elif type in {"args", "linelist"}: + # Expect a list of strings. + if not isinstance(value, list): + raise TypeError( + f"{self.inipath}: config option '{name}' expects a list for type '{type}', " + f"got {value_type}: {value!r}" + ) + for i, item in enumerate(value): + if not isinstance(item, str): + item_type = builtins.type(item).__name__ + raise TypeError( + f"{self.inipath}: config option '{name}' expects a list of strings, " + f"but item at index {i} is {item_type}: {item!r}" + ) + return list(value) + elif type == "bool": + # Expect a boolean. + if not isinstance(value, bool): + raise TypeError( + f"{self.inipath}: config option '{name}' expects a bool, " + f"got {value_type}: {value!r}" + ) + return value + elif type == "int": + # Expect an integer (but not bool, which is a subclass of int). + if not isinstance(value, int) or isinstance(value, bool): + raise TypeError( + f"{self.inipath}: config option '{name}' expects an int, " + f"got {value_type}: {value!r}" + ) + return value + elif type == "float": + # Expect a float or integer only. + if not isinstance(value, (float, int)) or isinstance(value, bool): + raise TypeError( + f"{self.inipath}: config option '{name}' expects a float, " + f"got {value_type}: {value!r}" + ) + return value + elif type == "string": + # Expect a string. + if not isinstance(value, str): + raise TypeError( + f"{self.inipath}: config option '{name}' expects a string, " + f"got {value_type}: {value!r}" + ) return value else: return self._getini_unknown_type(name, type, value) @@ -1658,31 +1877,15 @@ def _getconftest_pathlist( values.append(relroot) return values - def _get_override_ini_value(self, name: str) -> str | None: - value = None - # override_ini is a list of "ini=value" options. - # Always use the last item if multiple values are set for same ini-name, - # e.g. -o foo=bar1 -o foo=bar2 will set foo to bar2. - for ini_config in self._override_ini: - try: - key, user_ini_value = ini_config.split("=", 1) - except ValueError as e: - raise UsageError( - f"-o/--override-ini expects option=value style (got: {ini_config!r})." - ) from e - else: - if key == name: - value = user_ini_value - return value - - def getoption(self, name: str, default=notset, skip: bool = False): + def getoption(self, name: str, default: Any = notset, skip: bool = False): """Return command line option value. - :param name: Name of the option. You may also specify + :param name: Name of the option. You may also specify the literal ``--OPT`` option instead of the "dest" option name. - :param default: Default value if no option of that name exists. - :param skip: If True, raise pytest.skip if option does not exists - or has a None value. + :param default: Fallback value if no option of that name is **declared** via :hook:`pytest_addoption`. + Note this parameter will be ignored when the option is **declared** even if the option's value is ``None``. + :param skip: If ``True``, raise :func:`pytest.skip` if option is undeclared or has a ``None`` value. + Note that even if ``True``, if a default was specified it will be returned instead of a skip. """ name = self._opt2dest.get(name, name) try: @@ -1711,6 +1914,9 @@ def getvalueorskip(self, name: str, path=None): VERBOSITY_ASSERTIONS: Final = "assertions" #: Verbosity type for test case execution (see :confval:`verbosity_test_cases`). VERBOSITY_TEST_CASES: Final = "test_cases" + #: Verbosity type for failed subtests (see :confval:`verbosity_subtests`). + VERBOSITY_SUBTESTS: Final = "subtests" + _VERBOSITY_INI_DEFAULT: Final = "auto" def get_verbosity(self, verbosity_type: str | None = None) -> int: @@ -1729,11 +1935,19 @@ def get_verbosity(self, verbosity_type: str | None = None) -> int: Example: - .. code-block:: ini + .. tab:: toml + + .. code-block:: toml + + [tool.pytest] + verbosity_assertions = 2 + + .. tab:: ini - # content of pytest.ini - [pytest] - verbosity_assertions = 2 + .. code-block:: ini + + [pytest] + verbosity_assertions = 2 .. code-block:: console @@ -1744,7 +1958,7 @@ def get_verbosity(self, verbosity_type: str | None = None) -> int: print(config.get_verbosity()) # 1 print(config.get_verbosity(Config.VERBOSITY_ASSERTIONS)) # 2 """ - global_level = self.option.verbose + global_level = self.getoption("verbose", default=0) assert isinstance(global_level, int) if verbosity_type is None: return global_level @@ -1767,7 +1981,7 @@ def _verbosity_ini_name(verbosity_type: str) -> str: def _add_verbosity_ini(parser: Parser, verbosity_type: str, help: str) -> None: """Add a output verbosity configuration option for the given output type. - :param parser: Parser for command line arguments and ini-file values. + :param parser: Parser for command line arguments and config-file values. :param verbosity_type: Fine-grained verbosity category. :param help: Description of the output this type controls. @@ -1911,6 +2125,8 @@ def parse_warning_filter( raise UsageError(error_template.format(error=str(e))) from None try: category: type[Warning] = _resolve_warning_category(category_) + except ImportError: + raise except Exception: exc_info = ExceptionInfo.from_current() exception_text = exc_info.getrepr(style="native") @@ -1930,6 +2146,13 @@ def parse_warning_filter( ) from None else: lineno = 0 + try: + re.compile(message) + re.compile(module) + except re.error as e: + raise UsageError( + error_template.format(error=f"Invalid regex {e.pattern!r}: {e}") + ) from None return action, message, category, module, lineno @@ -1952,7 +2175,7 @@ def _resolve_warning_category(category: str) -> type[Warning]: cat = getattr(m, klass) if not issubclass(cat, Warning): raise UsageError(f"{cat} is not a Warning subclass") - return cast(Type[Warning], cat) + return cast(type[Warning], cat) def apply_warning_filters( @@ -1962,7 +2185,19 @@ def apply_warning_filters( # Filters should have this precedence: cmdline options, config. # Filters should be applied in the inverse order of precedence. for arg in config_filters: - warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) + try: + warnings.filterwarnings(*parse_warning_filter(arg, escape=False)) + except ImportError as e: + warnings.warn( + f"Failed to import filter module '{e.name}': {arg}", PytestConfigWarning + ) + continue for arg in cmdline_filters: - warnings.filterwarnings(*parse_warning_filter(arg, escape=True)) + try: + warnings.filterwarnings(*parse_warning_filter(arg, escape=True)) + except ImportError as e: + warnings.warn( + f"Failed to import filter module '{e.name}': {arg}", PytestConfigWarning + ) + continue diff --git a/src/_pytest/config/argparsing.py b/src/_pytest/config/argparsing.py index 85aa4632702..8216ad8b226 100644 --- a/src/_pytest/config/argparsing.py +++ b/src/_pytest/config/argparsing.py @@ -2,21 +2,18 @@ from __future__ import annotations import argparse -from gettext import gettext +from collections.abc import Callable +from collections.abc import Mapping +from collections.abc import Sequence import os import sys from typing import Any -from typing import Callable -from typing import cast from typing import final -from typing import List from typing import Literal -from typing import Mapping from typing import NoReturn -from typing import Sequence +from .exceptions import UsageError import _pytest._io -from _pytest.config.exceptions import UsageError from _pytest.deprecated import check_ispytest @@ -33,14 +30,12 @@ def __repr__(self) -> str: @final class Parser: - """Parser for command line arguments and ini-file values. + """Parser for command line arguments and config-file values. :ivar extra_info: Dict of generic param -> value to display in case there's an error processing the command line arguments. """ - prog: str | None = None - def __init__( self, usage: str | None = None, @@ -49,13 +44,31 @@ def __init__( _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) - self._anonymous = OptionGroup("Custom options", parser=self, _ispytest=True) - self._groups: list[OptionGroup] = [] + + from _pytest._argcomplete import filescompleter + self._processopt = processopt - self._usage = usage - self._inidict: dict[str, tuple[str, str | None, Any]] = {} - self._ininames: list[str] = [] self.extra_info: dict[str, Any] = {} + self.optparser = PytestArgumentParser(self, usage, self.extra_info) + anonymous_arggroup = self.optparser.add_argument_group("Custom options") + self._anonymous = OptionGroup( + anonymous_arggroup, "_anonymous", self, _ispytest=True + ) + self._groups = [self._anonymous] + file_or_dir_arg = self.optparser.add_argument(FILE_OR_DIR, nargs="*") + file_or_dir_arg.completer = filescompleter # type: ignore + + self._inidict: dict[str, tuple[str, str, Any]] = {} + # Maps alias -> canonical name. + self._ini_aliases: dict[str, str] = {} + + @property + def prog(self) -> str: + return self.optparser.prog + + @prog.setter + def prog(self, value: str) -> None: + self.optparser.prog = value def processoption(self, option: Argument) -> None: if self._processopt: @@ -80,12 +93,17 @@ def getgroup( for group in self._groups: if group.name == name: return group - group = OptionGroup(name, description, parser=self, _ispytest=True) + + arggroup = self.optparser.add_argument_group(description or name) + group = OptionGroup(arggroup, name, self, _ispytest=True) i = 0 for i, grp in enumerate(self._groups): if grp.name == after: break self._groups.insert(i + 1, group) + # argparse doesn't provide a way to control `--help` order, so must + # access its internals ☹. + self.optparser._action_groups.insert(i + 1, self.optparser._action_groups.pop()) return group def addoption(self, *opts: str, **attrs: Any) -> None: @@ -109,42 +127,24 @@ def parse( args: Sequence[str | os.PathLike[str]], namespace: argparse.Namespace | None = None, ) -> argparse.Namespace: + """Parse the arguments. + + Unlike ``parse_known_args`` and ``parse_known_and_unknown_args``, + raises PrintHelp on `--help` and UsageError on unknown flags + + :meta private: + """ from _pytest._argcomplete import try_argcomplete - self.optparser = self._getparser() try_argcomplete(self.optparser) strargs = [os.fspath(x) for x in args] - return self.optparser.parse_args(strargs, namespace=namespace) - - def _getparser(self) -> MyOptionParser: - from _pytest._argcomplete import filescompleter - - optparser = MyOptionParser(self, self.extra_info, prog=self.prog) - groups = [*self._groups, self._anonymous] - for group in groups: - if group.options: - desc = group.description or group.name - arggroup = optparser.add_argument_group(desc) - for option in group.options: - n = option.names() - a = option.attrs() - arggroup.add_argument(*n, **a) - file_or_dir_arg = optparser.add_argument(FILE_OR_DIR, nargs="*") - # bash like autocompletion for dirs (appending '/') - # Type ignored because typeshed doesn't know about argcomplete. - file_or_dir_arg.completer = filescompleter # type: ignore - return optparser - - def parse_setoption( - self, - args: Sequence[str | os.PathLike[str]], - option: argparse.Namespace, - namespace: argparse.Namespace | None = None, - ) -> list[str]: - parsedoption = self.parse(args, namespace=namespace) - for name, value in parsedoption.__dict__.items(): - setattr(option, name, value) - return cast(List[str], getattr(parsedoption, FILE_OR_DIR)) + if namespace is None: + namespace = argparse.Namespace() + try: + namespace._raise_print_help = True + return self.optparser.parse_intermixed_args(strargs, namespace=namespace) + finally: + del namespace._raise_print_help def parse_known_args( self, @@ -163,30 +163,43 @@ def parse_known_and_unknown_args( namespace: argparse.Namespace | None = None, ) -> tuple[argparse.Namespace, list[str]]: """Parse the known arguments at this point, and also return the - remaining unknown arguments. + remaining unknown flag arguments. :returns: A tuple containing an argparse namespace object for the known - arguments, and a list of the unknown arguments. + arguments, and a list of unknown flag arguments. """ - optparser = self._getparser() strargs = [os.fspath(x) for x in args] - return optparser.parse_known_args(strargs, namespace=namespace) + if sys.version_info < (3, 12, 8) or (3, 13) <= sys.version_info < (3, 13, 1): + # Older argparse have a bugged parse_known_intermixed_args. + namespace, unknown = self.optparser.parse_known_args(strargs, namespace) + assert namespace is not None + file_or_dir = getattr(namespace, FILE_OR_DIR) + unknown_flags: list[str] = [] + for arg in unknown: + (unknown_flags if arg.startswith("-") else file_or_dir).append(arg) + return namespace, unknown_flags + else: + return self.optparser.parse_known_intermixed_args(strargs, namespace) def addini( self, name: str, help: str, - type: Literal["string", "paths", "pathlist", "args", "linelist", "bool"] + type: Literal[ + "string", "paths", "pathlist", "args", "linelist", "bool", "int", "float" + ] | None = None, default: Any = NOT_SET, + *, + aliases: Sequence[str] = (), ) -> None: - """Register an ini-file option. + """Register a configuration file option. :param name: - Name of the ini-variable. + Name of the configuration. :param type: - Type of the variable. Can be: + Type of the configuration. Can be: * ``string``: a string * ``bool``: a boolean @@ -194,45 +207,81 @@ def addini( * ``linelist``: a list of strings, separated by line breaks * ``paths``: a list of :class:`pathlib.Path`, separated as in a shell * ``pathlist``: a list of ``py.path``, separated as in a shell + * ``int``: an integer + * ``float``: a floating-point number + + .. versionadded:: 8.4 + + The ``float`` and ``int`` types. - For ``paths`` and ``pathlist`` types, they are considered relative to the ini-file. - In case the execution is happening without an ini-file defined, + For ``paths`` and ``pathlist`` types, they are considered relative to the config-file. + In case the execution is happening without a config-file defined, they will be considered relative to the current working directory (for example with ``--override-ini``). .. versionadded:: 7.0 The ``paths`` variable type. .. versionadded:: 8.1 - Use the current working directory to resolve ``paths`` and ``pathlist`` in the absence of an ini-file. + Use the current working directory to resolve ``paths`` and ``pathlist`` in the absence of a config-file. Defaults to ``string`` if ``None`` or not passed. :param default: - Default value if no ini-file option exists but is queried. + Default value if no config-file option exists but is queried. + :param aliases: + Additional names by which this option can be referenced. + Aliases resolve to the canonical name. - The value of ini-variables can be retrieved via a call to + .. versionadded:: 9.0 + The ``aliases`` parameter. + + The value of configuration keys can be retrieved via a call to :py:func:`config.getini(name) `. """ - assert type in (None, "string", "paths", "pathlist", "args", "linelist", "bool") + assert type in ( + None, + "string", + "paths", + "pathlist", + "args", + "linelist", + "bool", + "int", + "float", + ) + if type is None: + type = "string" if default is NOT_SET: default = get_ini_default_for_type(type) self._inidict[name] = (help, type, default) - self._ininames.append(name) + + for alias in aliases: + if alias in self._inidict: + raise ValueError( + f"alias {alias!r} conflicts with existing configuration option" + ) + if (already := self._ini_aliases.get(alias)) is not None: + raise ValueError(f"{alias!r} is already an alias of {already!r}") + self._ini_aliases[alias] = name def get_ini_default_for_type( - type: Literal["string", "paths", "pathlist", "args", "linelist", "bool"] | None, + type: Literal[ + "string", "paths", "pathlist", "args", "linelist", "bool", "int", "float" + ], ) -> Any: """ - Used by addini to get the default value for a given ini-option type, when + Used by addini to get the default value for a given config option type, when default is not supplied. """ - if type is None: - return "" - elif type in ("paths", "pathlist", "args", "linelist"): + if type in ("paths", "pathlist", "args", "linelist"): return [] elif type == "bool": return False + elif type == "int": + return 0 + elif type == "float": + return 0.0 else: return "" @@ -293,9 +342,7 @@ def names(self) -> list[str]: def attrs(self) -> Mapping[str, Any]: # Update any attributes set by processopt. - attrs = "default dest help".split() - attrs.append(self.dest) - for attr in attrs: + for attr in ("default", "dest", "help", self.dest): try: self._attrs[attr] = getattr(self, attr) except AttributeError: @@ -350,15 +397,14 @@ class OptionGroup: def __init__( self, + arggroup: argparse._ArgumentGroup, name: str, - description: str = "", - parser: Parser | None = None, - *, + parser: Parser | None, _ispytest: bool = False, ) -> None: check_ispytest(_ispytest) + self._arggroup = arggroup self.name = name - self.description = description self.options: list[Argument] = [] self.parser = parser @@ -393,22 +439,24 @@ def _addoption_instance(self, option: Argument, shortupper: bool = False) -> Non for opt in option._short_opts: if opt[0] == "-" and opt[1].islower(): raise ValueError("lowercase shortoptions reserved") + if self.parser: self.parser.processoption(option) + + self._arggroup.add_argument(*option.names(), **option.attrs()) self.options.append(option) -class MyOptionParser(argparse.ArgumentParser): +class PytestArgumentParser(argparse.ArgumentParser): def __init__( self, parser: Parser, - extra_info: dict[str, Any] | None = None, - prog: str | None = None, + usage: str | None, + extra_info: dict[str, str], ) -> None: self._parser = parser super().__init__( - prog=prog, - usage=parser._usage, + usage=usage, add_help=False, formatter_class=DropShorterLongHelpFormatter, allow_abbrev=False, @@ -416,75 +464,17 @@ def __init__( ) # extra_info is a dict of (param -> value) to display if there's # an usage error to provide more contextual information to the user. - self.extra_info = extra_info if extra_info else {} + self.extra_info = extra_info def error(self, message: str) -> NoReturn: """Transform argparse error message into UsageError.""" msg = f"{self.prog}: error: {message}" - - if hasattr(self._parser, "_config_source_hint"): - msg = f"{msg} ({self._parser._config_source_hint})" - + if self.extra_info: + msg += "\n" + "\n".join( + f" {k}: {v}" for k, v in sorted(self.extra_info.items()) + ) raise UsageError(self.format_usage() + msg) - # Type ignored because typeshed has a very complex type in the superclass. - def parse_args( # type: ignore - self, - args: Sequence[str] | None = None, - namespace: argparse.Namespace | None = None, - ) -> argparse.Namespace: - """Allow splitting of positional arguments.""" - parsed, unrecognized = self.parse_known_args(args, namespace) - if unrecognized: - for arg in unrecognized: - if arg and arg[0] == "-": - lines = [ - "unrecognized arguments: {}".format(" ".join(unrecognized)) - ] - for k, v in sorted(self.extra_info.items()): - lines.append(f" {k}: {v}") - self.error("\n".join(lines)) - getattr(parsed, FILE_OR_DIR).extend(unrecognized) - return parsed - - if sys.version_info < (3, 9): # pragma: no cover - # Backport of https://github.com/python/cpython/pull/14316 so we can - # disable long --argument abbreviations without breaking short flags. - def _parse_optional( - self, arg_string: str - ) -> tuple[argparse.Action | None, str, str | None] | None: - if not arg_string: - return None - if arg_string[0] not in self.prefix_chars: - return None - if arg_string in self._option_string_actions: - action = self._option_string_actions[arg_string] - return action, arg_string, None - if len(arg_string) == 1: - return None - if "=" in arg_string: - option_string, explicit_arg = arg_string.split("=", 1) - if option_string in self._option_string_actions: - action = self._option_string_actions[option_string] - return action, option_string, explicit_arg - if self.allow_abbrev or not arg_string.startswith("--"): - option_tuples = self._get_option_tuples(arg_string) - if len(option_tuples) > 1: - msg = gettext( - "ambiguous option: %(option)s could match %(matches)s" - ) - options = ", ".join(option for _, option, _ in option_tuples) - self.error(msg % {"option": arg_string, "matches": options}) - elif len(option_tuples) == 1: - (option_tuple,) = option_tuples - return option_tuple - if self._negative_number_matcher.match(arg_string): - if not self._has_negative_number_optionals: - return None - if " " in arg_string: - return None - return None, arg_string, None - class DropShorterLongHelpFormatter(argparse.HelpFormatter): """Shorten help for long options that differ only in extra hyphens. @@ -549,3 +539,40 @@ def _split_lines(self, text, width): for line in text.splitlines(): lines.extend(textwrap.wrap(line.strip(), width)) return lines + + +class OverrideIniAction(argparse.Action): + """Custom argparse action that makes a CLI flag equivalent to overriding an + option, in addition to behaving like `store_true`. + + This can simplify things since code only needs to inspect the config option + and not consider the CLI flag. + """ + + def __init__( + self, + option_strings: Sequence[str], + dest: str, + nargs: int | str | None = None, + *args, + ini_option: str, + ini_value: str, + **kwargs, + ) -> None: + super().__init__(option_strings, dest, 0, *args, **kwargs) + self.ini_option = ini_option + self.ini_value = ini_value + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + *args, + **kwargs, + ) -> None: + setattr(namespace, self.dest, True) + current_overrides = getattr(namespace, "override_ini", None) + if current_overrides is None: + current_overrides = [] + current_overrides.append(f"{self.ini_option}={self.ini_value}") + setattr(namespace, "override_ini", current_overrides) diff --git a/src/_pytest/config/compat.py b/src/_pytest/config/compat.py index 2856d85d195..21eab4c7e47 100644 --- a/src/_pytest/config/compat.py +++ b/src/_pytest/config/compat.py @@ -1,9 +1,9 @@ from __future__ import annotations +from collections.abc import Mapping import functools from pathlib import Path from typing import Any -from typing import Mapping import warnings import pluggy diff --git a/src/_pytest/config/exceptions.py b/src/_pytest/config/exceptions.py index 90108eca904..d84a9ea67e0 100644 --- a/src/_pytest/config/exceptions.py +++ b/src/_pytest/config/exceptions.py @@ -7,6 +7,8 @@ class UsageError(Exception): """Error in pytest usage or invocation.""" + __module__ = "pytest" + class PrintHelp(Exception): """Raised when pytest should print its help to skip the rest of the diff --git a/src/_pytest/config/findpaths.py b/src/_pytest/config/findpaths.py index ce4c990b810..3c628a09c2d 100644 --- a/src/_pytest/config/findpaths.py +++ b/src/_pytest/config/findpaths.py @@ -1,10 +1,14 @@ from __future__ import annotations +from collections.abc import Iterable +from collections.abc import Sequence +from dataclasses import dataclass +from dataclasses import KW_ONLY import os from pathlib import Path import sys -from typing import Iterable -from typing import Sequence +from typing import Literal +from typing import TypeAlias import iniconfig @@ -15,6 +19,30 @@ from _pytest.pathlib import safe_exists +@dataclass(frozen=True) +class ConfigValue: + """Represents a configuration value with its origin and parsing mode. + + This allows tracking whether a value came from a configuration file + or from a CLI override (--override-ini), which is important for + determining precedence when dealing with ini option aliases. + + The mode tracks the parsing mode/data model used for the value: + - "ini": from INI files or [tool.pytest.ini_options], where the only + supported value types are `str` or `list[str]`. + - "toml": from TOML files (not in INI mode), where native TOML types + are preserved. + """ + + value: object + _: KW_ONLY + origin: Literal["file", "override"] + mode: Literal["ini", "toml"] + + +ConfigDict: TypeAlias = dict[str, ConfigValue] + + def _parse_ini_config(path: Path) -> iniconfig.IniConfig: """Parse the given generic '.ini' file using legacy IniConfig parser, returning the parsed object. @@ -29,7 +57,7 @@ def _parse_ini_config(path: Path) -> iniconfig.IniConfig: def load_config_dict_from_file( filepath: Path, -) -> dict[str, str | list[str]] | None: +) -> ConfigDict | None: """Load pytest configuration from the given file path, if supported. Return None if the file does not contain valid pytest configuration. @@ -39,10 +67,13 @@ def load_config_dict_from_file( iniconfig = _parse_ini_config(filepath) if "pytest" in iniconfig: - return dict(iniconfig["pytest"].items()) + return { + k: ConfigValue(v, origin="file", mode="ini") + for k, v in iniconfig["pytest"].items() + } else: # "pytest.ini" files are always the source of configuration, even if empty. - if filepath.name == "pytest.ini": + if filepath.name in {"pytest.ini", ".pytest.ini"}: return {} # '.cfg' files are considered if they contain a "[tool:pytest]" section. @@ -50,13 +81,18 @@ def load_config_dict_from_file( iniconfig = _parse_ini_config(filepath) if "tool:pytest" in iniconfig.sections: - return dict(iniconfig["tool:pytest"].items()) + return { + k: ConfigValue(v, origin="file", mode="ini") + for k, v in iniconfig["tool:pytest"].items() + } elif "pytest" in iniconfig.sections: # If a setup.cfg contains a "[pytest]" section, we raise a failure to indicate users that # plain "[pytest]" sections in setup.cfg files is no longer supported (#3086). fail(CFG_PYTEST_SECTION.format(filename="setup.cfg"), pytrace=False) - # '.toml' files are considered if they contain a [tool.pytest.ini_options] table. + # '.toml' files are considered if they contain a [tool.pytest] table (toml mode) + # or [tool.pytest.ini_options] table (ini mode) for pyproject.toml, + # or [pytest] table (toml mode) for pytest.toml/.pytest.toml. elif filepath.suffix == ".toml": if sys.version_info >= (3, 11): import tomllib @@ -69,15 +105,52 @@ def load_config_dict_from_file( except tomllib.TOMLDecodeError as exc: raise UsageError(f"{filepath}: {exc}") from exc - result = config.get("tool", {}).get("pytest", {}).get("ini_options", None) - if result is not None: - # TOML supports richer data types than ini files (strings, arrays, floats, ints, etc), - # however we need to convert all scalar values to str for compatibility with the rest - # of the configuration system, which expects strings only. - def make_scalar(v: object) -> str | list[str]: - return v if isinstance(v, list) else str(v) - - return {k: make_scalar(v) for k, v in result.items()} + # pytest.toml and .pytest.toml use [pytest] table directly. + if filepath.name in ("pytest.toml", ".pytest.toml"): + pytest_config = config.get("pytest", {}) + if pytest_config: + # TOML mode - preserve native TOML types. + return { + k: ConfigValue(v, origin="file", mode="toml") + for k, v in pytest_config.items() + } + # "pytest.toml" files are always the source of configuration, even if empty. + return {} + + # pyproject.toml uses [tool.pytest] or [tool.pytest.ini_options]. + else: + tool_pytest = config.get("tool", {}).get("pytest", {}) + + # Check for toml mode config: [tool.pytest] with content outside of ini_options. + toml_config = {k: v for k, v in tool_pytest.items() if k != "ini_options"} + # Check for ini mode config: [tool.pytest.ini_options]. + ini_config = tool_pytest.get("ini_options", None) + + if toml_config and ini_config: + raise UsageError( + f"{filepath}: Cannot use both [tool.pytest] (native TOML types) and " + "[tool.pytest.ini_options] (string-based INI format) simultaneously. " + "Please use [tool.pytest] with native TOML types (recommended) " + "or [tool.pytest.ini_options] for backwards compatibility." + ) + + if toml_config: + # TOML mode - preserve native TOML types. + return { + k: ConfigValue(v, origin="file", mode="toml") + for k, v in toml_config.items() + } + + elif ini_config is not None: + # INI mode - TOML supports richer data types than INI files, but we need to + # convert all scalar values to str for compatibility with the INI system. + def make_scalar(v: object) -> str | list[str]: + return v if isinstance(v, list) else str(v) + + return { + k: ConfigValue(make_scalar(v), origin="file", mode="ini") + for k, v in ini_config.items() + } return None @@ -85,10 +158,14 @@ def make_scalar(v: object) -> str | list[str]: def locate_config( invocation_dir: Path, args: Iterable[Path], -) -> tuple[Path | None, Path | None, dict[str, str | list[str]]]: +) -> tuple[Path | None, Path | None, ConfigDict, Sequence[str]]: """Search in the list of arguments for a valid ini-file for pytest, - and return a tuple of (rootdir, inifile, cfg-dict).""" + and return a tuple of (rootdir, inifile, cfg-dict, ignored-config-files), where + ignored-config-files is a list of config basenames found that contain + pytest configuration but were ignored.""" config_names = [ + "pytest.toml", + ".pytest.toml", "pytest.ini", ".pytest.ini", "pyproject.toml", @@ -99,6 +176,8 @@ def locate_config( if not args: args = [invocation_dir] found_pyproject_toml: Path | None = None + ignored_config_files: list[str] = [] + for arg in args: argpath = absolutepath(arg) for base in (argpath, *argpath.parents): @@ -109,10 +188,18 @@ def locate_config( found_pyproject_toml = p ini_config = load_config_dict_from_file(p) if ini_config is not None: - return base, p, ini_config + index = config_names.index(config_name) + for remainder in config_names[index + 1 :]: + p2 = base / remainder + if ( + p2.is_file() + and load_config_dict_from_file(p2) is not None + ): + ignored_config_files.append(remainder) + return base, p, ini_config, ignored_config_files if found_pyproject_toml is not None: - return found_pyproject_toml.parent, found_pyproject_toml, {} - return None, None, {} + return found_pyproject_toml.parent, found_pyproject_toml, {}, [] + return None, None, {}, [] def get_common_ancestor( @@ -163,30 +250,59 @@ def get_dir_from_path(path: Path) -> Path: return [get_dir_from_path(path) for path in possible_paths if safe_exists(path)] +def parse_override_ini(override_ini: Sequence[str] | None) -> ConfigDict: + """Parse the -o/--override-ini command line arguments and return the overrides. + + :raises UsageError: + If one of the values is malformed. + """ + overrides = {} + # override_ini is a list of "ini=value" options. + # Always use the last item if multiple values are set for same ini-name, + # e.g. -o foo=bar1 -o foo=bar2 will set foo to bar2. + for ini_config in override_ini or (): + try: + key, user_ini_value = ini_config.split("=", 1) + except ValueError as e: + raise UsageError( + f"-o/--override-ini expects option=value style (got: {ini_config!r})." + ) from e + else: + overrides[key] = ConfigValue(user_ini_value, origin="override", mode="ini") + return overrides + + CFG_PYTEST_SECTION = "[pytest] section in {filename} files is no longer supported, change to [tool:pytest] instead." def determine_setup( *, inifile: str | None, + override_ini: Sequence[str] | None, args: Sequence[str], rootdir_cmd_arg: str | None, invocation_dir: Path, -) -> tuple[Path, Path | None, dict[str, str | list[str]]]: +) -> tuple[Path, Path | None, ConfigDict, Sequence[str]]: """Determine the rootdir, inifile and ini configuration values from the command line arguments. :param inifile: The `--inifile` command line argument, if given. + :param override_ini: + The -o/--override-ini command line arguments, if given. :param args: The free command line arguments. :param rootdir_cmd_arg: The `--rootdir` command line argument, if given. :param invocation_dir: The working directory when pytest was invoked. + + :raises UsageError: """ rootdir = None dirs = get_dirs_from_args(args) + ignored_config_files: Sequence[str] = [] + if inifile: inipath_ = absolutepath(inifile) inipath: Path | None = inipath_ @@ -195,7 +311,9 @@ def determine_setup( rootdir = inipath_.parent else: ancestor = get_common_ancestor(invocation_dir, dirs) - rootdir, inipath, inicfg = locate_config(invocation_dir, [ancestor]) + rootdir, inipath, inicfg, ignored_config_files = locate_config( + invocation_dir, [ancestor] + ) if rootdir is None and rootdir_cmd_arg is None: for possible_rootdir in (ancestor, *ancestor.parents): if (possible_rootdir / "setup.py").is_file(): @@ -203,7 +321,7 @@ def determine_setup( break else: if dirs != [ancestor]: - rootdir, inipath, inicfg = locate_config(invocation_dir, dirs) + rootdir, inipath, inicfg, _ = locate_config(invocation_dir, dirs) if rootdir is None: rootdir = get_common_ancestor( invocation_dir, [invocation_dir, ancestor] @@ -216,8 +334,12 @@ def determine_setup( raise UsageError( f"Directory '{rootdir}' not found. Check your '--rootdir' option." ) + + ini_overrides = parse_override_ini(override_ini) + inicfg.update(ini_overrides) + assert rootdir is not None - return rootdir, inipath, inicfg or {} + return rootdir, inipath, inicfg, ignored_config_files def is_fs_root(p: Path) -> bool: diff --git a/src/_pytest/debugging.py b/src/_pytest/debugging.py index 3e1463fff26..de1b2688f76 100644 --- a/src/_pytest/debugging.py +++ b/src/_pytest/debugging.py @@ -5,12 +5,12 @@ from __future__ import annotations import argparse +from collections.abc import Callable +from collections.abc import Generator import functools import sys import types from typing import Any -from typing import Callable -from typing import Generator import unittest from _pytest import outcomes @@ -40,13 +40,13 @@ def _validate_usepdb_cls(value: str) -> tuple[str, str]: def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("general") - group._addoption( + group.addoption( "--pdb", dest="usepdb", action="store_true", help="Start the interactive Python debugger on errors or KeyboardInterrupt", ) - group._addoption( + group.addoption( "--pdbcls", dest="usepdb_cls", metavar="modulename:classname", @@ -54,7 +54,7 @@ def pytest_addoption(parser: Parser) -> None: help="Specify a custom interactive Python debugger for use with --pdb." "For example: --pdbcls=IPython.terminal.debugger:TerminalPdb", ) - group._addoption( + group.addoption( "--trace", dest="trace", action="store_true", @@ -159,6 +159,9 @@ def do_debug(self, arg): cls._recursive_debug -= 1 return ret + if hasattr(pdb_cls, "do_debug"): + do_debug.__doc__ = pdb_cls.do_debug.__doc__ + def do_continue(self, arg): ret = super().do_continue(arg) if cls._recursive_debug == 0: @@ -185,15 +188,17 @@ def do_continue(self, arg): self._continued = True return ret + if hasattr(pdb_cls, "do_continue"): + do_continue.__doc__ = pdb_cls.do_continue.__doc__ + do_c = do_cont = do_continue def do_quit(self, arg): - """Raise Exit outcome when quit command is used in pdb. - - This is a bit of a hack - it would be better if BdbQuit - could be handled, but this would require to wrap the - whole pytest run, and adjust the report etc. - """ + # Raise Exit outcome when quit command is used in pdb. + # + # This is a bit of a hack - it would be better if BdbQuit + # could be handled, but this would require to wrap the + # whole pytest run, and adjust the report etc. ret = super().do_quit(arg) if cls._recursive_debug == 0: @@ -201,6 +206,9 @@ def do_quit(self, arg): return ret + if hasattr(pdb_cls, "do_quit"): + do_quit.__doc__ = pdb_cls.do_quit.__doc__ + do_q = do_quit do_exit = do_quit @@ -292,8 +300,8 @@ def pytest_exception_interact( _enter_pdb(node, call.excinfo, report) def pytest_internalerror(self, excinfo: ExceptionInfo[BaseException]) -> None: - tb = _postmortem_traceback(excinfo) - post_mortem(tb) + exc_or_tb = _postmortem_exc_or_tb(excinfo) + post_mortem(exc_or_tb) class PdbTrace: @@ -332,7 +340,7 @@ def maybe_wrap_pytest_function_for_tracing(pyfuncitem) -> None: def _enter_pdb( node: Node, excinfo: ExceptionInfo[BaseException], rep: BaseReport ) -> BaseReport: - # XXX we re-use the TerminalReporter's terminalwriter + # XXX we reuse the TerminalReporter's terminalwriter # because this seems to avoid some encoding related troubles # for not completely clear reasons. tw = node.config.pluginmanager.getplugin("terminalreporter")._tw @@ -354,32 +362,46 @@ def _enter_pdb( tw.sep(">", "traceback") rep.toterminal(tw) tw.sep(">", "entering PDB") - tb = _postmortem_traceback(excinfo) + tb_or_exc = _postmortem_exc_or_tb(excinfo) rep._pdbshown = True # type: ignore[attr-defined] - post_mortem(tb) + post_mortem(tb_or_exc) return rep -def _postmortem_traceback(excinfo: ExceptionInfo[BaseException]) -> types.TracebackType: +def _postmortem_exc_or_tb( + excinfo: ExceptionInfo[BaseException], +) -> types.TracebackType | BaseException: from doctest import UnexpectedException + get_exc = sys.version_info >= (3, 13) if isinstance(excinfo.value, UnexpectedException): # A doctest.UnexpectedException is not useful for post_mortem. # Use the underlying exception instead: - return excinfo.value.exc_info[2] + underlying_exc = excinfo.value + if get_exc: + return underlying_exc.exc_info[1] + + return underlying_exc.exc_info[2] elif isinstance(excinfo.value, ConftestImportFailure): # A config.ConftestImportFailure is not useful for post_mortem. # Use the underlying exception instead: - assert excinfo.value.cause.__traceback__ is not None - return excinfo.value.cause.__traceback__ + cause = excinfo.value.cause + if get_exc: + return cause + + assert cause.__traceback__ is not None + return cause.__traceback__ else: assert excinfo._excinfo is not None + if get_exc: + return excinfo._excinfo[1] + return excinfo._excinfo[2] -def post_mortem(t: types.TracebackType) -> None: +def post_mortem(tb_or_exc: types.TracebackType | BaseException) -> None: p = pytestPDB._init_pdb("post_mortem") p.reset() - p.interaction(None, t) + p.interaction(None, tb_or_exc) if p.quitting: outcomes.exit("Quitting debugger") diff --git a/src/_pytest/deprecated.py b/src/_pytest/deprecated.py index a605c24e58f..cb5d2e93e93 100644 --- a/src/_pytest/deprecated.py +++ b/src/_pytest/deprecated.py @@ -15,6 +15,7 @@ from _pytest.warning_types import PytestDeprecationWarning from _pytest.warning_types import PytestRemovedIn9Warning +from _pytest.warning_types import PytestRemovedIn10Warning from _pytest.warning_types import UnformattedWarning @@ -24,11 +25,11 @@ "pytest_catchlog", "pytest_capturelog", "pytest_faulthandler", + "pytest_subtests", } -# This can be* removed pytest 8, but it's harmless and common, so no rush to remove. -# * If you're in the future: "could have been". +# This could have been removed pytest 8, but it's harmless and common, so no rush to remove. YIELD_FIXTURE = PytestDeprecationWarning( "@pytest.yield_fixture is deprecated.\n" "Use @pytest.fixture instead; they are the same." @@ -67,6 +68,13 @@ "See docs: https://docs.pytest.org/en/stable/deprecations.html#applying-a-mark-to-a-fixture-function" ) +MONKEYPATCH_LEGACY_NAMESPACE_PACKAGES = PytestRemovedIn10Warning( + "monkeypatch.syspath_prepend() called with pkg_resources legacy namespace packages detected.\n" + "Legacy namespace packages (using pkg_resources.declare_namespace) are deprecated.\n" + "Please use native namespace packages (PEP 420) instead.\n" + "See https://docs.pytest.org/en/stable/deprecations.html#monkeypatch-fixup-namespace-packages" +) + # You want to make some `__init__` or function "private". # # def my_private_function(some, args): diff --git a/src/_pytest/doctest.py b/src/_pytest/doctest.py index cb46d9a3bb5..cd255f5eeb6 100644 --- a/src/_pytest/doctest.py +++ b/src/_pytest/doctest.py @@ -4,21 +4,21 @@ from __future__ import annotations import bdb +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Sequence from contextlib import contextmanager import functools import inspect import os from pathlib import Path import platform +import re import sys import traceback import types from typing import Any -from typing import Callable -from typing import Generator -from typing import Iterable -from typing import Pattern -from typing import Sequence from typing import TYPE_CHECKING import warnings @@ -44,7 +44,8 @@ if TYPE_CHECKING: import doctest - from typing import Self + + from typing_extensions import Self DOCTEST_REPORT_CHOICE_NONE = "none" DOCTEST_REPORT_CHOICE_CDIFF = "cdiff" @@ -323,7 +324,7 @@ def repr_failure( # type: ignore[override] Sequence[doctest.DocTestFailure | doctest.UnexpectedException] | None ) = None if isinstance( - excinfo.value, (doctest.DocTestFailure, doctest.UnexpectedException) + excinfo.value, doctest.DocTestFailure | doctest.UnexpectedException ): failures = [excinfo.value] elif isinstance(excinfo.value, MultipleDoctestFailures): @@ -352,7 +353,7 @@ def repr_failure( # type: ignore[override] # add line numbers to the left of the error message assert test.lineno is not None lines = [ - "%03d %s" % (i + test.lineno + 1, x) for (i, x) in enumerate(lines) + f"{i + test.lineno + 1:03d} {x}" for (i, x) in enumerate(lines) ] # trim docstring error lines to 10 lines = lines[max(example.lineno - 9, 0) : example.lineno + 1] @@ -467,7 +468,7 @@ def _is_mocked(obj: object) -> bool: @contextmanager -def _patch_unwrap_mock_aware() -> Generator[None, None, None]: +def _patch_unwrap_mock_aware() -> Generator[None]: """Context manager which replaces ``inspect.unwrap`` with a version that's aware of mock objects and doesn't recurse into them.""" real_unwrap = inspect.unwrap @@ -529,24 +530,6 @@ def _find_lineno(self, obj, source_lines): source_lines, ) - if sys.version_info < (3, 10): - - def _find( - self, tests, obj, name, module, source_lines, globs, seen - ) -> None: - """Override _find to work around issue in stdlib. - - https://github.com/pytest-dev/pytest/issues/3456 - https://github.com/python/cpython/issues/69718 - """ - if _is_mocked(obj): - return # pragma: no cover - with _patch_unwrap_mock_aware(): - # Type ignored because this is a private function. - super()._find( # type:ignore[misc] - tests, obj, name, module, source_lines, globs, seen - ) - if sys.version_info < (3, 13): def _from_module(self, module, object): @@ -592,7 +575,6 @@ def _from_module(self, module, object): def _init_checker_class() -> type[doctest.OutputChecker]: import doctest - import re class LiteralsOutputChecker(doctest.OutputChecker): # Based on doctest_nose_plugin.py from the nltk project @@ -635,7 +617,7 @@ def check_output(self, want: str, got: str, optionflags: int) -> bool: if not allow_unicode and not allow_bytes and not allow_number: return False - def remove_prefixes(regex: Pattern[str], txt: str) -> str: + def remove_prefixes(regex: re.Pattern[str], txt: str) -> str: return re.sub(regex, r"\1\2", txt) if allow_unicode: @@ -657,7 +639,7 @@ def _remove_unwanted_precision(self, want: str, got: str) -> str: if len(wants) != len(gots): return got offset = 0 - for w, g in zip(wants, gots): + for w, g in zip(wants, gots, strict=True): fraction: str | None = w.group("fraction") exponent: str | None = w.group("exponent1") if exponent is None: diff --git a/src/_pytest/faulthandler.py b/src/_pytest/faulthandler.py index 07e60f03fc9..080cf583813 100644 --- a/src/_pytest/faulthandler.py +++ b/src/_pytest/faulthandler.py @@ -1,8 +1,8 @@ from __future__ import annotations +from collections.abc import Generator import os import sys -from typing import Generator from _pytest.config import Config from _pytest.config.argparsing import Parser @@ -16,11 +16,18 @@ def pytest_addoption(parser: Parser) -> None: - help = ( + help_timeout = ( "Dump the traceback of all threads if a test takes " "more than TIMEOUT seconds to finish" ) - parser.addini("faulthandler_timeout", help, default=0.0) + help_exit_on_timeout = ( + "Exit the test process if a test takes more than " + "faulthandler_timeout seconds to finish" + ) + parser.addini("faulthandler_timeout", help_timeout, default=0.0) + parser.addini( + "faulthandler_exit_on_timeout", help_exit_on_timeout, type="bool", default=False + ) def pytest_configure(config: Config) -> None: @@ -64,6 +71,7 @@ def get_stderr_fileno() -> int: # pytest-xdist monkeypatches sys.stderr with an object that is not an actual file. # https://docs.python.org/3/library/faulthandler.html#issue-with-file-descriptors # This is potentially dangerous, but the best we can do. + assert sys.__stderr__ is not None return sys.__stderr__.fileno() @@ -71,14 +79,21 @@ def get_timeout_config_value(config: Config) -> float: return float(config.getini("faulthandler_timeout") or 0.0) +def get_exit_on_timeout_config_value(config: Config) -> bool: + exit_on_timeout = config.getini("faulthandler_exit_on_timeout") + assert isinstance(exit_on_timeout, bool) + return exit_on_timeout + + @pytest.hookimpl(wrapper=True, trylast=True) def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]: timeout = get_timeout_config_value(item.config) + exit_on_timeout = get_exit_on_timeout_config_value(item.config) if timeout > 0: import faulthandler stderr = item.config.stash[fault_handler_stderr_fd_key] - faulthandler.dump_traceback_later(timeout, file=stderr) + faulthandler.dump_traceback_later(timeout, file=stderr, exit=exit_on_timeout) try: return (yield) finally: diff --git a/src/_pytest/fixtures.py b/src/_pytest/fixtures.py index 7d0b40b150a..27846db13a4 100644 --- a/src/_pytest/fixtures.py +++ b/src/_pytest/fixtures.py @@ -4,6 +4,15 @@ import abc from collections import defaultdict from collections import deque +from collections import OrderedDict +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import MutableMapping +from collections.abc import Sequence +from collections.abc import Set as AbstractSet import dataclasses import functools import inspect @@ -11,28 +20,15 @@ from pathlib import Path import sys import types -from typing import AbstractSet from typing import Any -from typing import Callable from typing import cast -from typing import Dict from typing import Final from typing import final -from typing import Generator from typing import Generic -from typing import Iterable -from typing import Iterator -from typing import Mapping -from typing import MutableMapping from typing import NoReturn -from typing import Optional -from typing import OrderedDict from typing import overload -from typing import Sequence -from typing import Tuple from typing import TYPE_CHECKING from typing import TypeVar -from typing import Union import warnings import _pytest @@ -42,17 +38,16 @@ from _pytest._code.code import FormattedExcinfo from _pytest._code.code import TerminalRepr from _pytest._io import TerminalWriter -from _pytest.compat import _PytestWrapper from _pytest.compat import assert_never from _pytest.compat import get_real_func -from _pytest.compat import get_real_method from _pytest.compat import getfuncargnames from _pytest.compat import getimfunc from _pytest.compat import getlocation -from _pytest.compat import is_generator from _pytest.compat import NOTSET from _pytest.compat import NotSetType from _pytest.compat import safe_getattr +from _pytest.compat import safe_isclass +from _pytest.compat import signature from _pytest.config import _PluggyPlugin from _pytest.config import Config from _pytest.config import ExitCode @@ -72,6 +67,8 @@ from _pytest.scope import _ScopeName from _pytest.scope import HIGH_SCOPES from _pytest.scope import Scope +from _pytest.warning_types import PytestRemovedIn9Warning +from _pytest.warning_types import PytestWarning if sys.version_info < (3, 11): @@ -85,36 +82,28 @@ # The value of the fixture -- return/yield of the fixture function (type variable). -FixtureValue = TypeVar("FixtureValue") +FixtureValue = TypeVar("FixtureValue", covariant=True) # The type of the fixture function (type variable). FixtureFunction = TypeVar("FixtureFunction", bound=Callable[..., object]) # The type of a fixture function (type alias generic in fixture value). -_FixtureFunc = Union[ - Callable[..., FixtureValue], Callable[..., Generator[FixtureValue, None, None]] -] +_FixtureFunc = Callable[..., FixtureValue] | Callable[..., Generator[FixtureValue]] # The type of FixtureDef.cached_result (type alias generic in fixture value). -_FixtureCachedResult = Union[ - Tuple[ +_FixtureCachedResult = ( + tuple[ # The result. FixtureValue, # Cache key. object, None, - ], - Tuple[ + ] + | tuple[ None, # Cache key. object, # The exception and the original traceback. - Tuple[BaseException, Optional[types.TracebackType]], - ], -] - - -@dataclasses.dataclass(frozen=True) -class PseudoFixtureDef(Generic[FixtureValue]): - cached_result: _FixtureCachedResult[FixtureValue] - _scope: Scope + tuple[BaseException, types.TracebackType | None], + ] +) def pytest_sessionstart(session: Session) -> None: @@ -134,6 +123,12 @@ def get_scope_package( def get_scope_node(node: nodes.Node, scope: Scope) -> nodes.Node | None: + """Get the closest parent node (including self) which matches the given + scope. + + If there is no parent node for the scope (e.g. asking for class scope on a + Module, or on a Function when not defined in a class), returns None. + """ import _pytest.python if scope is Scope.Function: @@ -152,13 +147,12 @@ def get_scope_node(node: nodes.Node, scope: Scope) -> nodes.Node | None: assert_never(scope) +# TODO: Try to use FixtureFunctionDefinition instead of the marker def getfixturemarker(obj: object) -> FixtureFunctionMarker | None: - """Return fixturemarker or None if it doesn't exist or raised - exceptions.""" - return cast( - Optional[FixtureFunctionMarker], - safe_getattr(obj, "_pytestfixturefunction", None), - ) + """Return fixturemarker or None if it doesn't exist""" + if isinstance(obj, FixtureFunctionDefinition): + return obj._fixture_function_marker + return None # Algorithm for sorting on a per-parametrized resource setup basis. @@ -168,22 +162,30 @@ def getfixturemarker(obj: object) -> FixtureFunctionMarker | None: @dataclasses.dataclass(frozen=True) -class FixtureArgKey: +class ParamArgKey: + """A key for a high-scoped parameter used by an item. + + For use as a hashable key in `reorder_items`. The combination of fields + is meant to uniquely identify a particular "instance" of a param, + potentially shared by multiple items in a scope. + """ + + #: The param name. argname: str param_index: int + #: For scopes Package, Module, Class, the path to the file (directory in + #: Package's case) of the package/module/class where the item is defined. scoped_item_path: Path | None + #: For Class scope, the class where the item is defined. item_cls: type | None _V = TypeVar("_V") -OrderedSet = Dict[_V, None] +OrderedSet = dict[_V, None] -def get_parametrized_fixture_argkeys( - item: nodes.Item, scope: Scope -) -> Iterator[FixtureArgKey]: - """Return list of keys for all parametrized arguments which match - the specified scope.""" +def get_param_argkeys(item: nodes.Item, scope: Scope) -> Iterator[ParamArgKey]: + """Return all ParamArgKeys for item matching the specified high scope.""" assert scope is not Scope.Function try: @@ -209,19 +211,17 @@ def get_parametrized_fixture_argkeys( if callspec._arg2scope[argname] != scope: continue param_index = callspec.indices[argname] - yield FixtureArgKey(argname, param_index, scoped_item_path, item_cls) + yield ParamArgKey(argname, param_index, scoped_item_path, item_cls) def reorder_items(items: Sequence[nodes.Item]) -> list[nodes.Item]: - argkeys_by_item: dict[Scope, dict[nodes.Item, OrderedSet[FixtureArgKey]]] = {} - items_by_argkey: dict[ - Scope, dict[FixtureArgKey, OrderedDict[nodes.Item, None]] - ] = {} + argkeys_by_item: dict[Scope, dict[nodes.Item, OrderedSet[ParamArgKey]]] = {} + items_by_argkey: dict[Scope, dict[ParamArgKey, OrderedDict[nodes.Item, None]]] = {} for scope in HIGH_SCOPES: scoped_argkeys_by_item = argkeys_by_item[scope] = {} scoped_items_by_argkey = items_by_argkey[scope] = defaultdict(OrderedDict) for item in items: - argkeys = dict.fromkeys(get_parametrized_fixture_argkeys(item, scope)) + argkeys = dict.fromkeys(get_param_argkeys(item, scope)) if argkeys: scoped_argkeys_by_item[item] = argkeys for argkey in argkeys: @@ -237,9 +237,9 @@ def reorder_items(items: Sequence[nodes.Item]) -> list[nodes.Item]: def reorder_items_atscope( items: OrderedSet[nodes.Item], - argkeys_by_item: Mapping[Scope, Mapping[nodes.Item, OrderedSet[FixtureArgKey]]], + argkeys_by_item: Mapping[Scope, Mapping[nodes.Item, OrderedSet[ParamArgKey]]], items_by_argkey: Mapping[ - Scope, Mapping[FixtureArgKey, OrderedDict[nodes.Item, None]] + Scope, Mapping[ParamArgKey, OrderedDict[nodes.Item, None]] ], scope: Scope, ) -> OrderedSet[nodes.Item]: @@ -249,7 +249,7 @@ def reorder_items_atscope( scoped_items_by_argkey = items_by_argkey[scope] scoped_argkeys_by_item = argkeys_by_item[scope] - ignore: set[FixtureArgKey] = set() + ignore: set[ParamArgKey] = set() items_deque = deque(items) items_done: OrderedSet[nodes.Item] = {} while items_deque: @@ -277,10 +277,18 @@ def reorder_items_atscope( for other_scope in HIGH_SCOPES: other_scoped_items_by_argkey = items_by_argkey[other_scope] for argkey in argkeys_by_item[other_scope].get(i, ()): - other_scoped_items_by_argkey[argkey][i] = None - other_scoped_items_by_argkey[argkey].move_to_end( - i, last=False - ) + argkey_dict = other_scoped_items_by_argkey[argkey] + if not hasattr(sys, "pypy_version_info"): + argkey_dict[i] = None + argkey_dict.move_to_end(i, last=False) + else: + # Work around a bug in PyPy: + # https://github.com/pypy/pypy/issues/5257 + # https://github.com/pytest-dev/pytest/issues/13312 + bkp = argkey_dict.copy() + argkey_dict.clear() + argkey_dict[i] = None + argkey_dict.update(bkp) break if no_argkey_items: reordered_no_argkey_items = reorder_items_atscope( @@ -306,7 +314,7 @@ class FuncFixtureInfo: these are not reflected here. """ - __slots__ = ("argnames", "initialnames", "names_closure", "name2fixturedefs") + __slots__ = ("argnames", "initialnames", "name2fixturedefs", "names_closure") # Fixture names that the item requests directly by function parameters. argnames: tuple[str, ...] @@ -406,7 +414,7 @@ def scope(self) -> _ScopeName: @abc.abstractmethod def _check_scope( self, - requested_fixturedef: FixtureDef[object] | PseudoFixtureDef[object], + requested_fixturedef: FixtureDef[object], requested_scope: Scope, ) -> None: raise NotImplementedError() @@ -545,12 +553,9 @@ def _iter_chain(self) -> Iterator[SubRequest]: yield current current = current._parent_request - def _get_active_fixturedef( - self, argname: str - ) -> FixtureDef[object] | PseudoFixtureDef[object]: + def _get_active_fixturedef(self, argname: str) -> FixtureDef[object]: if argname == "request": - cached_result = (self, [0], None) - return PseudoFixtureDef(cached_result, Scope.Function) + return RequestFixtureDef(self) # If we already finished computing a fixture by this name in this item, # return it. @@ -574,6 +579,7 @@ def _get_active_fixturedef( # The are no fixtures with this name applicable for the function. if not fixturedefs: raise FixtureLookupError(argname, self) + # A fixture may override another fixture with the same name, e.g. a # fixture in a module can override a fixture in a conftest, a fixture in # a class can override a fixture in the module, and so on. @@ -607,7 +613,12 @@ def _get_active_fixturedef( param_index = 0 scope = fixturedef._scope self._check_fixturedef_without_param(fixturedef) - self._check_scope(fixturedef, scope) + # The parametrize invocation scope only controls caching behavior while + # allowing wider-scoped fixtures to keep depending on the parametrized + # fixture. Scope control is enforced for parametrized fixtures + # by recreating the whole fixture tree on parameter change. + # Hence `fixturedef._scope`, not `scope`. + self._check_scope(fixturedef, fixturedef._scope) subrequest = SubRequest( self, scope, param, param_index, fixturedef, _ispytest=True ) @@ -676,7 +687,7 @@ def _scope(self) -> Scope: def _check_scope( self, - requested_fixturedef: FixtureDef[object] | PseudoFixtureDef[object], + requested_fixturedef: FixtureDef[object], requested_scope: Scope, ) -> None: # TopRequest always has function scope so always valid. @@ -748,16 +759,16 @@ def node(self): if node is None and scope is Scope.Class: # Fallback to function item itself. node = self._pyfuncitem - assert node, f'Could not obtain a node for scope "{scope}" for function {self._pyfuncitem!r}' + assert node, ( + f'Could not obtain a node for scope "{scope}" for function {self._pyfuncitem!r}' + ) return node def _check_scope( self, - requested_fixturedef: FixtureDef[object] | PseudoFixtureDef[object], + requested_fixturedef: FixtureDef[object], requested_scope: Scope, ) -> None: - if isinstance(requested_fixturedef, PseudoFixtureDef): - return if self._scope > requested_scope: # Try to report something helpful. argname = requested_fixturedef.argname @@ -779,8 +790,8 @@ def _format_fixturedef_line(self, fixturedef: FixtureDef[object]) -> str: path, lineno = getfslineno(factory) if isinstance(path, Path): path = bestrelpath(self._pyfuncitem.session.path, path) - signature = inspect.signature(factory) - return f"{path}:{lineno + 1}: def {factory.__name__}{signature}" + sig = signature(factory) + return f"{path}:{lineno + 1}: def {factory.__name__}{sig}" def addfinalizer(self, finalizer: Callable[[], object]) -> None: self._fixturedef.addfinalizer(finalizer) @@ -804,6 +815,15 @@ def formatrepr(self) -> FixtureLookupErrorRepr: stack = [self.request._pyfuncitem.obj] stack.extend(map(lambda x: x.func, self.fixturestack)) msg = self.msg + # This function currently makes an assumption that a non-None msg means we + # have a non-empty `self.fixturestack`. This is currently true, but if + # somebody at some point want to extend the use of FixtureLookupError to + # new cases it might break. + # Add the assert to make it clearer to developer that this will fail, otherwise + # it crashes because `fspath` does not get set due to `stack` being empty. + assert self.msg is None or self.fixturestack, ( + "formatrepr assumptions broken, rewrite it to handle it" + ) if msg is not None: # The last fixture raise an error, let's present # it at the requesting side. @@ -875,16 +895,14 @@ def toterminal(self, tw: TerminalWriter) -> None: red=True, ) tw.line() - tw.line("%s:%d" % (os.fspath(self.filename), self.firstlineno + 1)) + tw.line(f"{os.fspath(self.filename)}:{self.firstlineno + 1}") def call_fixture_func( fixturefunc: _FixtureFunc[FixtureValue], request: FixtureRequest, kwargs ) -> FixtureValue: - if is_generator(fixturefunc): - fixturefunc = cast( - Callable[..., Generator[FixtureValue, None, None]], fixturefunc - ) + if inspect.isgeneratorfunction(fixturefunc): + fixturefunc = cast(Callable[..., Generator[FixtureValue]], fixturefunc) generator = fixturefunc(**kwargs) try: fixture_result = next(generator) @@ -939,7 +957,6 @@ def _eval_scope_callable( return result -@final class FixtureDef(Generic[FixtureValue]): """A container for a fixture definition. @@ -958,6 +975,8 @@ def __init__( ids: tuple[object | None, ...] | Callable[[Any], object | None] | None = None, *, _ispytest: bool = False, + # only used in a deprecationwarning msg, can be removed in pytest9 + _autouse: bool = False, ) -> None: check_ispytest(_ispytest) # The "base" node ID for the fixture. @@ -1004,6 +1023,9 @@ def __init__( self.cached_result: _FixtureCachedResult[FixtureValue] | None = None self._finalizers: Final[list[Callable[[], object]]] = [] + # only used to emit a deprecationwarning, can be removed in pytest9 + self._autouse = _autouse + @property def scope(self) -> _ScopeName: """Scope string, one of "function", "class", "module", "package", "session".""" @@ -1049,8 +1071,7 @@ def execute(self, request: SubRequest) -> FixtureValue: # down first. This is generally handled by SetupState, but still currently # needed when this fixture is not parametrized but depends on a parametrized # fixture. - if not isinstance(fixturedef, PseudoFixtureDef): - requested_fixtures_that_should_finalize_us.append(fixturedef) + requested_fixtures_that_should_finalize_us.append(fixturedef) # Check for (and return) cached value/exception. if self.cached_result is not None: @@ -1069,8 +1090,7 @@ def execute(self, request: SubRequest) -> FixtureValue: exc, exc_tb = self.cached_result[2] raise exc.with_traceback(exc_tb) else: - result = self.cached_result[0] - return result + return self.cached_result[0] # We have a previous but differently parametrized fixture instance # so we need to tear it down before creating a new one. self.finish(request) @@ -1086,10 +1106,12 @@ def execute(self, request: SubRequest) -> FixtureValue: ihook = request.node.ihook try: # Setup the fixture, run the code in it, and cache the value - # in self.cached_result - result = ihook.pytest_fixture_setup(fixturedef=self, request=request) + # in self.cached_result. + result: FixtureValue = ihook.pytest_fixture_setup( + fixturedef=self, request=request + ) finally: - # schedule our finalizer, even if the setup failed + # Schedule our finalizer, even if the setup failed. request.node.addfinalizer(finalizer) return result @@ -1101,6 +1123,28 @@ def __repr__(self) -> str: return f"" +class RequestFixtureDef(FixtureDef[FixtureRequest]): + """A custom FixtureDef for the special "request" fixture. + + A new one is generated on-demand whenever "request" is requested. + """ + + def __init__(self, request: FixtureRequest) -> None: + super().__init__( + config=request.config, + baseid=None, + argname="request", + func=lambda: request, + scope=Scope.Function, + params=None, + _ispytest=True, + ) + self.cached_result = (request, [0], None) + + def addfinalizer(self, finalizer: Callable[[], object]) -> None: + pass + + def resolve_fixture_function( fixturedef: FixtureDef[FixtureValue], request: FixtureRequest ) -> _FixtureFunc[FixtureValue]: @@ -1135,6 +1179,25 @@ def pytest_fixture_setup( fixturefunc = resolve_fixture_function(fixturedef, request) my_cache_key = fixturedef.cache_key(request) + + if inspect.isasyncgenfunction(fixturefunc) or inspect.iscoroutinefunction( + fixturefunc + ): + auto_str = " with autouse=True" if fixturedef._autouse else "" + + warnings.warn( + PytestRemovedIn9Warning( + f"{request.node.name!r} requested an async fixture " + f"{request.fixturename!r}{auto_str}, with no plugin or hook that " + "handled it. This is usually an error, as pytest does not natively " + "support it. " + "This will turn into an error in pytest 9.\n" + "See: https://docs.pytest.org/en/stable/deprecations.html#sync-test-depending-on-async-fixture" + ), + # no stacklevel will point at users code, so we just point here + stacklevel=1, + ) + try: result = call_fixture_func(fixturefunc, request, kwargs) except TEST_OUTCOME as e: @@ -1149,31 +1212,6 @@ def pytest_fixture_setup( return result -def wrap_function_to_error_out_if_called_directly( - function: FixtureFunction, - fixture_marker: FixtureFunctionMarker, -) -> FixtureFunction: - """Wrap the given fixture function so we can raise an error about it being called directly, - instead of used as an argument in a test function.""" - name = fixture_marker.name or function.__name__ - message = ( - f'Fixture "{name}" called directly. Fixtures are not meant to be called directly,\n' - "but are created automatically when test functions request them as parameters.\n" - "See https://docs.pytest.org/en/stable/explanation/fixtures.html for more information about fixtures, and\n" - "https://docs.pytest.org/en/stable/deprecations.html#calling-fixtures-directly about how to update your code." - ) - - @functools.wraps(function) - def result(*args, **kwargs): - fail(message, pytrace=False) - - # Keep reference to the original function in our own custom attribute so we don't unwrap - # further than this point and lose useful wrappings like @mock.patch (#3774). - result.__pytest_wrapped__ = _PytestWrapper(function) # type: ignore[attr-defined] - - return cast(FixtureFunction, result) - - @final @dataclasses.dataclass(frozen=True) class FixtureFunctionMarker: @@ -1188,11 +1226,11 @@ class FixtureFunctionMarker: def __post_init__(self, _ispytest: bool) -> None: check_ispytest(_ispytest) - def __call__(self, function: FixtureFunction) -> FixtureFunction: + def __call__(self, function: FixtureFunction) -> FixtureFunctionDefinition: if inspect.isclass(function): raise ValueError("class fixtures not supported (maybe in the future)") - if getattr(function, "_pytestfixturefunction", False): + if isinstance(function, FixtureFunctionDefinition): raise ValueError( f"@pytest.fixture is being applied more than once to the same function {function.__name__!r}" ) @@ -1200,7 +1238,9 @@ def __call__(self, function: FixtureFunction) -> FixtureFunction: if hasattr(function, "pytestmark"): warnings.warn(MARKED_FIXTURE, stacklevel=2) - function = wrap_function_to_error_out_if_called_directly(function, self) + fixture_definition = FixtureFunctionDefinition( + function=function, fixture_function_marker=self, _ispytest=True + ) name = self.name or function.__name__ if name == "request": @@ -1210,21 +1250,68 @@ def __call__(self, function: FixtureFunction) -> FixtureFunction: pytrace=False, ) - # Type ignored because https://github.com/python/mypy/issues/2087. - function._pytestfixturefunction = self # type: ignore[attr-defined] - return function + return fixture_definition + + +# TODO: paramspec/return type annotation tracking and storing +class FixtureFunctionDefinition: + def __init__( + self, + *, + function: Callable[..., Any], + fixture_function_marker: FixtureFunctionMarker, + instance: object | None = None, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self.name = fixture_function_marker.name or function.__name__ + # In order to show the function that this fixture contains in messages. + # Set the __name__ to be same as the function __name__ or the given fixture name. + self.__name__ = self.name + self._fixture_function_marker = fixture_function_marker + if instance is not None: + self._fixture_function = cast( + Callable[..., Any], function.__get__(instance) + ) + else: + self._fixture_function = function + functools.update_wrapper(self, function) + + def __repr__(self) -> str: + return f"" + + def __get__(self, instance, owner=None): + """Behave like a method if the function it was applied to was a method.""" + return FixtureFunctionDefinition( + function=self._fixture_function, + fixture_function_marker=self._fixture_function_marker, + instance=instance, + _ispytest=True, + ) + + def __call__(self, *args: Any, **kwds: Any) -> Any: + message = ( + f'Fixture "{self.name}" called directly. Fixtures are not meant to be called directly,\n' + "but are created automatically when test functions request them as parameters.\n" + "See https://docs.pytest.org/en/stable/explanation/fixtures.html for more information about fixtures, and\n" + "https://docs.pytest.org/en/stable/deprecations.html#calling-fixtures-directly" + ) + fail(message, pytrace=False) + + def _get_wrapped_function(self) -> Callable[..., Any]: + return self._fixture_function @overload def fixture( - fixture_function: FixtureFunction, + fixture_function: Callable[..., object], *, scope: _ScopeName | Callable[[str, Config], _ScopeName] = ..., params: Iterable[object] | None = ..., autouse: bool = ..., ids: Sequence[object | None] | Callable[[Any], object | None] | None = ..., name: str | None = ..., -) -> FixtureFunction: ... +) -> FixtureFunctionDefinition: ... @overload @@ -1247,7 +1334,7 @@ def fixture( autouse: bool = False, ids: Sequence[object | None] | Callable[[Any], object | None] | None = None, name: str | None = None, -) -> FixtureFunctionMarker | FixtureFunction: +) -> FixtureFunctionMarker | FixtureFunctionDefinition: """Decorator to mark a fixture factory function. This decorator can be used, with or without parameters, to define a @@ -1348,7 +1435,7 @@ def pytestconfig(request: FixtureRequest) -> Config: Example:: def test_foo(pytestconfig): - if pytestconfig.getoption("verbose") > 0: + if pytestconfig.get_verbosity() > 0: ... """ @@ -1433,7 +1520,7 @@ class FixtureManager: relevant for a particular function. An initial list of fixtures is assembled like this: - - ini-defined usefixtures + - config-defined usefixtures - autouse-marked fixtures along the collection chain up from the function - usefixtures markers at module/class/function level - test function funcargs @@ -1531,7 +1618,13 @@ def _getautousenames(self, node: nodes.Node) -> Iterator[str]: def _getusefixturesnames(self, node: nodes.Item) -> Iterator[str]: """Return the names of usefixtures fixtures applicable to node.""" - for mark in node.iter_markers(name="usefixtures"): + for marker_node, mark in node.iter_markers_with_node(name="usefixtures"): + if not mark.args: + marker_node.warn( + PytestWarning( + f"usefixtures() in {node.nodeid} without arguments has no effect" + ) + ) yield from mark.args def getfixtureclosure( @@ -1550,20 +1643,44 @@ def getfixtureclosure( fixturenames_closure = list(initialnames) arg2fixturedefs: dict[str, Sequence[FixtureDef[Any]]] = {} - lastlen = -1 - while lastlen != len(fixturenames_closure): - lastlen = len(fixturenames_closure) - for argname in fixturenames_closure: - if argname in ignore_args: - continue - if argname in arg2fixturedefs: - continue + + # Track the index for each fixture name in the simulated stack. + # Needed for handling override chains correctly, similar to _get_active_fixturedef. + # Using negative indices: -1 is the most specific (last), -2 is second to last, etc. + current_indices: dict[str, int] = {} + + def process_argname(argname: str) -> None: + # Optimization: already processed this argname. + if current_indices.get(argname) == -1: + return + + if argname not in fixturenames_closure: + fixturenames_closure.append(argname) + + if argname in ignore_args: + return + + fixturedefs = arg2fixturedefs.get(argname) + if not fixturedefs: fixturedefs = self.getfixturedefs(argname, parentnode) - if fixturedefs: - arg2fixturedefs[argname] = fixturedefs - for arg in fixturedefs[-1].argnames: - if arg not in fixturenames_closure: - fixturenames_closure.append(arg) + if not fixturedefs: + # Fixture not defined or not visible (will error during runtest). + return + arg2fixturedefs[argname] = fixturedefs + + index = current_indices.get(argname, -1) + if -index > len(fixturedefs): + # Exhausted the override chain (will error during runtest). + return + fixturedef = fixturedefs[index] + + current_indices[argname] = index - 1 + for dep in fixturedef.argnames: + process_argname(dep) + current_indices[argname] = index + + for name in initialnames: + process_argname(name) def sort_by_scope(arg_name: str) -> Scope: try: @@ -1665,6 +1782,7 @@ def _register_fixture( params=params, ids=ids, _ispytest=True, + _autouse=autouse, ) faclist = self._arg2fixturedefs.setdefault(name, []) @@ -1724,35 +1842,42 @@ def parsefactories( if holderobj in self._holderobjseen: return + # Avoid accessing `@property` (and other descriptors) when iterating fixtures. + if not safe_isclass(holderobj) and not isinstance(holderobj, types.ModuleType): + holderobj_tp: object = type(holderobj) + else: + holderobj_tp = holderobj + self._holderobjseen.add(holderobj) for name in dir(holderobj): # The attribute can be an arbitrary descriptor, so the attribute - # access below can raise. safe_getatt() ignores such exceptions. - obj = safe_getattr(holderobj, name, None) - marker = getfixturemarker(obj) - if not isinstance(marker, FixtureFunctionMarker): - # Magic globals with __getattr__ might have got us a wrong - # fixture attribute. - continue - - if marker.name: - name = marker.name - - # During fixture definition we wrap the original fixture function - # to issue a warning if called directly, so here we unwrap it in - # order to not emit the warning when pytest itself calls the - # fixture function. - func = get_real_method(obj, holderobj) - - self._register_fixture( - name=name, - nodeid=nodeid, - func=func, - scope=marker.scope, - params=marker.params, - ids=marker.ids, - autouse=marker.autouse, - ) + # access below can raise. safe_getattr() ignores such exceptions. + obj_ub = safe_getattr(holderobj_tp, name, None) + if type(obj_ub) is FixtureFunctionDefinition: + marker = obj_ub._fixture_function_marker + if marker.name: + fixture_name = marker.name + else: + fixture_name = name + + # OK we know it is a fixture -- now safe to look up on the _instance_. + try: + obj = getattr(holderobj, name) + # if the fixture is named in the decorator we cannot find it in the module + except AttributeError: + obj = obj_ub + + func = obj._get_wrapped_function() + + self._register_fixture( + name=fixture_name, + nodeid=nodeid, + func=func, + scope=marker.scope, + params=marker.params, + ids=marker.ids, + autouse=marker.autouse, + ) def getfixturedefs( self, argname: str, node: nodes.Node @@ -1807,7 +1932,7 @@ def _show_fixtures_per_test(config: Config, session: Session) -> None: session.perform_collect() invocation_dir = config.invocation_params.dir tw = _pytest.config.create_terminal_writer(config) - verbose = config.getvalue("verbose") + verbose = config.get_verbosity() def get_best_relpath(func) -> str: loc = getlocation(func, invocation_dir) @@ -1866,7 +1991,7 @@ def _showfixtures_main(config: Config, session: Session) -> None: session.perform_collect() invocation_dir = config.invocation_params.dir tw = _pytest.config.create_terminal_writer(config) - verbose = config.getvalue("verbose") + verbose = config.get_verbosity() fm = session._fixturemanager diff --git a/src/_pytest/freeze_support.py b/src/_pytest/freeze_support.py index 2ba6f9b8bcc..959ff071d86 100644 --- a/src/_pytest/freeze_support.py +++ b/src/_pytest/freeze_support.py @@ -3,8 +3,8 @@ from __future__ import annotations +from collections.abc import Iterator import types -from typing import Iterator def freeze_includes() -> list[str]: diff --git a/src/_pytest/helpconfig.py b/src/_pytest/helpconfig.py index 1886d5c9342..6a22c9f58ac 100644 --- a/src/_pytest/helpconfig.py +++ b/src/_pytest/helpconfig.py @@ -3,10 +3,12 @@ from __future__ import annotations -from argparse import Action +import argparse +from collections.abc import Generator +from collections.abc import Sequence import os import sys -from typing import Generator +from typing import Any from _pytest.config import Config from _pytest.config import ExitCode @@ -16,31 +18,41 @@ import pytest -class HelpAction(Action): - """An argparse Action that will raise an exception in order to skip the - rest of the argument parsing when --help is passed. +class HelpAction(argparse.Action): + """An argparse Action that will raise a PrintHelp exception in order to skip + the rest of the argument parsing when --help is passed. - This prevents argparse from quitting due to missing required arguments - when any are defined, for example by ``pytest_addoption``. - This is similar to the way that the builtin argparse --help option is - implemented by raising SystemExit. + This prevents argparse from raising UsageError when `--help` is used along + with missing required arguments when any are defined, for example by + ``pytest_addoption``. This is similar to the way that the builtin argparse + --help option is implemented by raising SystemExit. + + To opt in to this behavior, the parse caller must set + `namespace._raise_print_help = True`. Otherwise it just sets the option. """ - def __init__(self, option_strings, dest=None, default=False, help=None): + def __init__( + self, option_strings: Sequence[str], dest: str, *, help: str | None = None + ) -> None: super().__init__( option_strings=option_strings, dest=dest, - const=True, - default=default, nargs=0, + const=True, + default=False, help=help, ) - def __call__(self, parser, namespace, values, option_string=None): + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[Any] | None, + option_string: str | None = None, + ) -> None: setattr(namespace, self.dest, self.const) - # We should only skip the rest of the parsing after preparse is done. - if getattr(parser._parser, "after_preparse", False): + if getattr(namespace, "_raise_print_help", False): raise PrintHelp @@ -55,14 +67,14 @@ def pytest_addoption(parser: Parser) -> None: help="Display pytest version and information about plugins. " "When given twice, also display information about plugins.", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-h", "--help", action=HelpAction, dest="help", help="Show help message and configuration info", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-p", action="append", dest="plugins", @@ -70,7 +82,14 @@ def pytest_addoption(parser: Parser) -> None: metavar="name", help="Early-load given plugin module name or entry point (multi-allowed). " "To avoid loading of plugins, use the `no:` prefix, e.g. " - "`no:doctest`.", + "`no:doctest`. See also --disable-plugin-autoload.", + ) + group.addoption( + "--disable-plugin-autoload", + action="store_true", + default=False, + help="Disable plugin auto-loading through entry point packaging metadata. " + "Only plugins explicitly specified in -p or env var PYTEST_PLUGINS will be loaded.", ) group.addoption( "--traceconfig", @@ -90,13 +109,13 @@ def pytest_addoption(parser: Parser) -> None: "This file is opened with 'w' and truncated as a result, care advised. " "Default: pytestdebug.log.", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-o", "--override-ini", dest="override_ini", action="append", - help='Override ini option with "option=value" style, ' - "e.g. `-o xfail_strict=True -o cache_dir=cache`.", + help='Override configuration option with "option=value" style, ' + "e.g. `-o strict_xfail=True -o cache_dir=cache`.", ) @@ -133,28 +152,28 @@ def unset_tracing() -> None: return config -def showversion(config: Config) -> None: - if config.option.version > 1: - sys.stdout.write( - f"This is pytest version {pytest.__version__}, imported from {pytest.__file__}\n" - ) - plugininfo = getpluginversioninfo(config) - if plugininfo: - for line in plugininfo: - sys.stdout.write(line + "\n") - else: - sys.stdout.write(f"pytest {pytest.__version__}\n") +def show_version_verbose(config: Config) -> None: + """Show verbose pytest version installation, including plugins.""" + sys.stdout.write( + f"This is pytest version {pytest.__version__}, imported from {pytest.__file__}\n" + ) + plugininfo = getpluginversioninfo(config) + if plugininfo: + for line in plugininfo: + sys.stdout.write(line + "\n") def pytest_cmdline_main(config: Config) -> int | ExitCode | None: - if config.option.version > 0: - showversion(config) - return 0 + # Note: a single `--version` argument is handled directly by `Config.main()` to avoid starting up the entire + # pytest infrastructure just to display the version (#13574). + if config.option.version > 1: + show_version_verbose(config) + return ExitCode.OK elif config.option.help: config._do_configure() showhelp(config) config._ensure_unconfigure() - return 0 + return ExitCode.OK return None @@ -169,18 +188,16 @@ def showhelp(config: Config) -> None: tw.write(config._parser.optparser.format_help()) tw.line() tw.line( - "[pytest] ini-options in the first " - "pytest.ini|tox.ini|setup.cfg|pyproject.toml file found:" + "[pytest] configuration options in the first " + "pytest.toml|pytest.ini|tox.ini|setup.cfg|pyproject.toml file found:" ) tw.line() columns = tw.fullwidth # costly call indent_len = 24 # based on argparse's max_help_position=24 indent = " " * indent_len - for name in config._parser._ininames: - help, type, default = config._parser._inidict[name] - if type is None: - type = "string" + for name in config._parser._inidict: + help, type, _default = config._parser._inidict[name] if help is None: raise TypeError(f"help argument cannot be None for {name}") spec = f"{name} ({type}):" @@ -214,7 +231,7 @@ def showhelp(config: Config) -> None: vars = [ ( "CI", - "When set (regardless of value), pytest knows it is running in a " + "When set to a non-empty value, pytest knows it is running in a " "CI process and does not truncate summary info", ), ("BUILD_NUMBER", "Equivalent to CI"), @@ -222,6 +239,9 @@ def showhelp(config: Config) -> None: ("PYTEST_PLUGINS", "Comma-separated plugins to load during startup"), ("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "Set to disable plugin auto-loading"), ("PYTEST_DEBUG", "Set to enable debug tracing of pytest's internals"), + ("PYTEST_DEBUG_TEMPROOT", "Override the system temporary directory"), + ("PYTEST_THEME", "The Pygments style to use for code output"), + ("PYTEST_THEME_MODE", "Set the PYTEST_THEME to be either 'dark' or 'light'"), ] for name, help in vars: tw.line(f" {name:<24} {help}") @@ -240,9 +260,6 @@ def showhelp(config: Config) -> None: tw.line("warning : " + warningreport.message, red=True) -conftest_options = [("pytest_plugins", "list of plugin names to load")] - - def getpluginversioninfo(config: Config) -> list[str]: lines = [] plugininfo = config.pluginmanager.list_plugin_distinfo() diff --git a/src/_pytest/hookspec.py b/src/_pytest/hookspec.py index 99614899994..dab3fb698a2 100644 --- a/src/_pytest/hookspec.py +++ b/src/_pytest/hookspec.py @@ -5,10 +5,10 @@ from __future__ import annotations +from collections.abc import Mapping +from collections.abc import Sequence from pathlib import Path from typing import Any -from typing import Mapping -from typing import Sequence from typing import TYPE_CHECKING from pluggy import HookspecMarker @@ -98,13 +98,13 @@ def pytest_plugin_registered( @hookspec(historic=True) def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None: - """Register argparse-style options and ini-style config values, + """Register argparse-style options and config-style config values, called once at the beginning of a test run. :param parser: To add command line options, call :py:func:`parser.addoption(...) `. - To add ini-file values call :py:func:`parser.addini(...) + To add config-file values call :py:func:`parser.addini(...) `. :param pluginmanager: @@ -119,7 +119,7 @@ def pytest_addoption(parser: Parser, pluginmanager: PytestPluginManager) -> None retrieve the value of a command line option. - :py:func:`config.getini(name) ` to retrieve - a value read from an ini-style file. + a value read from a configuration file. The config object is passed around on many internal objects via the ``.config`` attribute or can be retrieved as the ``pytestconfig`` fixture. @@ -251,8 +251,8 @@ def pytest_collection(session: Session) -> object | None: 1. ``pytest_deselected(items)`` for any deselected items (may be called multiple times) - 3. ``pytest_collection_finish(session)`` - 4. Set ``session.items`` to the list of collected items + 3. Set ``session.items`` to the list of collected items + 4. ``pytest_collection_finish(session)`` 5. Set ``session.testscollected`` to the number of collected items You can implement this hook to only perform some action before collection, @@ -274,6 +274,11 @@ def pytest_collection_modifyitems( """Called after collection has been performed. May filter or re-order the items in-place. + When items are deselected (filtered out from ``items``), + the hook :hook:`pytest_deselected` must be called explicitly + with the deselected items to properly notify other plugins, + e.g. with ``config.hook.pytest_deselected(items=deselected_items)``. + :param session: The pytest session object. :param config: The pytest config object. :param items: List of item objects. @@ -454,6 +459,12 @@ def pytest_collectreport(report: CollectReport) -> None: def pytest_deselected(items: Sequence[Item]) -> None: """Called for deselected test items, e.g. by keyword. + Note that this hook has two integration aspects for plugins: + + - it can be *implemented* to be notified of deselected items + - it must be *called* from :hook:`pytest_collection_modifyitems` + implementations when items are deselected (to properly notify other plugins). + May be called multiple times. :param items: @@ -987,13 +998,22 @@ def pytest_assertion_pass(item: Item, lineno: int, orig: str, expl: str) -> None and the pytest introspected assertion information is available in the `expl` string. - This hook must be explicitly enabled by the ``enable_assertion_pass_hook`` - ini-file option: + This hook must be explicitly enabled by the :confval:`enable_assertion_pass_hook` + configuration option: + + .. tab:: toml + + .. code-block:: toml + + [pytest] + enable_assertion_pass_hook = true + + .. tab:: ini - .. code-block:: ini + .. code-block:: ini - [pytest] - enable_assertion_pass_hook=true + [pytest] + enable_assertion_pass_hook = true You need to **clean the .pyc** files in your project directory and interpreter libraries when enabling this option, as assertions will require to be re-written. diff --git a/src/_pytest/junitxml.py b/src/_pytest/junitxml.py index 3a2cb59a6c1..ae8d2b94d36 100644 --- a/src/_pytest/junitxml.py +++ b/src/_pytest/junitxml.py @@ -10,14 +10,11 @@ from __future__ import annotations -from datetime import datetime -from datetime import timezone +from collections.abc import Callable import functools import os import platform import re -from typing import Callable -from typing import Match import xml.etree.ElementTree as ET from _pytest import nodes @@ -48,7 +45,7 @@ def bin_xml_escape(arg: object) -> str: The idea is to escape visually for the user rather than for XML itself. """ - def repl(matchobj: Match[str]) -> str: + def repl(matchobj: re.Match[str]) -> str: i = ord(matchobj.group()) if i <= 0xFF: return f"#x{i:02X}" @@ -74,10 +71,10 @@ def merge_family(left, right) -> None: left.update(result) -families = {} -families["_base"] = {"testcase": ["classname", "name"]} -families["_base_legacy"] = {"testcase": ["file", "line", "url"]} - +families = { # pylint: disable=dict-init-mutate + "_base": {"testcase": ["classname", "name"]}, + "_base_legacy": {"testcase": ["file", "line", "url"]}, +} # xUnit 1.x inherits legacy attributes. families["xunit1"] = families["_base"].copy() merge_family(families["xunit1"], families["_base_legacy"]) @@ -637,7 +634,7 @@ def pytest_internalerror(self, excrepr: ExceptionRepr) -> None: reporter._add_simple("error", "internal error", str(excrepr)) def pytest_sessionstart(self) -> None: - self.suite_start_time = timing.time() + self.suite_start = timing.Instant() def pytest_sessionfinish(self) -> None: dirname = os.path.dirname(os.path.abspath(self.logfile)) @@ -645,8 +642,7 @@ def pytest_sessionfinish(self) -> None: os.makedirs(dirname, exist_ok=True) with open(self.logfile, "w", encoding="utf-8") as logfile: - suite_stop_time = timing.time() - suite_time_delta = suite_stop_time - self.suite_start_time + duration = self.suite_start.elapsed() numtests = ( self.stats["passed"] @@ -664,10 +660,8 @@ def pytest_sessionfinish(self) -> None: failures=str(self.stats["failure"]), skipped=str(self.stats["skipped"]), tests=str(numtests), - time=f"{suite_time_delta:.3f}", - timestamp=datetime.fromtimestamp(self.suite_start_time, timezone.utc) - .astimezone() - .isoformat(), + time=f"{duration.seconds:.3f}", + timestamp=self.suite_start.as_utc().astimezone().isoformat(), hostname=platform.node(), ) global_properties = self._get_global_properties_node() @@ -676,11 +670,15 @@ def pytest_sessionfinish(self) -> None: for node_reporter in self.node_reporters_ordered: suite_node.append(node_reporter.to_xml()) testsuites = ET.Element("testsuites") + testsuites.set("name", "pytest tests") testsuites.append(suite_node) logfile.write(ET.tostring(testsuites, encoding="unicode")) - def pytest_terminal_summary(self, terminalreporter: TerminalReporter) -> None: - terminalreporter.write_sep("-", f"generated xml file: {self.logfile}") + def pytest_terminal_summary( + self, terminalreporter: TerminalReporter, config: pytest.Config + ) -> None: + if config.get_verbosity() >= 0: + terminalreporter.write_sep("-", f"generated xml file: {self.logfile}") def add_global_property(self, name: str, value: object) -> None: __tracebackhide__ = True diff --git a/src/_pytest/legacypath.py b/src/_pytest/legacypath.py index 61476d68932..59e8ef6e742 100644 --- a/src/_pytest/legacypath.py +++ b/src/_pytest/legacypath.py @@ -304,16 +304,11 @@ def tmpdir_factory(request: FixtureRequest) -> TempdirFactory: @staticmethod @fixture def tmpdir(tmp_path: Path) -> LEGACY_PATH: - """Return a temporary directory path object which is unique to each test - function invocation, created as a sub directory of the base temporary - directory. - - By default, a new base temporary directory is created each test session, - and old bases are removed after 3 sessions, to aid in debugging. If - ``--basetemp`` is used then it is cleared each session. See - :ref:`temporary directory location and retention`. - - The returned object is a `legacy_path`_ object. + """Return a temporary directory (as `legacy_path`_ object) + which is unique to each test function invocation. + The temporary directory is created as a subdirectory + of the base temporary directory, with configurable retention, + as discussed in :ref:`temporary directory location and retention`. .. note:: These days, it is preferred to use ``tmp_path``. diff --git a/src/_pytest/logging.py b/src/_pytest/logging.py index 44af8ff2041..e4fed579d21 100644 --- a/src/_pytest/logging.py +++ b/src/_pytest/logging.py @@ -3,6 +3,9 @@ from __future__ import annotations +from collections.abc import Generator +from collections.abc import Mapping +from collections.abc import Set as AbstractSet from contextlib import contextmanager from contextlib import nullcontext from datetime import datetime @@ -16,14 +19,9 @@ from pathlib import Path import re from types import TracebackType -from typing import AbstractSet -from typing import Dict from typing import final -from typing import Generator from typing import Generic -from typing import List from typing import Literal -from typing import Mapping from typing import TYPE_CHECKING from typing import TypeVar @@ -53,7 +51,7 @@ DEFAULT_LOG_DATE_FORMAT = "%H:%M:%S" _ANSI_ESCAPE_SEQ = re.compile(r"\x1b\[[\d;]+m") caplog_handler_key = StashKey["LogCaptureHandler"]() -caplog_records_key = StashKey[Dict[str, List[logging.LogRecord]]]() +caplog_records_key = StashKey[dict[str, list[logging.LogRecord]]]() def _remove_ansi_escape_sequences(text: str) -> str: @@ -554,9 +552,7 @@ def set_level(self, level: int | str, logger: str | None = None) -> None: self._initial_disabled_logging_level = initial_disabled_logging_level @contextmanager - def at_level( - self, level: int | str, logger: str | None = None - ) -> Generator[None, None, None]: + def at_level(self, level: int | str, logger: str | None = None) -> Generator[None]: """Context manager that sets the level for capturing of logs. After the end of the 'with' statement the level is restored to its original value. @@ -580,7 +576,7 @@ def at_level( logging.disable(original_disable_level) @contextmanager - def filtering(self, filter_: logging.Filter) -> Generator[None, None, None]: + def filtering(self, filter_: logging.Filter) -> Generator[None]: """Context manager that temporarily adds the given filter to the caplog's :meth:`handler` for the 'with' statement block, and removes that filter at the end of the block. @@ -597,7 +593,7 @@ def filtering(self, filter_: logging.Filter) -> Generator[None, None, None]: @fixture -def caplog(request: FixtureRequest) -> Generator[LogCaptureFixture, None, None]: +def caplog(request: FixtureRequest) -> Generator[LogCaptureFixture]: """Access and control log capturing. Captured logs are available through the following properties/methods:: @@ -776,7 +772,7 @@ def _log_cli_enabled(self) -> bool: return True @hookimpl(wrapper=True, tryfirst=True) - def pytest_sessionstart(self) -> Generator[None, None, None]: + def pytest_sessionstart(self) -> Generator[None]: self.log_cli_handler.set_when("sessionstart") with catching_logs(self.log_cli_handler, level=self.log_cli_level): @@ -784,7 +780,7 @@ def pytest_sessionstart(self) -> Generator[None, None, None]: return (yield) @hookimpl(wrapper=True, tryfirst=True) - def pytest_collection(self) -> Generator[None, None, None]: + def pytest_collection(self) -> Generator[None]: self.log_cli_handler.set_when("collection") with catching_logs(self.log_cli_handler, level=self.log_cli_level): @@ -796,7 +792,7 @@ def pytest_runtestloop(self, session: Session) -> Generator[None, object, object if session.config.option.collectonly: return (yield) - if self._log_cli_enabled() and self._config.getoption("verbose") < 1: + if self._log_cli_enabled() and self._config.get_verbosity() < 1: # The verbose flag is needed to avoid messy test progress output. self._config.option.verbose = 1 @@ -813,15 +809,19 @@ def pytest_runtest_logstart(self) -> None: def pytest_runtest_logreport(self) -> None: self.log_cli_handler.set_when("logreport") - def _runtest_for(self, item: nodes.Item, when: str) -> Generator[None, None, None]: + @contextmanager + def _runtest_for(self, item: nodes.Item, when: str) -> Generator[None]: """Implement the internals of the pytest_runtest_xxx() hooks.""" - with catching_logs( - self.caplog_handler, - level=self.log_level, - ) as caplog_handler, catching_logs( - self.report_handler, - level=self.log_level, - ) as report_handler: + with ( + catching_logs( + self.caplog_handler, + level=self.log_level, + ) as caplog_handler, + catching_logs( + self.report_handler, + level=self.log_level, + ) as report_handler, + ): caplog_handler.reset() report_handler.reset() item.stash[caplog_records_key][when] = caplog_handler.records @@ -834,25 +834,28 @@ def _runtest_for(self, item: nodes.Item, when: str) -> Generator[None, None, Non item.add_report_section(when, "log", log) @hookimpl(wrapper=True) - def pytest_runtest_setup(self, item: nodes.Item) -> Generator[None, None, None]: + def pytest_runtest_setup(self, item: nodes.Item) -> Generator[None]: self.log_cli_handler.set_when("setup") empty: dict[str, list[logging.LogRecord]] = {} item.stash[caplog_records_key] = empty - yield from self._runtest_for(item, "setup") + with self._runtest_for(item, "setup"): + yield @hookimpl(wrapper=True) - def pytest_runtest_call(self, item: nodes.Item) -> Generator[None, None, None]: + def pytest_runtest_call(self, item: nodes.Item) -> Generator[None]: self.log_cli_handler.set_when("call") - yield from self._runtest_for(item, "call") + with self._runtest_for(item, "call"): + yield @hookimpl(wrapper=True) - def pytest_runtest_teardown(self, item: nodes.Item) -> Generator[None, None, None]: + def pytest_runtest_teardown(self, item: nodes.Item) -> Generator[None]: self.log_cli_handler.set_when("teardown") try: - yield from self._runtest_for(item, "teardown") + with self._runtest_for(item, "teardown"): + yield finally: del item.stash[caplog_records_key] del item.stash[caplog_handler_key] @@ -862,7 +865,7 @@ def pytest_runtest_logfinish(self) -> None: self.log_cli_handler.set_when("finish") @hookimpl(wrapper=True, tryfirst=True) - def pytest_sessionfinish(self) -> Generator[None, None, None]: + def pytest_sessionfinish(self) -> Generator[None]: self.log_cli_handler.set_when("sessionfinish") with catching_logs(self.log_cli_handler, level=self.log_cli_level): diff --git a/src/_pytest/main.py b/src/_pytest/main.py index 8ec26906003..9bc930df8e8 100644 --- a/src/_pytest/main.py +++ b/src/_pytest/main.py @@ -3,6 +3,11 @@ from __future__ import annotations import argparse +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Sequence +from collections.abc import Set as AbstractSet import dataclasses import fnmatch import functools @@ -11,15 +16,9 @@ import os from pathlib import Path import sys -from typing import AbstractSet -from typing import Callable -from typing import Dict from typing import final -from typing import Iterable -from typing import Iterator from typing import Literal from typing import overload -from typing import Sequence from typing import TYPE_CHECKING import warnings @@ -33,6 +32,7 @@ from _pytest.config import hookimpl from _pytest.config import PytestPluginManager from _pytest.config import UsageError +from _pytest.config.argparsing import OverrideIniAction from _pytest.config.argparsing import Parser from _pytest.config.compat import PathAwareHookProxy from _pytest.outcomes import exit @@ -40,6 +40,7 @@ from _pytest.pathlib import bestrelpath from _pytest.pathlib import fnmatch_ex from _pytest.pathlib import safe_exists +from _pytest.pathlib import samefile_nofollow from _pytest.pathlib import scandir from _pytest.reports import CollectReport from _pytest.reports import TestReport @@ -49,37 +50,14 @@ if TYPE_CHECKING: - from typing import Self + from typing_extensions import Self from _pytest.fixtures import FixtureManager def pytest_addoption(parser: Parser) -> None: - parser.addini( - "norecursedirs", - "Directory patterns to avoid for recursion", - type="args", - default=[ - "*.egg", - ".*", - "_darcs", - "build", - "CVS", - "dist", - "node_modules", - "venv", - "{arch}", - ], - ) - parser.addini( - "testpaths", - "Directories to search for tests when no files or directories are given on the " - "command line", - type="args", - default=[], - ) - group = parser.getgroup("general", "Running and selection options") - group._addoption( + group = parser.getgroup("general") + group._addoption( # private to use reserved lower-case short option "-x", "--exitfirst", action="store_const", @@ -87,21 +65,7 @@ def pytest_addoption(parser: Parser) -> None: const=1, help="Exit instantly on first error or failed test", ) - group = parser.getgroup("pytest-warnings") group.addoption( - "-W", - "--pythonwarnings", - action="append", - help="Set which warnings to report, see -W option of Python itself", - ) - parser.addini( - "filterwarnings", - type="linelist", - help="Each line specifies a pattern for " - "warnings.filterwarnings. " - "Processed after -W/--pythonwarnings.", - ) - group._addoption( "--maxfail", metavar="num", action="store", @@ -110,46 +74,64 @@ def pytest_addoption(parser: Parser) -> None: default=0, help="Exit after first num failures or errors", ) - group._addoption( + group.addoption( "--strict-config", - action="store_true", - help="Any warnings encountered while parsing the `pytest` section of the " - "configuration file raise errors", + action=OverrideIniAction, + ini_option="strict_config", + ini_value="true", + help="Enables the strict_config option", ) - group._addoption( + group.addoption( "--strict-markers", - action="store_true", - help="Markers not registered in the `markers` section of the configuration " - "file raise errors", + action=OverrideIniAction, + ini_option="strict_markers", + ini_value="true", + help="Enables the strict_markers option", ) - group._addoption( + group.addoption( "--strict", - action="store_true", - help="(Deprecated) alias to --strict-markers", + action=OverrideIniAction, + ini_option="strict", + ini_value="true", + help="Enables the strict option", ) - group._addoption( - "-c", - "--config-file", - metavar="FILE", - type=str, - dest="inifilename", - help="Load configuration from `FILE` instead of trying to locate one of the " - "implicit configuration files.", + parser.addini( + "strict_config", + "Any warnings encountered while parsing the `pytest` section of the " + "configuration file raise errors", + type="bool", + # None => fallback to `strict`. + default=None, ) - group._addoption( - "--continue-on-collection-errors", - action="store_true", + parser.addini( + "strict_markers", + "Markers not registered in the `markers` section of the configuration " + "file raise errors", + type="bool", + # None => fallback to `strict`. + default=None, + ) + parser.addini( + "strict", + "Enables all strictness options, currently: " + "strict_config, strict_markers, strict_xfail, strict_parametrization_ids", + type="bool", default=False, - dest="continue_on_collection_errors", - help="Force test execution even if collection errors occur", ) - group._addoption( - "--rootdir", - action="store", - dest="rootdir", - help="Define root directory for tests. Can be relative path: 'root_dir', './root_dir', " - "'root_dir/another_dir/'; absolute path: '/home/user/root_dir'; path with variables: " - "'$HOME/root_dir'.", + + group = parser.getgroup("pytest-warnings") + group.addoption( + "-W", + "--pythonwarnings", + action="append", + help="Set which warnings to report, see -W option of Python itself", + ) + parser.addini( + "filterwarnings", + type="linelist", + help="Each line specifies a pattern for " + "warnings.filterwarnings. " + "Processed after -W/--pythonwarnings.", ) group = parser.getgroup("collect", "collection") @@ -213,6 +195,13 @@ def pytest_addoption(parser: Parser) -> None: default=False, help="Don't ignore tests in a local virtualenv directory", ) + group.addoption( + "--continue-on-collection-errors", + action="store_true", + default=False, + dest="continue_on_collection_errors", + help="Force test execution even if collection errors occur", + ) group.addoption( "--import-mode", default="prepend", @@ -221,6 +210,35 @@ def pytest_addoption(parser: Parser) -> None: help="Prepend/append to sys.path when importing test modules and conftest " "files. Default: prepend.", ) + parser.addini( + "norecursedirs", + "Directory patterns to avoid for recursion", + type="args", + default=[ + "*.egg", + ".*", + "_darcs", + "build", + "CVS", + "dist", + "node_modules", + "venv", + "{arch}", + ], + ) + parser.addini( + "testpaths", + "Directories to search for tests when no files or directories are given on the " + "command line", + type="args", + default=[], + ) + parser.addini( + "collect_imported_tests", + "Whether to collect tests in imported modules outside `testpaths`", + type="bool", + default=True, + ) parser.addini( "consider_namespace_packages", type="bool", @@ -229,6 +247,23 @@ def pytest_addoption(parser: Parser) -> None: ) group = parser.getgroup("debugconfig", "test session debugging and configuration") + group._addoption( # private to use reserved lower-case short option + "-c", + "--config-file", + metavar="FILE", + type=str, + dest="inifilename", + help="Load configuration from `FILE` instead of trying to locate one of the " + "implicit configuration files.", + ) + group.addoption( + "--rootdir", + action="store", + dest="rootdir", + help="Define root directory for tests. Can be relative path: 'root_dir', './root_dir', " + "'root_dir/another_dir/'; absolute path: '/home/user/root_dir'; path with variables: " + "'$HOME/root_dir'.", + ) group.addoption( "--basetemp", dest="basetemp", @@ -350,8 +385,7 @@ def pytest_collection(session: Session) -> None: def pytest_runtestloop(session: Session) -> bool: if session.testsfailed and not session.config.option.continue_on_collection_errors: raise session.Interrupted( - "%d error%s during collection" - % (session.testsfailed, "s" if session.testsfailed != 1 else "") + f"{session.testsfailed} error{'s' if session.testsfailed != 1 else ''} during collection" ) if session.config.option.collectonly: @@ -370,9 +404,20 @@ def pytest_runtestloop(session: Session) -> bool: def _in_venv(path: Path) -> bool: """Attempt to detect if ``path`` is the root of a Virtual Environment by checking for the existence of the pyvenv.cfg file. - [https://peps.python.org/pep-0405/]""" + + [https://peps.python.org/pep-0405/] + + For regression protection we also check for conda environments that do not include pyenv.cfg yet -- + https://github.com/conda/conda/issues/13337 is the conda issue tracking adding pyenv.cfg. + + Checking for the `conda-meta/history` file per https://github.com/pytest-dev/pytest/issues/12652#issuecomment-2246336902. + + """ try: - return path.joinpath("pyvenv.cfg").is_file() + return ( + path.joinpath("pyvenv.cfg").is_file() + or path.joinpath("conda-meta", "history").is_file() + ) except OSError: return False @@ -465,7 +510,7 @@ class Failed(Exception): @dataclasses.dataclass -class _bestrelpath_cache(Dict[Path, str]): +class _bestrelpath_cache(dict[Path, str]): __slots__ = ("path",) path: Path @@ -575,13 +620,12 @@ def from_config(cls, config: Config) -> Session: return session def __repr__(self) -> str: - return "<%s %s exitstatus=%r testsfailed=%d testscollected=%d>" % ( - self.__class__.__name__, - self.name, - getattr(self, "exitstatus", ""), - self.testsfailed, - self.testscollected, - ) + return ( + f"<{self.__class__.__name__} {self.name} " + f"exitstatus=%r " + f"testsfailed={self.testsfailed} " + f"testscollected={self.testscollected}>" + ) % getattr(self, "exitstatus", "") @property def shouldstop(self) -> bool | str: @@ -644,7 +688,7 @@ def pytest_runtest_logreport(self, report: TestReport | CollectReport) -> None: self.testsfailed += 1 maxfail = self.config.getvalue("maxfail") if maxfail and self.testsfailed >= maxfail: - self.shouldfail = "stopping after %d failures" % (self.testsfailed) + self.shouldfail = f"stopping after {self.testsfailed} failures" pytest_collectreport = pytest_runtest_logreport @@ -759,16 +803,31 @@ def perform_collect( self._collection_cache = {} self.items = [] items: Sequence[nodes.Item | nodes.Collector] = self.items + consider_namespace_packages: bool = self.config.getini( + "consider_namespace_packages" + ) try: initialpaths: list[Path] = [] initialpaths_with_parents: list[Path] = [] - for arg in args: - collection_argument = resolve_collection_argument( + + collection_args = [ + resolve_collection_argument( self.config.invocation_params.dir, arg, + i, as_pypath=self.config.option.pyargs, + consider_namespace_packages=consider_namespace_packages, ) - self._initial_parts.append(collection_argument) + for i, arg in enumerate(args) + ] + + if not self.config.getoption("keepduplicates"): + # Normalize the collection arguments -- remove duplicates and overlaps. + self._initial_parts = normalize_collection_arguments(collection_args) + else: + self._initial_parts = collection_args + + for collection_argument in self._initial_parts: initialpaths.append(collection_argument.path) initialpaths_with_parents.append(collection_argument.path) initialpaths_with_parents.extend(collection_argument.path.parents) @@ -839,6 +898,7 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: argpath = collection_argument.path names = collection_argument.parts + parametrization = collection_argument.parametrization module_name = collection_argument.module_name # resolve_collection_argument() ensures this. @@ -916,23 +976,25 @@ def collect(self) -> Iterator[nodes.Item | nodes.Collector]: is_match = node.path == matchparts[0] if sys.platform == "win32" and not is_match: # In case the file paths do not match, fallback to samefile() to - # account for short-paths on Windows (#11895). - same_file = os.path.samefile(node.path, matchparts[0]) - # We don't want to match links to the current node, - # otherwise we would match the same file more than once (#12039). - is_match = same_file and ( - os.path.islink(node.path) - == os.path.islink(matchparts[0]) - ) + # account for short-paths on Windows (#11895). But use a version + # which doesn't resolve symlinks, otherwise we might match the + # same file more than once (#12039). + is_match = samefile_nofollow(node.path, matchparts[0]) # Name part e.g. `TestIt` in `/a/b/test_file.py::TestIt::test_it`. else: - # TODO: Remove parametrized workaround once collection structure contains - # parametrization. - is_match = ( - node.name == matchparts[0] - or node.name.split("[")[0] == matchparts[0] - ) + if len(matchparts) == 1: + # This the last part, one parametrization goes. + if parametrization is not None: + # A parametrized arg must match exactly. + is_match = node.name == matchparts[0] + parametrization + else: + # A non-parameterized arg matches all parametrizations (if any). + # TODO: Remove the hacky split once the collection structure + # contains parametrization. + is_match = node.name.split("[")[0] == matchparts[0] + else: + is_match = node.name == matchparts[0] if is_match: work.append((node, matchparts[1:])) any_matched_in_collector = True @@ -953,12 +1015,9 @@ def genitems(self, node: nodes.Item | nodes.Collector) -> Iterator[nodes.Item]: yield node else: assert isinstance(node, nodes.Collector) - keepduplicates = self.config.getoption("keepduplicates") # For backward compat, dedup only applies to files. - handle_dupes = not (keepduplicates and isinstance(node, nodes.File)) + handle_dupes = not isinstance(node, nodes.File) rep, duplicate = self._collect_one_node(node, handle_dupes) - if duplicate and not keepduplicates: - return if rep.passed: for subnode in rep.result: yield from self.genitems(subnode) @@ -966,7 +1025,9 @@ def genitems(self, node: nodes.Item | nodes.Collector) -> Iterator[nodes.Item]: node.ihook.pytest_collectreport(report=rep) -def search_pypath(module_name: str) -> str | None: +def search_pypath( + module_name: str, *, consider_namespace_packages: bool = False +) -> str | None: """Search sys.path for the given a dotted module name, and return its file system path if found.""" try: @@ -976,13 +1037,29 @@ def search_pypath(module_name: str) -> str | None: # ValueError: not a module name except (AttributeError, ImportError, ValueError): return None - if spec is None or spec.origin is None or spec.origin == "namespace": + + if spec is None: return None - elif spec.submodule_search_locations: - return os.path.dirname(spec.origin) - else: + + if ( + spec.submodule_search_locations is None + or len(spec.submodule_search_locations) == 0 + ): + # Must be a simple module. return spec.origin + if consider_namespace_packages: + # If submodule_search_locations is set, it's a package (regular or namespace). + # Typically there is a single entry, but documentation claims it can be empty too + # (e.g. if the package has no physical location). + return spec.submodule_search_locations[0] + + if spec.origin is None: + # This is only the case for namespace packages + return None + + return os.path.dirname(spec.origin) + @dataclasses.dataclass(frozen=True) class CollectionArgument: @@ -990,11 +1067,18 @@ class CollectionArgument: path: Path parts: Sequence[str] + parametrization: str | None module_name: str | None + original_index: int def resolve_collection_argument( - invocation_path: Path, arg: str, *, as_pypath: bool = False + invocation_path: Path, + arg: str, + arg_index: int, + *, + as_pypath: bool = False, + consider_namespace_packages: bool = False, ) -> CollectionArgument: """Parse path arguments optionally containing selection parts and return (fspath, names). @@ -1014,7 +1098,7 @@ def resolve_collection_argument( When as_pypath is True, expects that the command-line argument actually contains module paths instead of file-system paths: - "pkg.tests.test_foo::TestClass::test_foo" + "pkg.tests.test_foo::TestClass::test_foo[a,b]" In which case we search sys.path for a matching module, and then return the *path* to the found module, which may look like this: @@ -1022,19 +1106,23 @@ def resolve_collection_argument( CollectionArgument( path=Path("/home/u/myvenv/lib/site-packages/pkg/tests/test_foo.py"), parts=["TestClass", "test_foo"], + parametrization="[a,b]", module_name="pkg.tests.test_foo", ) If the path doesn't exist, raise UsageError. If the path is a directory and selection parts are present, raise UsageError. """ - base, squacket, rest = str(arg).partition("[") + base, squacket, rest = arg.partition("[") strpath, *parts = base.split("::") - if parts: - parts[-1] = f"{parts[-1]}{squacket}{rest}" + if squacket and not parts: + raise UsageError(f"path cannot contain [] parametrization: {arg}") + parametrization = f"{squacket}{rest}" if squacket else None module_name = None if as_pypath: - pyarg_strpath = search_pypath(strpath) + pyarg_strpath = search_pypath( + strpath, consider_namespace_packages=consider_namespace_packages + ) if pyarg_strpath is not None: module_name = strpath strpath = pyarg_strpath @@ -1057,5 +1145,59 @@ def resolve_collection_argument( return CollectionArgument( path=fspath, parts=parts, + parametrization=parametrization, module_name=module_name, + original_index=arg_index, + ) + + +def is_collection_argument_subsumed_by( + arg: CollectionArgument, by: CollectionArgument +) -> bool: + """Check if `arg` is subsumed (contained) by `by`.""" + # First check path subsumption. + if by.path != arg.path: + # `by` subsumes `arg` if `by` is a parent directory of `arg` and has no + # parts (collects everything in that directory). + if not by.parts: + return arg.path.is_relative_to(by.path) + return False + # Paths are equal, check parts. + # For example: ("TestClass",) is a prefix of ("TestClass", "test_method"). + if len(by.parts) > len(arg.parts) or arg.parts[: len(by.parts)] != by.parts: + return False + # Paths and parts are equal, check parametrization. + # A `by` without parametrization (None) matches everything, e.g. + # `pytest x.py::test_it` matches `x.py::test_it[0]`. Otherwise must be + # exactly equal. + if by.parametrization is not None and by.parametrization != arg.parametrization: + return False + return True + + +def normalize_collection_arguments( + collection_args: Sequence[CollectionArgument], +) -> list[CollectionArgument]: + """Normalize collection arguments to eliminate overlapping paths and parts. + + Detects when collection arguments overlap in either paths or parts and only + keeps the shorter prefix, or the earliest argument if duplicate, preserving + order. The result is prefix-free. + """ + # A quadratic algorithm is not acceptable since large inputs are possible. + # So this uses an O(n*log(n)) algorithm which takes advantage of the + # property that after sorting, a collection argument will immediately + # precede collection arguments it subsumes. An O(n) algorithm is not worth + # it. + collection_args_sorted = sorted( + collection_args, + key=lambda arg: (arg.path, arg.parts, arg.parametrization or ""), ) + normalized: list[CollectionArgument] = [] + last_kept = None + for arg in collection_args_sorted: + if last_kept is None or not is_collection_argument_subsumed_by(arg, last_kept): + normalized.append(arg) + last_kept = arg + normalized.sort(key=lambda arg: arg.original_index) + return normalized diff --git a/src/_pytest/mark/__init__.py b/src/_pytest/mark/__init__.py index a4f942c5ae3..841d7811fdd 100644 --- a/src/_pytest/mark/__init__.py +++ b/src/_pytest/mark/__init__.py @@ -3,17 +3,17 @@ from __future__ import annotations import collections +from collections.abc import Collection +from collections.abc import Iterable +from collections.abc import Set as AbstractSet import dataclasses -from typing import AbstractSet -from typing import Collection -from typing import Iterable -from typing import Optional from typing import TYPE_CHECKING from .expression import Expression -from .expression import ParseError +from .structures import _HiddenParam from .structures import EMPTY_PARAMETERSET_OPTION from .structures import get_empty_parameterset_mark +from .structures import HIDDEN_PARAM from .structures import Mark from .structures import MARK_GEN from .structures import MarkDecorator @@ -33,6 +33,7 @@ __all__ = [ + "HIDDEN_PARAM", "MARK_GEN", "Mark", "MarkDecorator", @@ -42,13 +43,13 @@ ] -old_mark_config_key = StashKey[Optional[Config]]() +old_mark_config_key = StashKey[Config | None]() def param( *values: object, marks: MarkDecorator | Collection[MarkDecorator | Mark] = (), - id: str | None = None, + id: str | _HiddenParam | None = None, ) -> ParameterSet: """Specify a parameter in `pytest.mark.parametrize`_ calls or :ref:`parametrized fixtures `. @@ -66,15 +67,27 @@ def test_eval(test_input, expected): assert eval(test_input) == expected :param values: Variable args of the values of the parameter set, in order. - :param marks: A single mark or a list of marks to be applied to this parameter set. - :param id: The id to attribute to this parameter set. + + :param marks: + A single mark or a list of marks to be applied to this parameter set. + + :ref:`pytest.mark.usefixtures ` cannot be added via this parameter. + + :type id: str | Literal[pytest.HIDDEN_PARAM] | None + :param id: + The id to attribute to this parameter set. + + .. versionadded:: 8.4 + :ref:`hidden-param` means to hide the parameter set + from the test name. Can only be used at most 1 time, as + test names need to be unique. """ return ParameterSet.param(*values, marks=marks, id=id) def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("general") - group._addoption( + group._addoption( # private to use reserved lower-case short option "-k", action="store", dest="keyword", @@ -94,7 +107,7 @@ def pytest_addoption(parser: Parser) -> None: "The matching is case-insensitive.", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-m", action="store", dest="markexpr", @@ -188,12 +201,7 @@ def __call__(self, subname: str, /, **kwargs: str | int | bool | None) -> bool: if kwargs: raise UsageError("Keyword expressions do not support call parameters.") subname = subname.lower() - names = (name.lower() for name in self._names) - - for name in names: - if subname in name: - return True - return False + return any(subname in name.lower() for name in self._names) def deselect_by_keyword(items: list[Item], config: Config) -> None: @@ -238,10 +246,9 @@ def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool: if not (matches := self.own_mark_name_mapping.get(name, [])): return False - for mark in matches: + for mark in matches: # pylint: disable=consider-using-any-or-all if all(mark.kwargs.get(k, NOT_SET) == v for k, v in kwargs.items()): return True - return False @@ -266,8 +273,10 @@ def deselect_by_mark(items: list[Item], config: Config) -> None: def _parse_expression(expr: str, exc_message: str) -> Expression: try: return Expression.compile(expr) - except ParseError as e: - raise UsageError(f"{exc_message}: {expr}: {e}") from None + except SyntaxError as e: + raise UsageError( + f"{exc_message}: {e.text}: at column {e.offset}: {e.msg}" + ) from None def pytest_collection_modifyitems(items: list[Item], config: Config) -> None: diff --git a/src/_pytest/mark/expression.py b/src/_pytest/mark/expression.py index 89cc0e94d3b..3bdbd03c2b5 100644 --- a/src/_pytest/mark/expression.py +++ b/src/_pytest/mark/expression.py @@ -16,33 +16,38 @@ - Empty expression evaluates to False. - ident evaluates to True or False according to a provided matcher function. -- or/and/not evaluate according to the usual boolean semantics. - ident with parentheses and keyword arguments evaluates to True or False according to a provided matcher function. +- or/and/not evaluate according to the usual boolean semantics. """ from __future__ import annotations import ast +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence import dataclasses import enum import keyword import re import types -from typing import Iterator +from typing import Final +from typing import final from typing import Literal -from typing import Mapping from typing import NoReturn from typing import overload from typing import Protocol -from typing import Sequence __all__ = [ "Expression", - "ParseError", + "ExpressionMatcher", ] +FILE_NAME: Final = "" + + class TokenType(enum.Enum): LPAREN = "left parenthesis" RPAREN = "right parenthesis" @@ -58,31 +63,17 @@ class TokenType(enum.Enum): @dataclasses.dataclass(frozen=True) class Token: - __slots__ = ("type", "value", "pos") + __slots__ = ("pos", "type", "value") type: TokenType value: str pos: int -class ParseError(Exception): - """The expression contains invalid syntax. - - :param column: The column in the line where the error occurred (1-based). - :param message: A description of the error. - """ - - def __init__(self, column: int, message: str) -> None: - self.column = column - self.message = message - - def __str__(self) -> str: - return f"at column {self.column}: {self.message}" - - class Scanner: - __slots__ = ("tokens", "current") + __slots__ = ("current", "input", "tokens") def __init__(self, input: str) -> None: + self.input = input self.tokens = self.lex(input) self.current = next(self.tokens) @@ -106,15 +97,15 @@ def lex(self, input: str) -> Iterator[Token]: elif (quote_char := input[pos]) in ("'", '"'): end_quote_pos = input.find(quote_char, pos + 1) if end_quote_pos == -1: - raise ParseError( - pos + 1, + raise SyntaxError( f'closing quote "{quote_char}" is missing', + (FILE_NAME, 1, pos + 1, input), ) value = input[pos : end_quote_pos + 1] if (backslash_pos := input.find("\\")) != -1: - raise ParseError( - backslash_pos + 1, + raise SyntaxError( r'escaping with "\" not supported in marker expression', + (FILE_NAME, 1, backslash_pos + 1, input), ) yield Token(TokenType.STRING, value, pos) pos += len(value) @@ -132,9 +123,9 @@ def lex(self, input: str) -> Iterator[Token]: yield Token(TokenType.IDENT, value, pos) pos += len(value) else: - raise ParseError( - pos + 1, + raise SyntaxError( f'unexpected character "{input[pos]}"', + (FILE_NAME, 1, pos + 1, input), ) yield Token(TokenType.EOF, "", pos) @@ -157,12 +148,12 @@ def accept(self, type: TokenType, *, reject: bool = False) -> Token | None: return None def reject(self, expected: Sequence[TokenType]) -> NoReturn: - raise ParseError( - self.current.pos + 1, + raise SyntaxError( "expected {}; got {}".format( " OR ".join(type.value for type in expected), self.current.type.value, ), + (FILE_NAME, 1, self.current.pos + 1, self.input), ) @@ -223,14 +214,14 @@ def not_expr(s: Scanner) -> ast.expr: def single_kwarg(s: Scanner) -> ast.keyword: keyword_name = s.accept(TokenType.IDENT, reject=True) if not keyword_name.value.isidentifier(): - raise ParseError( - keyword_name.pos + 1, + raise SyntaxError( f"not a valid python identifier {keyword_name.value}", + (FILE_NAME, 1, keyword_name.pos + 1, s.input), ) if keyword.iskeyword(keyword_name.value): - raise ParseError( - keyword_name.pos + 1, + raise SyntaxError( f"unexpected reserved python keyword `{keyword_name.value}`", + (FILE_NAME, 1, keyword_name.pos + 1, s.input), ) s.accept(TokenType.EQUAL, reject=True) @@ -238,18 +229,16 @@ def single_kwarg(s: Scanner) -> ast.keyword: value: str | int | bool | None = value_token.value[1:-1] # strip quotes else: value_token = s.accept(TokenType.IDENT, reject=True) - if ( - (number := value_token.value).isdigit() - or number.startswith("-") - and number[1:].isdigit() + if (number := value_token.value).isdigit() or ( + number.startswith("-") and number[1:].isdigit() ): value = int(number) elif value_token.value in BUILTIN_MATCHERS: value = BUILTIN_MATCHERS[value_token.value] else: - raise ParseError( - value_token.pos + 1, + raise SyntaxError( f'unexpected character/s "{value_token.value}"', + (FILE_NAME, 1, value_token.pos + 1, s.input), ) ret = ast.keyword(keyword_name.value, ast.Constant(value)) @@ -263,13 +252,36 @@ def all_kwargs(s: Scanner) -> list[ast.keyword]: return ret -class MatcherCall(Protocol): +class ExpressionMatcher(Protocol): + """A callable which, given an identifier and optional kwargs, should return + whether it matches in an :class:`Expression` evaluation. + + Should be prepared to handle arbitrary strings as input. + + If no kwargs are provided, the expression of the form `foo`. + If kwargs are provided, the expression is of the form `foo(1, b=True, "s")`. + + If the expression is not supported (e.g. don't want to accept the kwargs + syntax variant), should raise :class:`~pytest.UsageError`. + + Example:: + + def matcher(name: str, /, **kwargs: str | int | bool | None) -> bool: + # Match `cat`. + if name == "cat" and not kwargs: + return True + # Match `dog(barks=True)`. + if name == "dog" and kwargs == {"barks": False}: + return True + return False + """ + def __call__(self, name: str, /, **kwargs: str | int | bool | None) -> bool: ... @dataclasses.dataclass class MatcherNameAdapter: - matcher: MatcherCall + matcher: ExpressionMatcher name: str def __bool__(self) -> bool: @@ -282,7 +294,7 @@ def __call__(self, **kwargs: str | int | bool | None) -> bool: class MatcherAdapter(Mapping[str, MatcherNameAdapter]): """Adapts a matcher function to a locals mapping as required by eval().""" - def __init__(self, matcher: MatcherCall) -> None: + def __init__(self, matcher: ExpressionMatcher) -> None: self.matcher = matcher def __getitem__(self, key: str) -> MatcherNameAdapter: @@ -295,39 +307,47 @@ def __len__(self) -> int: raise NotImplementedError() +@final class Expression: """A compiled match expression as used by -k and -m. The expression can be evaluated against different matchers. """ - __slots__ = ("code",) + __slots__ = ("_code", "input") - def __init__(self, code: types.CodeType) -> None: - self.code = code + def __init__(self, input: str, code: types.CodeType) -> None: + #: The original input line, as a string. + self.input: Final = input + self._code: Final = code @classmethod - def compile(self, input: str) -> Expression: + def compile(cls, input: str) -> Expression: """Compile a match expression. :param input: The input expression - one line. + + :raises SyntaxError: If the expression is malformed. """ astexpr = expression(Scanner(input)) - code: types.CodeType = compile( + code = compile( astexpr, filename="", mode="eval", ) - return Expression(code) + return Expression(input, code) - def evaluate(self, matcher: MatcherCall) -> bool: + def evaluate(self, matcher: ExpressionMatcher) -> bool: """Evaluate the match expression. :param matcher: - Given an identifier, should return whether it matches or not. - Should be prepared to handle arbitrary strings as input. + A callback which determines whether an identifier matches or not. + See the :class:`ExpressionMatcher` protocol for details and example. :returns: Whether the expression matches or not. + + :raises UsageError: + If the matcher doesn't support the expression. Cannot happen if the + matcher supports all expressions. """ - ret: bool = bool(eval(self.code, {"__builtins__": {}}, MatcherAdapter(matcher))) - return ret + return bool(eval(self._code, {"__builtins__": {}}, MatcherAdapter(matcher))) diff --git a/src/_pytest/mark/structures.py b/src/_pytest/mark/structures.py index 92ade55f7c0..97842fc5704 100644 --- a/src/_pytest/mark/structures.py +++ b/src/_pytest/mark/structures.py @@ -2,32 +2,32 @@ from __future__ import annotations import collections.abc +from collections.abc import Callable +from collections.abc import Collection +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import MutableMapping +from collections.abc import Sequence import dataclasses +import enum import inspect from typing import Any -from typing import Callable -from typing import Collection from typing import final -from typing import Iterable -from typing import Iterator -from typing import Mapping -from typing import MutableMapping from typing import NamedTuple from typing import overload -from typing import Sequence from typing import TYPE_CHECKING from typing import TypeVar -from typing import Union import warnings from .._code import getfslineno -from ..compat import ascii_escaped from ..compat import NOTSET from ..compat import NotSetType from _pytest.config import Config from _pytest.deprecated import check_ispytest from _pytest.deprecated import MARKED_FIXTURE from _pytest.outcomes import fail +from _pytest.raises import AbstractRaises from _pytest.scope import _ScopeName from _pytest.warning_types import PytestUnknownMarkWarning @@ -39,6 +39,16 @@ EMPTY_PARAMETERSET_OPTION = "empty_parameter_set_mark" +# Singleton type for HIDDEN_PARAM, as described in: +# https://www.python.org/dev/peps/pep-0484/#support-for-singleton-types-in-unions +class _HiddenParam(enum.Enum): + token = 0 + + +#: Can be used as a parameter set id to hide it from the test name. +HIDDEN_PARAM = _HiddenParam.token + + def istestfunc(func) -> bool: return callable(func) and getattr(func, "__name__", "") != "" @@ -48,24 +58,18 @@ def get_empty_parameterset_mark( ) -> MarkDecorator: from ..nodes import Collector - fs, lineno = getfslineno(func) - reason = "got empty parameter set %r, function %s at %s:%d" % ( - argnames, - func.__name__, - fs, - lineno, - ) + argslisting = ", ".join(argnames) + _fs, lineno = getfslineno(func) + reason = f"got empty parameter set for ({argslisting})" requested_mark = config.getini(EMPTY_PARAMETERSET_OPTION) if requested_mark in ("", None, "skip"): mark = MARK_GEN.skip(reason=reason) elif requested_mark == "xfail": mark = MARK_GEN.xfail(reason=reason, run=False) elif requested_mark == "fail_at_collect": - f_name = func.__name__ - _, lineno = getfslineno(func) raise Collector.CollectError( - "Empty parameter set in '%s' at line %d" % (f_name, lineno + 1) + f"Empty parameter set in '{func.__name__}' at line {lineno + 1}" ) else: raise LookupError(requested_mark) @@ -73,26 +77,60 @@ def get_empty_parameterset_mark( class ParameterSet(NamedTuple): + """A set of values for a set of parameters along with associated marks and + an optional ID for the set. + + Examples:: + + pytest.param(1, 2, 3) + # ParameterSet(values=(1, 2, 3), marks=(), id=None) + + pytest.param("hello", id="greeting") + # ParameterSet(values=("hello",), marks=(), id="greeting") + + # Parameter set with marks + pytest.param(42, marks=pytest.mark.xfail) + # ParameterSet(values=(42,), marks=(MarkDecorator(...),), id=None) + + # From parametrize mark (parameter names + list of parameter sets) + pytest.mark.parametrize( + ("a", "b", "expected"), + [ + (1, 2, 3), + pytest.param(40, 2, 42, id="everything"), + ], + ) + # ParameterSet(values=(1, 2, 3), marks=(), id=None) + # ParameterSet(values=(40, 2, 42), marks=(), id="everything") + """ + values: Sequence[object | NotSetType] marks: Collection[MarkDecorator | Mark] - id: str | None + id: str | _HiddenParam | None @classmethod def param( cls, *values: object, marks: MarkDecorator | Collection[MarkDecorator | Mark] = (), - id: str | None = None, + id: str | _HiddenParam | None = None, ) -> ParameterSet: if isinstance(marks, MarkDecorator): marks = (marks,) else: assert isinstance(marks, collections.abc.Collection) + if any(i.name == "usefixtures" for i in marks): + raise ValueError( + "pytest.param cannot add pytest.mark.usefixtures; see " + "https://docs.pytest.org/en/stable/reference/reference.html#pytest-param" + ) if id is not None: - if not isinstance(id, str): - raise TypeError(f"Expected id to be a string, got {type(id)}: {id!r}") - id = ascii_escaped(id) + if not isinstance(id, str) and id is not HIDDEN_PARAM: + raise TypeError( + "Expected id to be a string or a `pytest.HIDDEN_PARAM` sentinel, " + f"got {type(id)}: {id!r}", + ) return cls(values, marks, id) @classmethod @@ -184,7 +222,9 @@ def _for_parametrize( # parameter set with NOTSET values, with the "empty parameter set" mark applied to it. mark = get_empty_parameterset_mark(config, argnames, func) parameters.append( - ParameterSet(values=(NOTSET,) * len(argnames), marks=[mark], id=None) + ParameterSet( + values=(NOTSET,) * len(argnames), marks=[mark], id="NOTSET" + ) ) return argnames, parameters @@ -261,7 +301,7 @@ def combined_with(self, other: Mark) -> Mark: # A generic parameter designating an object to which a Mark may # be applied -- a test function (callable) or class. # Note: a lambda is not allowed, but this can't be represented. -Markable = TypeVar("Markable", bound=Union[Callable[..., object], type]) +Markable = TypeVar("Markable", bound=Callable[..., object] | type) @dataclasses.dataclass @@ -352,8 +392,13 @@ def __call__(self, *args: object, **kwargs: object): if args and not kwargs: func = args[0] is_class = inspect.isclass(func) - if len(args) == 1 and (istestfunc(func) or is_class): - store_mark(func, self.mark, stacklevel=3) + # For staticmethods/classmethods, the marks are eventually fetched from the + # function object, not the descriptor, so unwrap. + unwrapped_func = func + if isinstance(func, staticmethod | classmethod): + unwrapped_func = func.__func__ + if len(args) == 1 and (istestfunc(unwrapped_func) or is_class): + store_mark(unwrapped_func, self.mark, stacklevel=3) return func return self.with_args(*args, **kwargs) @@ -451,11 +496,14 @@ def __call__(self, arg: Markable) -> Markable: ... @overload def __call__( self, - condition: str | bool = False, + condition: str | bool = True, *conditions: str | bool, reason: str = ..., run: bool = ..., - raises: None | type[BaseException] | tuple[type[BaseException], ...] = ..., + raises: None + | type[BaseException] + | tuple[type[BaseException], ...] + | AbstractRaises[BaseException] = ..., strict: bool = ..., ) -> MarkDecorator: ... @@ -532,17 +580,20 @@ def __getattr__(self, name: str) -> MarkDecorator: # If the name is not in the set of known marks after updating, # then it really is time to issue a warning or an error. if name not in self._markers: - if self._config.option.strict_markers or self._config.option.strict: - fail( - f"{name!r} not found in `markers` configuration option", - pytrace=False, - ) - # Raise a specific error for common misspellings of "parametrize". if name in ["parameterize", "parametrise", "parameterise"]: __tracebackhide__ = True fail(f"Unknown '{name}' mark, did you mean 'parametrize'?") + strict_markers = self._config.getini("strict_markers") + if strict_markers is None: + strict_markers = self._config.getini("strict") + if strict_markers: + fail( + f"{name!r} not found in `markers` configuration option", + pytrace=False, + ) + warnings.warn( f"Unknown pytest.mark.{name} - is this a typo? You can register " "custom marks to avoid this warning - for details, see " @@ -559,7 +610,7 @@ def __getattr__(self, name: str) -> MarkDecorator: @final class NodeKeywords(MutableMapping[str, Any]): - __slots__ = ("node", "parent", "_markers") + __slots__ = ("_markers", "node", "parent") def __init__(self, node: Node) -> None: self.node = node @@ -581,10 +632,8 @@ def __setitem__(self, key: str, value: Any) -> None: # below and use the collections.abc fallback, but that would be slow. def __contains__(self, key: object) -> bool: - return ( - key in self._markers - or self.parent is not None - and key in self.parent.keywords + return key in self._markers or ( + self.parent is not None and key in self.parent.keywords ) def update( # type: ignore[override] diff --git a/src/_pytest/monkeypatch.py b/src/_pytest/monkeypatch.py index 75b019a3be6..07cc3fc4b0f 100644 --- a/src/_pytest/monkeypatch.py +++ b/src/_pytest/monkeypatch.py @@ -3,19 +3,21 @@ from __future__ import annotations +from collections.abc import Generator +from collections.abc import Mapping +from collections.abc import MutableMapping from contextlib import contextmanager import os +from pathlib import Path import re import sys from typing import Any from typing import final -from typing import Generator -from typing import Mapping -from typing import MutableMapping from typing import overload from typing import TypeVar import warnings +from _pytest.deprecated import MONKEYPATCH_LEGACY_NAMESPACE_PACKAGES from _pytest.fixtures import fixture from _pytest.warning_types import PytestWarning @@ -28,7 +30,7 @@ @fixture -def monkeypatch() -> Generator[MonkeyPatch, None, None]: +def monkeypatch() -> Generator[MonkeyPatch]: """A convenient fixture for monkey-patching. The fixture provides these methods to modify objects, dictionaries, or @@ -135,7 +137,7 @@ def __init__(self) -> None: @classmethod @contextmanager - def context(cls) -> Generator[MonkeyPatch, None, None]: + def context(cls) -> Generator[MonkeyPatch]: """Context manager that returns a new :class:`MonkeyPatch` object which undoes any patching done inside the ``with`` block upon exit. @@ -346,8 +348,26 @@ def syspath_prepend(self, path) -> None: # https://github.com/pypa/setuptools/blob/d8b901bc/docs/pkg_resources.txt#L162-L171 # this is only needed when pkg_resources was already loaded by the namespace package if "pkg_resources" in sys.modules: + import pkg_resources from pkg_resources import fixup_namespace_packages + # Only issue deprecation warning if this call would actually have an + # effect for this specific path. + if ( + hasattr(pkg_resources, "_namespace_packages") + and pkg_resources._namespace_packages + ): + path_obj = Path(str(path)) + for ns_pkg in pkg_resources._namespace_packages: + if ns_pkg is None: + continue + ns_pkg_path = path_obj / ns_pkg.replace(".", os.sep) + if ns_pkg_path.is_dir(): + warnings.warn( + MONKEYPATCH_LEGACY_NAMESPACE_PACKAGES, stacklevel=2 + ) + break + fixup_namespace_packages(str(path)) # A call to syspathinsert() usually means that the caller wants to diff --git a/src/_pytest/nodes.py b/src/_pytest/nodes.py index bbde2664b90..6690f6ab1f8 100644 --- a/src/_pytest/nodes.py +++ b/src/_pytest/nodes.py @@ -2,17 +2,17 @@ from __future__ import annotations import abc +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import MutableMapping from functools import cached_property -from inspect import signature +from functools import lru_cache import os import pathlib from pathlib import Path from typing import Any -from typing import Callable from typing import cast -from typing import Iterable -from typing import Iterator -from typing import MutableMapping from typing import NoReturn from typing import overload from typing import TYPE_CHECKING @@ -28,6 +28,7 @@ from _pytest._code.code import Traceback from _pytest._code.code import TracebackStyle from _pytest.compat import LEGACY_PATH +from _pytest.compat import signature from _pytest.config import Config from _pytest.config import ConftestImportFailure from _pytest.config.compat import _check_path @@ -37,13 +38,12 @@ from _pytest.mark.structures import NodeKeywords from _pytest.outcomes import fail from _pytest.pathlib import absolutepath -from _pytest.pathlib import commonpath from _pytest.stash import Stash from _pytest.warning_types import PytestWarning if TYPE_CHECKING: - from typing import Self + from typing_extensions import Self # Imported here due to circular import. from _pytest.main import Session @@ -143,14 +143,14 @@ class Node(abc.ABC, metaclass=NodeMeta): # Use __slots__ to make attribute access faster. # Note that __dict__ is still available. __slots__ = ( + "__dict__", + "_nodeid", + "_store", + "config", "name", "parent", - "config", - "session", "path", - "_nodeid", - "_store", - "__dict__", + "session", ) def __init__( @@ -435,12 +435,12 @@ def _repr_failure_py( else: style = "long" - if self.config.getoption("verbose", 0) > 1: + if self.config.get_verbosity() > 1: truncate_locals = False else: truncate_locals = True - truncate_args = False if self.config.getoption("verbose", 0) > 2 else True + truncate_args = False if self.config.get_verbosity() > 2 else True # excinfo.getrepr() formats paths relative to the CWD if `abspath` is False. # It is possible for a fixture/test to change the CWD while this code runs, which @@ -543,11 +543,17 @@ def _traceback_filter(self, excinfo: ExceptionInfo[BaseException]) -> Traceback: return excinfo.traceback -def _check_initialpaths_for_relpath(session: Session, path: Path) -> str | None: - for initial_path in session._initialpaths: - if commonpath(path, initial_path) == initial_path: - rel = str(path.relative_to(initial_path)) - return "" if rel == "." else rel +@lru_cache(maxsize=1000) +def _check_initialpaths_for_relpath( + initial_paths: frozenset[Path], path: Path +) -> str | None: + if path in initial_paths: + return "" + + for parent in path.parents: + if parent in initial_paths: + return str(path.relative_to(parent)) + return None @@ -594,7 +600,7 @@ def __init__( try: nodeid = str(self.path.relative_to(session.config.rootpath)) except ValueError: - nodeid = _check_initialpaths_for_relpath(session, path) + nodeid = _check_initialpaths_for_relpath(session._initialpaths, path) if nodeid and os.sep != SEP: nodeid = nodeid.replace(os.sep, SEP) diff --git a/src/_pytest/outcomes.py b/src/_pytest/outcomes.py index 5b20803e586..766be95c0f7 100644 --- a/src/_pytest/outcomes.py +++ b/src/_pytest/outcomes.py @@ -5,12 +5,8 @@ import sys from typing import Any -from typing import Callable -from typing import cast +from typing import ClassVar from typing import NoReturn -from typing import Protocol -from typing import Type -from typing import TypeVar from .warning_types import PytestDeprecationWarning @@ -78,35 +74,11 @@ def __init__( super().__init__(msg) -# Elaborate hack to work around https://github.com/python/mypy/issues/2087. -# Ideally would just be `exit.Exception = Exit` etc. - -_F = TypeVar("_F", bound=Callable[..., object]) -_ET = TypeVar("_ET", bound=Type[BaseException]) - - -class _WithException(Protocol[_F, _ET]): - Exception: _ET - __call__: _F - - -def _with_exception(exception_type: _ET) -> Callable[[_F], _WithException[_F, _ET]]: - def decorate(func: _F) -> _WithException[_F, _ET]: - func_with_exception = cast(_WithException[_F, _ET], func) - func_with_exception.Exception = exception_type - return func_with_exception - - return decorate - - -# Exposed helper methods. +class XFailed(Failed): + """Raised from an explicit call to pytest.xfail().""" -@_with_exception(Exit) -def exit( - reason: str = "", - returncode: int | None = None, -) -> NoReturn: +class _Exit: """Exit testing process. :param reason: @@ -114,21 +86,24 @@ def exit( only because `msg` is deprecated. :param returncode: - Return code to be used when exiting pytest. None means the same as ``0`` (no error), same as :func:`sys.exit`. + Return code to be used when exiting pytest. None means the same as ``0`` (no error), + same as :func:`sys.exit`. :raises pytest.exit.Exception: The exception that is raised. """ - __tracebackhide__ = True - raise Exit(reason, returncode) + Exception: ClassVar[type[Exit]] = Exit -@_with_exception(Skipped) -def skip( - reason: str = "", - *, - allow_module_level: bool = False, -) -> NoReturn: + def __call__(self, reason: str = "", returncode: int | None = None) -> NoReturn: + __tracebackhide__ = True + raise Exit(msg=reason, returncode=returncode) + + +exit: _Exit = _Exit() + + +class _Skip: """Skip an executing test with the given message. This function should be called only during testing (setup, call or teardown) or @@ -156,12 +131,18 @@ def skip( Similarly, use the ``# doctest: +SKIP`` directive (see :py:data:`doctest.SKIP`) to skip a doctest statically. """ - __tracebackhide__ = True - raise Skipped(msg=reason, allow_module_level=allow_module_level) + Exception: ClassVar[type[Skipped]] = Skipped + + def __call__(self, reason: str = "", allow_module_level: bool = False) -> NoReturn: + __tracebackhide__ = True + raise Skipped(msg=reason, allow_module_level=allow_module_level) + + +skip: _Skip = _Skip() -@_with_exception(Failed) -def fail(reason: str = "", pytrace: bool = True) -> NoReturn: + +class _Fail: """Explicitly fail an executing test with the given message. :param reason: @@ -174,16 +155,18 @@ def fail(reason: str = "", pytrace: bool = True) -> NoReturn: :raises pytest.fail.Exception: The exception that is raised. """ - __tracebackhide__ = True - raise Failed(msg=reason, pytrace=pytrace) + Exception: ClassVar[type[Failed]] = Failed -class XFailed(Failed): - """Raised from an explicit call to pytest.xfail().""" + def __call__(self, reason: str = "", pytrace: bool = True) -> NoReturn: + __tracebackhide__ = True + raise Failed(msg=reason, pytrace=pytrace) -@_with_exception(XFailed) -def xfail(reason: str = "") -> NoReturn: +fail: _Fail = _Fail() + + +class _XFail: """Imperatively xfail an executing test or setup function with the given reason. This function should be called only during testing (setup, call or teardown). @@ -202,8 +185,15 @@ def xfail(reason: str = "") -> NoReturn: :raises pytest.xfail.Exception: The exception that is raised. """ - __tracebackhide__ = True - raise XFailed(reason) + + Exception: ClassVar[type[XFailed]] = XFailed + + def __call__(self, reason: str = "") -> NoReturn: + __tracebackhide__ = True + raise XFailed(msg=reason) + + +xfail: _XFail = _XFail() def importorskip( diff --git a/src/_pytest/pastebin.py b/src/_pytest/pastebin.py index 69c011ed24a..c7b39d96f02 100644 --- a/src/_pytest/pastebin.py +++ b/src/_pytest/pastebin.py @@ -20,7 +20,7 @@ def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("terminal reporting") - group._addoption( + group.addoption( "--pastebin", metavar="mode", action="store", @@ -76,6 +76,7 @@ def create_new_paste(contents: str | bytes) -> str: :returns: URL to the pasted contents, or an error message. """ import re + from urllib.error import HTTPError from urllib.parse import urlencode from urllib.request import urlopen @@ -85,8 +86,11 @@ def create_new_paste(contents: str | bytes) -> str: response: str = ( urlopen(url, data=urlencode(params).encode("ascii")).read().decode("utf-8") ) - except OSError as exc_info: # urllib errors - return f"bad response: {exc_info}" + except HTTPError as e: + with e: # HTTPErrors are also http responses that must be closed! + return f"bad response: {e}" + except OSError as e: # eg urllib.error.URLError + return f"bad response: {e}" m = re.search(r'href="/raw/(\w+)"', response) if m: return f"{url}/show/{m.group(1)}" diff --git a/src/_pytest/pathlib.py b/src/_pytest/pathlib.py index e4dc4eddc9c..cd15434605d 100644 --- a/src/_pytest/pathlib.py +++ b/src/_pytest/pathlib.py @@ -1,6 +1,9 @@ from __future__ import annotations import atexit +from collections.abc import Callable +from collections.abc import Iterable +from collections.abc import Iterator import contextlib from enum import Enum from errno import EBADF @@ -10,6 +13,7 @@ import fnmatch from functools import partial from importlib.machinery import ModuleSpec +from importlib.machinery import PathFinder import importlib.util import itertools import os @@ -25,9 +29,6 @@ import types from types import ModuleType from typing import Any -from typing import Callable -from typing import Iterable -from typing import Iterator from typing import TypeVar import uuid import warnings @@ -37,8 +38,12 @@ from _pytest.warning_types import PytestWarning -LOCK_TIMEOUT = 60 * 60 * 24 * 3 +if sys.version_info < (3, 11): + from importlib._bootstrap_external import _NamespaceLoader as NamespaceLoader +else: + from importlib.machinery import NamespaceLoader +LOCK_TIMEOUT = 60 * 60 * 24 * 3 _AnyPurePath = TypeVar("_AnyPurePath", bound=PurePath) @@ -343,7 +348,7 @@ def cleanup_candidates(root: Path, prefix: str, keep: int) -> Iterator[Path]: entries = find_prefixed(root, prefix) entries, entries2 = itertools.tee(entries) numbers = map(parse_num, extract_suffixes(entries2, prefix)) - for entry, number in zip(entries, numbers): + for entry, number in zip(entries, numbers, strict=True): if number <= max_delete: yield Path(entry) @@ -611,50 +616,109 @@ def _import_module_using_spec( module_name: str, module_path: Path, module_location: Path, *, insert_modules: bool ) -> ModuleType | None: """ - Tries to import a module by its canonical name, path to the .py file, and its - parent location. + Tries to import a module by its canonical name, path, and its parent location. + + :param module_name: + The expected module name, will become the key of `sys.modules`. + + :param module_path: + The file path of the module, for example `/foo/bar/test_demo.py`. + If module is a package, pass the path to the `__init__.py` of the package. + If module is a namespace package, pass directory path. + + :param module_location: + The parent location of the module. + If module is a package, pass the directory containing the `__init__.py` file. :param insert_modules: - If True, will call insert_missing_modules to create empty intermediate modules - for made-up module names (when importing test files not reachable from sys.path). + If True, will call `insert_missing_modules` to create empty intermediate modules + with made-up module names (when importing test files not reachable from `sys.path`). + + Example 1 of parent_module_*: + + module_name: "a.b.c.demo" + module_path: Path("a/b/c/demo.py") + module_location: Path("a/b/c/") + if "a.b.c" is package ("a/b/c/__init__.py" exists), then + parent_module_name: "a.b.c" + parent_module_path: Path("a/b/c/__init__.py") + parent_module_location: Path("a/b/c/") + else: + parent_module_name: "a.b.c" + parent_module_path: Path("a/b/c") + parent_module_location: Path("a/b/") + + Example 2 of parent_module_*: + + module_name: "a.b.c" + module_path: Path("a/b/c/__init__.py") + module_location: Path("a/b/c/") + if "a.b" is package ("a/b/__init__.py" exists), then + parent_module_name: "a.b" + parent_module_path: Path("a/b/__init__.py") + parent_module_location: Path("a/b/") + else: + parent_module_name: "a.b" + parent_module_path: Path("a/b/") + parent_module_location: Path("a/") """ + # Attempt to import the parent module, seems is our responsibility: + # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 + parent_module_name, _, name = module_name.rpartition(".") + parent_module: ModuleType | None = None + if parent_module_name: + parent_module = sys.modules.get(parent_module_name) + # If the parent_module lacks the `__path__` attribute, AttributeError when finding a submodule's spec, + # requiring re-import according to the path. + need_reimport = not hasattr(parent_module, "__path__") + if parent_module is None or need_reimport: + # Get parent_location based on location, get parent_path based on path. + if module_path.name == "__init__.py": + # If the current module is in a package, + # need to leave the package first and then enter the parent module. + parent_module_path = module_path.parent.parent + else: + parent_module_path = module_path.parent + + if (parent_module_path / "__init__.py").is_file(): + # If the parent module is a package, loading by __init__.py file. + parent_module_path = parent_module_path / "__init__.py" + + parent_module = _import_module_using_spec( + parent_module_name, + parent_module_path, + parent_module_path.parent, + insert_modules=insert_modules, + ) + # Checking with sys.meta_path first in case one of its hooks can import this module, # such as our own assertion-rewrite hook. for meta_importer in sys.meta_path: - spec = meta_importer.find_spec(module_name, [str(module_location)]) + module_name_of_meta = getattr(meta_importer.__class__, "__module__", "") + if module_name_of_meta == "_pytest.assertion.rewrite" and module_path.is_file(): + # Import modules in subdirectories by module_path + # to ensure assertion rewrites are not missed (#12659). + find_spec_path = [str(module_location), str(module_path)] + else: + find_spec_path = [str(module_location)] + + spec = meta_importer.find_spec(module_name, find_spec_path) + if spec_matches_module_path(spec, module_path): break else: - spec = importlib.util.spec_from_file_location(module_name, str(module_path)) + loader = None + if module_path.is_dir(): + # The `spec_from_file_location` matches a loader based on the file extension by default. + # For a namespace package, need to manually specify a loader. + loader = NamespaceLoader(name, module_path, PathFinder()) # type: ignore[arg-type] + + spec = importlib.util.spec_from_file_location( + module_name, str(module_path), loader=loader + ) if spec_matches_module_path(spec, module_path): assert spec is not None - # Attempt to import the parent module, seems is our responsibility: - # https://github.com/python/cpython/blob/73906d5c908c1e0b73c5436faeff7d93698fc074/Lib/importlib/_bootstrap.py#L1308-L1311 - parent_module_name, _, name = module_name.rpartition(".") - parent_module: ModuleType | None = None - if parent_module_name: - parent_module = sys.modules.get(parent_module_name) - if parent_module is None: - # Find the directory of this module's parent. - parent_dir = ( - module_path.parent.parent - if module_path.name == "__init__.py" - else module_path.parent - ) - # Consider the parent module path as its __init__.py file, if it has one. - parent_module_path = ( - parent_dir / "__init__.py" - if (parent_dir / "__init__.py").is_file() - else parent_dir - ) - parent_module = _import_module_using_spec( - parent_module_name, - parent_module_path, - parent_dir, - insert_modules=insert_modules, - ) - # Find spec and import this module. mod = importlib.util.module_from_spec(spec) sys.modules[module_name] = mod @@ -673,10 +737,21 @@ def _import_module_using_spec( def spec_matches_module_path(module_spec: ModuleSpec | None, module_path: Path) -> bool: """Return true if the given ModuleSpec can be used to import the given module path.""" - if module_spec is None or module_spec.origin is None: + if module_spec is None: return False - return Path(module_spec.origin) == module_path + if module_spec.origin: + return Path(module_spec.origin) == module_path + + # Compare the path with the `module_spec.submodule_Search_Locations` in case + # the module is part of a namespace package. + # https://docs.python.org/3/library/importlib.html#importlib.machinery.ModuleSpec.submodule_search_locations + if module_spec.submodule_search_locations: # can be None. + for path in module_spec.submodule_search_locations: + if Path(path) == module_path: + return True + + return False # Implement a special _is_same function on Windows which returns True if the two filenames @@ -880,17 +955,24 @@ def scandir( The returned entries are sorted according to the given key. The default is to sort by name. + If the directory does not exist, return an empty list. """ entries = [] - with os.scandir(path) as s: - # Skip entries with symlink loops and other brokenness, so the caller - # doesn't have to deal with it. + # Attempt to create a scandir iterator for the given path. + try: + scandir_iter = os.scandir(path) + except FileNotFoundError: + # If the directory does not exist, return an empty list. + return [] + # Use the scandir iterator in a context manager to ensure it is properly closed. + with scandir_iter as s: for entry in s: try: entry.is_file() except OSError as err: if _ignore_error(err): continue + # Reraise non-ignorable errors to avoid hiding issues. raise entries.append(entry) entries.sort(key=sort_key) # type: ignore[arg-type] @@ -971,3 +1053,11 @@ def safe_exists(p: Path) -> bool: # ValueError: stat: path too long for Windows # OSError: [WinError 123] The filename, directory name, or volume label syntax is incorrect return False + + +def samefile_nofollow(p1: Path, p2: Path) -> bool: + """Test whether two paths reference the same actual file or directory. + + Unlike Path.samefile(), does not resolve symlinks. + """ + return os.path.samestat(p1.lstat(), p2.lstat()) diff --git a/src/_pytest/pytester.py b/src/_pytest/pytester.py index 5c6ce5e889f..1cd5f05dd7e 100644 --- a/src/_pytest/pytester.py +++ b/src/_pytest/pytester.py @@ -7,6 +7,10 @@ from __future__ import annotations import collections.abc +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Sequence import contextlib from fnmatch import fnmatch import gc @@ -22,15 +26,11 @@ import sys import traceback from typing import Any -from typing import Callable from typing import Final from typing import final -from typing import Generator from typing import IO -from typing import Iterable from typing import Literal from typing import overload -from typing import Sequence from typing import TextIO from typing import TYPE_CHECKING from weakref import WeakKeyDictionary @@ -65,7 +65,7 @@ from _pytest.reports import CollectReport from _pytest.reports import TestReport from _pytest.tmpdir import TempPathFactory -from _pytest.warning_types import PytestWarning +from _pytest.warning_types import PytestFDWarning if TYPE_CHECKING: @@ -188,7 +188,7 @@ def pytest_runtest_protocol(self, item: Item) -> Generator[None, object, object] "*** function {}:{}: {} ".format(*item.location), "See issue #2366", ] - item.warn(PytestWarning("\n".join(error))) + item.warn(PytestFDWarning("\n".join(error))) # used at least by pytest-xdist plugin @@ -491,7 +491,7 @@ def pytester( @fixture -def _sys_snapshot() -> Generator[None, None, None]: +def _sys_snapshot() -> Generator[None]: snappaths = SysPathsSnapshot() snapmods = SysModulesSnapshot() yield @@ -500,7 +500,7 @@ def _sys_snapshot() -> Generator[None, None, None]: @fixture -def _config_for_test() -> Generator[Config, None, None]: +def _config_for_test() -> Generator[Config]: from _pytest.config import get_config config = get_config() @@ -547,8 +547,10 @@ def __init__( def __repr__(self) -> str: return ( - "" - % (self.ret, len(self.stdout.lines), len(self.stderr.lines), self.duration) + f"" ) def parseoutcomes(self) -> dict[str, int]: @@ -680,9 +682,11 @@ def __init__( self._name = name self._path: Path = tmp_path_factory.mktemp(name, numbered=True) #: A list of plugins to use with :py:meth:`parseconfig` and - #: :py:meth:`runpytest`. Initially this is an empty list but plugins can - #: be added to the list. The type of items to add to the list depends on - #: the method using them so refer to them for details. + #: :py:meth:`runpytest`. Initially this is an empty list but plugins can + #: be added to the list. + #: + #: When running in subprocess mode, specify plugins by name (str) - adding + #: plugin objects directly is not supported. self.plugins: list[str | _PluggyPlugin] = [] self._sys_path_snapshot = SysPathsSnapshot() self._sys_modules_snapshot = self.__take_sys_modules_snapshot() @@ -833,6 +837,16 @@ def makeini(self, source: str) -> Path: """ return self.makefile(".ini", tox=source) + def maketoml(self, source: str) -> Path: + """Write a pytest.toml file. + + :param source: The contents. + :returns: The pytest.toml file. + + .. versionadded:: 9.0 + """ + return self.makefile(".toml", pytest=source) + def getinicfg(self, source: str) -> SectionWrapper: """Return the pytest section from the tox.ini config file.""" p = self.makeini(source) @@ -1090,6 +1104,8 @@ def inline_run( Typically we reraise keyboard interrupts from the child run. If True, the KeyboardInterrupt exception is captured. """ + from _pytest.unraisableexception import gc_collect_iterations_key + # (maybe a cpython bug?) the importlib cache sometimes isn't updated # properly between file creation and inline_run (especially if imports # are interspersed with file creation) @@ -1113,11 +1129,16 @@ def inline_run( rec = [] - class Collect: - def pytest_configure(x, config: Config) -> None: + class PytesterHelperPlugin: + @staticmethod + def pytest_configure(config: Config) -> None: rec.append(self.make_hook_recorder(config.pluginmanager)) - plugins.append(Collect()) + # The unraisable plugin GC collect slows down inline + # pytester runs too much. + config.stash[gc_collect_iterations_key] = 0 + + plugins.append(PytesterHelperPlugin()) ret = main([str(x) for x in args], plugins=plugins) if len(rec) == 1: reprec = rec.pop() @@ -1148,7 +1169,7 @@ def runpytest_inprocess( if syspathinsert: self.syspathinsert() - now = timing.time() + instant = timing.Instant() capture = _get_multicapture("sys") capture.start_capturing() try: @@ -1178,7 +1199,7 @@ class reprec: # type: ignore assert reprec.ret is not None res = RunResult( - reprec.ret, out.splitlines(), err.splitlines(), timing.time() - now + reprec.ret, out.splitlines(), err.splitlines(), instant.elapsed().seconds ) res.reprec = reprec # type: ignore return res @@ -1220,10 +1241,9 @@ def parseconfig(self, *args: str | os.PathLike[str]) -> Config: """ import _pytest.config - new_args = self._ensure_basetemp(args) - new_args = [str(x) for x in new_args] + new_args = [str(x) for x in self._ensure_basetemp(args)] - config = _pytest.config._prepareconfig(new_args, self.plugins) # type: ignore[arg-type] + config = _pytest.config._prepareconfig(new_args, self.plugins) # we don't know what the test will do with this half-setup config # object and thus we make sure it gets unconfigured properly in any # case (otherwise capturing could still be active, for example) @@ -1406,13 +1426,12 @@ def run( print(" in:", Path.cwd()) with p1.open("w", encoding="utf8") as f1, p2.open("w", encoding="utf8") as f2: - now = timing.time() + instant = timing.Instant() popen = self.popen( cmdargs, stdin=stdin, stdout=f1, stderr=f2, - close_fds=(sys.platform != "win32"), ) if popen.stdin is not None: popen.stdin.close() @@ -1433,6 +1452,8 @@ def handle_timeout() -> None: ret = popen.wait(timeout) except subprocess.TimeoutExpired: handle_timeout() + f1.flush() + f2.flush() with p1.open(encoding="utf8") as f1, p2.open(encoding="utf8") as f2: out = f1.read().splitlines() @@ -1443,7 +1464,7 @@ def handle_timeout() -> None: with contextlib.suppress(ValueError): ret = ExitCode(ret) - return RunResult(ret, out, err, timing.time() - now) + return RunResult(ret, out, err, instant.elapsed().seconds) def _dump_lines(self, lines, fp): try: @@ -1485,9 +1506,13 @@ def runpytest_subprocess( __tracebackhide__ = True p = make_numbered_dir(root=self.path, prefix="runpytest-", mode=0o700) args = (f"--basetemp={p}", *args) - plugins = [x for x in self.plugins if isinstance(x, str)] - if plugins: - args = ("-p", plugins[0], *args) + for plugin in self.plugins: + if not isinstance(plugin, str): + raise ValueError( + f"Specifying plugins as objects is not supported in pytester subprocess mode; " + f"specify by name instead: {plugin}" + ) + args = ("-p", plugin, *args) args = self._getpytestargs() + args return self.run(*args, timeout=timeout) diff --git a/src/_pytest/pytester_assertions.py b/src/_pytest/pytester_assertions.py index d543798f75a..915cc8a10ff 100644 --- a/src/_pytest/pytester_assertions.py +++ b/src/_pytest/pytester_assertions.py @@ -6,7 +6,7 @@ # module to not be already imported. from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from _pytest.reports import CollectReport from _pytest.reports import TestReport diff --git a/src/_pytest/python.py b/src/_pytest/python.py index 9182ce7dfe9..e63751877a4 100644 --- a/src/_pytest/python.py +++ b/src/_pytest/python.py @@ -6,6 +6,12 @@ import abc from collections import Counter from collections import defaultdict +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence import dataclasses import enum import fnmatch @@ -14,18 +20,14 @@ import itertools import os from pathlib import Path +import re +import textwrap import types from typing import Any -from typing import Callable -from typing import Dict +from typing import cast from typing import final -from typing import Generator -from typing import Iterable -from typing import Iterator from typing import Literal -from typing import Mapping -from typing import Pattern -from typing import Sequence +from typing import NoReturn from typing import TYPE_CHECKING import warnings @@ -43,7 +45,6 @@ from _pytest.compat import get_real_func from _pytest.compat import getimfunc from _pytest.compat import is_async_function -from _pytest.compat import is_generator from _pytest.compat import LEGACY_PATH from _pytest.compat import NOTSET from _pytest.compat import safe_getattr @@ -57,9 +58,10 @@ from _pytest.fixtures import FuncFixtureInfo from _pytest.fixtures import get_scope_node from _pytest.main import Session -from _pytest.mark import MARK_GEN from _pytest.mark import ParameterSet +from _pytest.mark.structures import _HiddenParam from _pytest.mark.structures import get_unpacked_marks +from _pytest.mark.structures import HIDDEN_PARAM from _pytest.mark.structures import Mark from _pytest.mark.structures import MarkDecorator from _pytest.mark.structures import normalize_mark_list @@ -74,11 +76,10 @@ from _pytest.stash import StashKey from _pytest.warning_types import PytestCollectionWarning from _pytest.warning_types import PytestReturnNotNoneWarning -from _pytest.warning_types import PytestUnhandledCoroutineWarning if TYPE_CHECKING: - from typing import Self + from typing_extensions import Self def pytest_addoption(parser: Parser) -> None: @@ -108,6 +109,13 @@ def pytest_addoption(parser: Parser) -> None: help="Disable string escape non-ASCII characters, might cause unwanted " "side effects(use at your own risk)", ) + parser.addini( + "strict_parametrization_ids", + type="bool", + # None => fallback to `strict`. + default=None, + help="Emit an error if non-unique parameter set IDs are detected", + ) def pytest_generate_tests(metafunc: Metafunc) -> None: @@ -135,35 +143,35 @@ def pytest_configure(config: Config) -> None: ) -def async_warn_and_skip(nodeid: str) -> None: - msg = "async def functions are not natively supported and have been skipped.\n" - msg += ( +def async_fail(nodeid: str) -> None: + msg = ( + "async def functions are not natively supported.\n" "You need to install a suitable plugin for your async framework, for example:\n" + " - anyio\n" + " - pytest-asyncio\n" + " - pytest-tornasync\n" + " - pytest-trio\n" + " - pytest-twisted" ) - msg += " - anyio\n" - msg += " - pytest-asyncio\n" - msg += " - pytest-tornasync\n" - msg += " - pytest-trio\n" - msg += " - pytest-twisted" - warnings.warn(PytestUnhandledCoroutineWarning(msg.format(nodeid))) - skip(reason="async def function and no async plugin installed (see warnings)") + fail(msg, pytrace=False) @hookimpl(trylast=True) def pytest_pyfunc_call(pyfuncitem: Function) -> object | None: testfunction = pyfuncitem.obj if is_async_function(testfunction): - async_warn_and_skip(pyfuncitem.nodeid) + async_fail(pyfuncitem.nodeid) funcargs = pyfuncitem.funcargs testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} result = testfunction(**testargs) if hasattr(result, "__await__") or hasattr(result, "__aiter__"): - async_warn_and_skip(pyfuncitem.nodeid) + async_fail(pyfuncitem.nodeid) elif result is not None: warnings.warn( PytestReturnNotNoneWarning( - f"Expected None, but {pyfuncitem.nodeid} returned {result!r}, which will be an error in a " - "future version of pytest. Did you mean to use `assert` instead of `return`?" + f"Test functions should return None, but {pyfuncitem.nodeid} returned {type(result)!r}.\n" + "Did you mean to use `assert` instead of `return`?\n" + "See https://docs.pytest.org/en/stable/how-to/assert.html#return-not-none for more information." ) ) return True @@ -211,7 +219,7 @@ def pytest_pycollect_makemodule(module_path: Path, parent) -> Module: def pytest_pycollect_makeitem( collector: Module | Class, name: str, obj: object ) -> None | nodes.Item | nodes.Collector | list[nodes.Item | nodes.Collector]: - assert isinstance(collector, (Class, Module)), type(collector) + assert isinstance(collector, Class | Module), type(collector) # Nothing was collected elsewhere, let's do it here. if safe_isclass(obj): if collector.istestclass(obj, name): @@ -233,16 +241,13 @@ def pytest_pycollect_makeitem( lineno=lineno + 1, ) elif getattr(obj, "__test__", True): - if is_generator(obj): - res = Function.from_parent(collector, name=name) - reason = ( - f"yield tests were removed in pytest 4.0 - {name} will be ignored" + if inspect.isgeneratorfunction(obj): + fail( + f"'yield' keyword is allowed in fixtures, but not in tests ({name})", + pytrace=False, ) - res.add_marker(MARK_GEN.xfail(run=False, reason=reason)) - res.warn(PytestCollectionWarning(reason)) - return res - else: - return list(collector._genfunctions(name, obj)) + return list(collector._genfunctions(name, obj)) + return None return None @@ -362,7 +367,7 @@ def classnamefilter(self, name: str) -> bool: def istestfunction(self, obj: object, name: str) -> bool: if self.funcnamefilter(name) or self.isnosetest(obj): - if isinstance(obj, (staticmethod, classmethod)): + if isinstance(obj, staticmethod | classmethod): # staticmethods and classmethods need to be unwrapped. obj = safe_getattr(obj, "__func__", False) return callable(obj) and fixtures.getfixturemarker(obj) is None @@ -378,7 +383,7 @@ def istestclass(self, obj: object, name: str) -> bool: def _matches_prefix_or_glob_option(self, option_name: str, name: str) -> bool: """Check if the given name matches the prefix or glob-pattern defined - in ini configuration.""" + in configuration.""" for option in self.config.getini(option_name): if name.startswith(option): return True @@ -405,6 +410,7 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: # __dict__ is definition ordered. seen: set[str] = set() dict_values: list[list[nodes.Item | nodes.Collector]] = [] + collect_imported_tests = self.session.config.getini("collect_imported_tests") ihook = self.ihook for dic in dicts: values: list[nodes.Item | nodes.Collector] = [] @@ -416,6 +422,13 @@ def collect(self) -> Iterable[nodes.Item | nodes.Collector]: if name in seen: continue seen.add(name) + + if not collect_imported_tests and isinstance(self, Module): + # Do not collect functions and classes from other modules. + if inspect.isfunction(obj) or inspect.isclass(obj): + if obj.__module__ != self._getobj().__name__: + continue + res = ihook.pytest_pycollect_makeitem( collector=self, name=name, obj=obj ) @@ -439,7 +452,7 @@ def _genfunctions(self, name: str, funcobj) -> Iterator[Function]: assert modulecol is not None module = modulecol.obj clscol = self.getparent(Class) - cls = clscol and clscol.obj or None + cls = (clscol and clscol.obj) or None definition = FunctionDefinition.from_parent(self, name=name, callobj=funcobj) fixtureinfo = definition._fixtureinfo @@ -464,6 +477,7 @@ def _genfunctions(self, name: str, funcobj) -> Iterator[Function]: if not metafunc._calls: yield Function.from_parent(self, name=name, fixtureinfo=fixtureinfo) else: + metafunc._recompute_direct_params_indices() # Direct parametrizations taking place in module/class-specific # `metafunc.parametrize` calls may have shadowed some fixtures, so make sure # we update what the function really needs a.k.a its fixture closure. Note that @@ -472,7 +486,7 @@ def _genfunctions(self, name: str, funcobj) -> Iterator[Function]: fixtureinfo.prune_dependency_tree() for callspec in metafunc._calls: - subname = f"{name}[{callspec.id}]" + subname = f"{name}[{callspec.id}]" if callspec._idlist else name yield Function.from_parent( self, name=subname, @@ -512,7 +526,7 @@ def importtestmodule( ) from e except ImportError as e: exc_info = ExceptionInfo.from_current() - if config.getoption("verbose") < 2: + if config.get_verbosity() < 2: exc_info.traceback = exc_info.traceback.filter(filter_traceback) exc_repr = ( exc_info.getrepr(style="short") @@ -568,7 +582,7 @@ def _register_setup_module_fixture(self) -> None: if setup_module is None and teardown_module is None: return - def xunit_setup_module_fixture(request) -> Generator[None, None, None]: + def xunit_setup_module_fixture(request) -> Generator[None]: module = request.module if setup_module is not None: _call_with_optional_argument(setup_module, module) @@ -599,7 +613,7 @@ def _register_setup_function_fixture(self) -> None: if setup_function is None and teardown_function is None: return - def xunit_setup_function_fixture(request) -> Generator[None, None, None]: + def xunit_setup_function_fixture(request) -> Generator[None]: if request.instance is not None: # in this case we are bound to an instance, so we need to let # setup_method handle this @@ -780,7 +794,7 @@ def _register_setup_class_fixture(self) -> None: if setup_class is None and teardown_class is None: return - def xunit_setup_class_fixture(request) -> Generator[None, None, None]: + def xunit_setup_class_fixture(request) -> Generator[None]: cls = request.cls if setup_class is not None: func = getimfunc(setup_class) @@ -813,7 +827,7 @@ def _register_setup_method_fixture(self) -> None: if setup_method is None and teardown_method is None: return - def xunit_setup_method_fixture(request) -> Generator[None, None, None]: + def xunit_setup_method_fixture(request) -> Generator[None]: instance = request.instance method = request.function if setup_method is not None: @@ -855,12 +869,12 @@ class IdMaker: __slots__ = ( "argnames", - "parametersets", + "config", + "func_name", "idfn", "ids", - "config", "nodeid", - "func_name", + "parametersets", ) # The argnames of the parametrization. @@ -873,8 +887,8 @@ class IdMaker: # Optionally, explicit IDs for ParameterSets by index. ids: Sequence[object | None] | None # Optionally, the pytest config. - # Used for controlling ASCII escaping, and for calling the - # :hook:`pytest_make_parametrize_id` hook. + # Used for controlling ASCII escaping, determining parametrization ID + # strictness, and for calling the :hook:`pytest_make_parametrize_id` hook. config: Config | None # Optionally, the ID of the node being parametrized. # Used only for clearer error messages. @@ -883,10 +897,13 @@ class IdMaker: # Used only for clearer error messages. func_name: str | None - def make_unique_parameterset_ids(self) -> list[str]: + def make_unique_parameterset_ids(self) -> list[str | _HiddenParam]: """Make a unique identifier for each ParameterSet, that may be used to identify the parametrization in a node ID. + If strict_parametrization_ids is enabled, and duplicates are detected, + raises CollectError. Otherwise makes the IDs unique as follows: + Format is -...-[counter], where prm_x_token is - user-provided id, if given - else an id derived from the value, applicable for certain types @@ -899,11 +916,40 @@ def make_unique_parameterset_ids(self) -> list[str]: if len(resolved_ids) != len(set(resolved_ids)): # Record the number of occurrences of each ID. id_counts = Counter(resolved_ids) + + if self._strict_parametrization_ids_enabled(): + parameters = ", ".join(self.argnames) + parametersets = ", ".join( + [saferepr(list(param.values)) for param in self.parametersets] + ) + ids = ", ".join( + id if id is not HIDDEN_PARAM else "" for id in resolved_ids + ) + duplicates = ", ".join( + id if id is not HIDDEN_PARAM else "" + for id, count in id_counts.items() + if count > 1 + ) + msg = textwrap.dedent(f""" + Duplicate parametrization IDs detected, but strict_parametrization_ids is set. + + Test name: {self.nodeid} + Parameters: {parameters} + Parameter sets: {parametersets} + IDs: {ids} + Duplicates: {duplicates} + + You can fix this problem using `@pytest.mark.parametrize(..., ids=...)` or `pytest.param(..., id=...)`. + """).strip() # noqa: E501 + raise nodes.Collector.CollectError(msg) + # Map the ID to its next suffix. id_suffixes: dict[str, int] = defaultdict(int) # Suffix non-unique IDs to make them unique. for index, id in enumerate(resolved_ids): if id_counts[id] > 1: + if id is HIDDEN_PARAM: + self._complain_multiple_hidden_parameter_sets() suffix = "" if id and id[-1].isdigit(): suffix = "_" @@ -913,25 +959,41 @@ def make_unique_parameterset_ids(self) -> list[str]: new_id = f"{id}{suffix}{id_suffixes[id]}" resolved_ids[index] = new_id id_suffixes[id] += 1 - assert len(resolved_ids) == len( - set(resolved_ids) - ), f"Internal error: {resolved_ids=}" + assert len(resolved_ids) == len(set(resolved_ids)), ( + f"Internal error: {resolved_ids=}" + ) return resolved_ids - def _resolve_ids(self) -> Iterable[str]: + def _strict_parametrization_ids_enabled(self) -> bool: + if self.config is None: + return False + strict_parametrization_ids = self.config.getini("strict_parametrization_ids") + if strict_parametrization_ids is None: + strict_parametrization_ids = self.config.getini("strict") + return cast(bool, strict_parametrization_ids) + + def _resolve_ids(self) -> Iterable[str | _HiddenParam]: """Resolve IDs for all ParameterSets (may contain duplicates).""" for idx, parameterset in enumerate(self.parametersets): if parameterset.id is not None: # ID provided directly - pytest.param(..., id="...") - yield parameterset.id + if parameterset.id is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield _ascii_escaped_by_config(parameterset.id, self.config) elif self.ids and idx < len(self.ids) and self.ids[idx] is not None: # ID provided in the IDs list - parametrize(..., ids=[...]). - yield self._idval_from_value_required(self.ids[idx], idx) + if self.ids[idx] is HIDDEN_PARAM: + yield HIDDEN_PARAM + else: + yield self._idval_from_value_required(self.ids[idx], idx) else: # ID not provided - generate it. yield "-".join( self._idval(val, argname, idx) - for val, argname in zip(parameterset.values, self.argnames) + for val, argname in zip( + parameterset.values, self.argnames, strict=True + ) ) def _idval(self, val: object, argname: str, idx: int) -> str: @@ -976,11 +1038,11 @@ def _idval_from_hook(self, val: object, argname: str) -> str | None: def _idval_from_value(self, val: object) -> str | None: """Try to make an ID for a parameter in a ParameterSet from its value, if the value type is supported.""" - if isinstance(val, (str, bytes)): + if isinstance(val, str | bytes): return _ascii_escaped_by_config(val, self.config) - elif val is None or isinstance(val, (float, int, bool, complex)): + elif val is None or isinstance(val, float | int | bool | complex): return str(val) - elif isinstance(val, Pattern): + elif isinstance(val, re.Pattern): return ascii_escaped(val.pattern) elif val is NOTSET: # Fallback to default. Note that NOTSET is an enum.Enum. @@ -1000,12 +1062,7 @@ def _idval_from_value_required(self, val: object, idx: int) -> str: return id # Fail. - if self.func_name is not None: - prefix = f"In {self.func_name}: " - elif self.nodeid is not None: - prefix = f"In {self.nodeid}: " - else: - prefix = "" + prefix = self._make_error_prefix() msg = ( f"{prefix}ids contains unsupported value {saferepr(val)} (type: {type(val)!r}) at index {idx}. " "Supported types are: str, bytes, int, float, complex, bool, enum, regex or anything with a __name__." @@ -1018,6 +1075,21 @@ def _idval_from_argname(argname: str, idx: int) -> str: and the index of the ParameterSet.""" return str(argname) + str(idx) + def _complain_multiple_hidden_parameter_sets(self) -> NoReturn: + fail( + f"{self._make_error_prefix()}multiple instances of HIDDEN_PARAM " + "cannot be used in the same parametrize call, " + "because the tests names need to be unique." + ) + + def _make_error_prefix(self) -> str: + if self.func_name is not None: + return f"In {self.func_name}: " + elif self.nodeid is not None: + return f"In {self.nodeid}: " + else: + return "" + @final @dataclasses.dataclass(frozen=True) @@ -1034,6 +1106,7 @@ class CallSpec2: params: dict[str, object] = dataclasses.field(default_factory=dict) # arg name -> arg index. indices: dict[str, int] = dataclasses.field(default_factory=dict) + # arg name -> parameter scope. # Used for sorting parametrized resources. _arg2scope: Mapping[str, Scope] = dataclasses.field(default_factory=dict) # Parts which will be added to the item's name in `[..]` separated by "-". @@ -1046,17 +1119,20 @@ def setmulti( *, argnames: Iterable[str], valset: Iterable[object], - id: str, + id: str | _HiddenParam, marks: Iterable[Mark | MarkDecorator], scope: Scope, param_index: int, + nodeid: str, ) -> CallSpec2: params = self.params.copy() indices = self.indices.copy() arg2scope = dict(self._arg2scope) - for arg, val in zip(argnames, valset): + for arg, val in zip(argnames, valset, strict=True): if arg in params: - raise ValueError(f"duplicate parametrization of {arg!r}") + raise nodes.Collector.CollectError( + f"{nodeid}: duplicate parametrization of {arg!r}" + ) params[arg] = val indices[arg] = param_index arg2scope[arg] = scope @@ -1064,7 +1140,7 @@ def setmulti( params=params, indices=indices, _arg2scope=arg2scope, - _idlist=[*self._idlist, id], + _idlist=self._idlist if id is HIDDEN_PARAM else [*self._idlist, id], marks=[*self.marks, *normalize_mark_list(marks)], ) @@ -1084,7 +1160,7 @@ def get_direct_param_fixture_func(request: FixtureRequest) -> Any: # Used for storing pseudo fixturedefs for direct parametrization. -name2pseudofixturedef_key = StashKey[Dict[str, FixtureDef[Any]]]() +name2pseudofixturedef_key = StashKey[dict[str, FixtureDef[Any]]]() @final @@ -1131,6 +1207,8 @@ def __init__( # Result of parametrize(). self._calls: list[CallSpec2] = [] + self._params_directness: dict[str, Literal["indirect", "direct"]] = {} + def parametrize( self, argnames: str | Sequence[str], @@ -1144,7 +1222,7 @@ def parametrize( """Add new invocations to the underlying test function using the list of argvalues for the given argnames. Parametrization is performed during the collection phase. If you need to setup expensive resources - see about setting indirect to do it rather than at test setup time. + see about setting ``indirect`` to do it at test setup time instead. Can be called multiple times per test function (but only on different argument names), in which case each call parametrizes all previous @@ -1168,7 +1246,7 @@ def parametrize( If N argnames were specified, argvalues must be a list of N-tuples, where each tuple-element specifies a value for its respective argname. - :type argvalues: Iterable[_pytest.mark.structures.ParameterSet | Sequence[object] | object] + :param indirect: A list of arguments' names (subset of argnames) or a boolean. If True the list contains all names from the argnames. Each @@ -1187,6 +1265,11 @@ def parametrize( They are mapped to the corresponding index in ``argvalues``. ``None`` means to use the auto-generated id. + .. versionadded:: 8.4 + :ref:`hidden-param` means to hide the parameter set + from the test name. Can only be used at most 1 time, as + test names need to be unique. + If it is a callable it will be called for each entry in ``argvalues``, and the return value is used as part of the auto-generated id for the whole set (where parts are joined with @@ -1203,6 +1286,8 @@ def parametrize( It will also override any fixture-function defined scope, allowing to set a dynamic scope using test context or configuration. """ + nodeid = self.definition.nodeid + argnames, parametersets = ParameterSet._for_parametrize( argnames, argvalues, @@ -1214,7 +1299,7 @@ def parametrize( if "request" in argnames: fail( - "'request' is a reserved name and cannot be used in @pytest.mark.parametrize", + f"{nodeid}: 'request' is a reserved name and cannot be used in @pytest.mark.parametrize", pytrace=False, ) @@ -1241,15 +1326,20 @@ def parametrize( if _param_mark and _param_mark._param_ids_from and generated_ids is None: object.__setattr__(_param_mark._param_ids_from, "_param_ids_generated", ids) - # Add funcargs as fixturedefs to fixtureinfo.arg2fixturedefs by registering - # artificial "pseudo" FixtureDef's so that later at test execution time we can - # rely on a proper FixtureDef to exist for fixture setup. + # Calculate directness. + arg_directness = self._resolve_args_directness(argnames, indirect) + self._params_directness.update(arg_directness) + + # Add direct parametrizations as fixturedefs to arg2fixturedefs by + # registering artificial "pseudo" FixtureDef's such that later at test + # setup time we can rely on FixtureDefs to exist for all argnames. node = None - # If we have a scope that is higher than function, we need - # to make sure we only ever create an according fixturedef on - # a per-scope basis. We thus store and cache the fixturedef on the - # node related to the scope. - if scope_ is not Scope.Function: + # For scopes higher than function, a "pseudo" FixtureDef might have + # already been created for the scope. We thus store and cache the + # FixtureDef on the node related to the scope. + if scope_ is Scope.Function: + name2pseudofixturedef = None + else: collector = self.definition.parent assert collector is not None node = get_scope_node(collector, scope_) @@ -1265,14 +1355,10 @@ def parametrize( node = collector.session else: assert False, f"Unhandled missing scope: {scope}" - if node is None: - name2pseudofixturedef = None - else: default: dict[str, FixtureDef[Any]] = {} name2pseudofixturedef = node.stash.setdefault( name2pseudofixturedef_key, default ) - arg_directness = self._resolve_args_directness(argnames, indirect) for argname in argnames: if arg_directness[argname] == "indirect": continue @@ -1299,7 +1385,7 @@ def parametrize( newcalls = [] for callspec in self._calls or [CallSpec2()]: for param_index, (param_id, param_set) in enumerate( - zip(ids, parametersets) + zip(ids, parametersets, strict=True) ): newcallspec = callspec.setmulti( argnames=argnames, @@ -1308,6 +1394,7 @@ def parametrize( marks=param_set.marks, scope=scope_, param_index=param_index, + nodeid=nodeid, ) newcalls.append(newcallspec) self._calls = newcalls @@ -1318,7 +1405,7 @@ def _resolve_parameter_set_ids( ids: Iterable[object | None] | Callable[[Any], object | None] | None, parametersets: Sequence[ParameterSet], nodeid: str, - ) -> list[str]: + ) -> list[str | _HiddenParam]: """Resolve the actual ids for the given parameter sets. :param argnames: @@ -1445,6 +1532,12 @@ def _validate_if_using_arg_names( pytrace=False, ) + def _recompute_direct_params_indices(self) -> None: + for argname, param_type in self._params_directness.items(): + if param_type == "direct": + for i, callspec in enumerate(self._calls): + callspec.indices[argname] = i + def _find_parametrized_scope( argnames: Sequence[str], diff --git a/src/_pytest/python_api.py b/src/_pytest/python_api.py index 4174a55b589..bab70aa4a8c 100644 --- a/src/_pytest/python_api.py +++ b/src/_pytest/python_api.py @@ -2,29 +2,16 @@ from __future__ import annotations from collections.abc import Collection +from collections.abc import Mapping +from collections.abc import Sequence from collections.abc import Sized from decimal import Decimal import math from numbers import Complex import pprint -import re -from types import TracebackType +import sys from typing import Any -from typing import Callable -from typing import cast -from typing import ContextManager -from typing import final -from typing import Mapping -from typing import overload -from typing import Pattern -from typing import Sequence -from typing import Tuple -from typing import Type from typing import TYPE_CHECKING -from typing import TypeVar - -import _pytest._code -from _pytest.outcomes import fail if TYPE_CHECKING: @@ -126,7 +113,7 @@ def _check_type(self) -> None: def _recursive_sequence_map(f, x): """Recursively map a function over a sequence of arbitrary depth""" - if isinstance(x, (list, tuple)): + if isinstance(x, list | tuple): seq_type = type(x) return seq_type(_recursive_sequence_map(f, xi) for xi in x) elif _is_sequence_like(x): @@ -244,11 +231,23 @@ class ApproxMapping(ApproxBase): with numeric values (the keys can be anything).""" def __repr__(self) -> str: - return f"approx({({k: self._approx_scalar(v) for k, v in self.expected.items()})!r})" + return f"approx({ ({k: self._approx_scalar(v) for k, v in self.expected.items()})!r})" def _repr_compare(self, other_side: Mapping[object, float]) -> list[str]: import math + if len(self.expected) != len(other_side): + return [ + "Impossible to compare mappings with different sizes.", + f"Lengths: {len(self.expected)} and {len(other_side)}", + ] + + if self.expected.keys() != other_side.keys(): + return [ + "comparison failed.", + f"Mappings has different keys: expected {self.expected.keys()} but got {other_side.keys()}", + ] + approx_side_as_map = { k: self._approx_scalar(v) for k, v in self.expected.items() } @@ -257,24 +256,26 @@ def _repr_compare(self, other_side: Mapping[object, float]) -> list[str]: max_abs_diff = -math.inf max_rel_diff = -math.inf different_ids = [] - for (approx_key, approx_value), other_value in zip( - approx_side_as_map.items(), other_side.values() - ): + for approx_key, approx_value in approx_side_as_map.items(): + other_value = other_side[approx_key] if approx_value != other_value: if approx_value.expected is not None and other_value is not None: - max_abs_diff = max( - max_abs_diff, abs(approx_value.expected - other_value) - ) - if approx_value.expected == 0.0: - max_rel_diff = math.inf - else: - max_rel_diff = max( - max_rel_diff, - abs( - (approx_value.expected - other_value) - / approx_value.expected - ), + try: + max_abs_diff = max( + max_abs_diff, abs(approx_value.expected - other_value) ) + if approx_value.expected == 0.0: + max_rel_diff = math.inf + else: + max_rel_diff = max( + max_rel_diff, + abs( + (approx_value.expected - other_value) + / approx_value.expected + ), + ) + except ZeroDivisionError: + pass different_ids.append(approx_key) message_data = [ @@ -337,17 +338,21 @@ def _repr_compare(self, other_side: Sequence[float]) -> list[str]: max_rel_diff = -math.inf different_ids = [] for i, (approx_value, other_value) in enumerate( - zip(approx_side_as_map, other_side) + zip(approx_side_as_map, other_side, strict=True) ): if approx_value != other_value: - abs_diff = abs(approx_value.expected - other_value) - max_abs_diff = max(max_abs_diff, abs_diff) - if other_value == 0.0: - max_rel_diff = math.inf + try: + abs_diff = abs(approx_value.expected - other_value) + max_abs_diff = max(max_abs_diff, abs_diff) + # Ignore non-numbers for the diff calculations (#13012). + except TypeError: + pass else: - max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) + if other_value == 0.0: + max_rel_diff = math.inf + else: + max_rel_diff = max(max_rel_diff, abs_diff / abs(other_value)) different_ids.append(i) - message_data = [ (str(i), str(other_side[i]), str(approx_side_as_map[i])) for i in different_ids @@ -371,7 +376,7 @@ def __eq__(self, actual) -> bool: return super().__eq__(actual) def _yield_comparisons(self, actual): - return zip(actual, self.expected) + return zip(actual, self.expected, strict=True) def _check_type(self) -> None: __tracebackhide__ = True @@ -398,15 +403,21 @@ def __repr__(self) -> str: # Don't show a tolerance for values that aren't compared using # tolerances, i.e. non-numerics and infinities. Need to call abs to # handle complex numbers, e.g. (inf + 1j). - if (not isinstance(self.expected, (Complex, Decimal))) or math.isinf( - abs(self.expected) + if ( + isinstance(self.expected, bool) + or (not isinstance(self.expected, Complex | Decimal)) + or math.isinf(abs(self.expected) or isinstance(self.expected, bool)) ): return str(self.expected) # If a sensible tolerance can't be calculated, self.tolerance will # raise a ValueError. In this case, display '???'. try: - vetted_tolerance = f"{self.tolerance:.1e}" + if 1e-3 <= self.tolerance < 1e3: + vetted_tolerance = f"{self.tolerance:n}" + else: + vetted_tolerance = f"{self.tolerance:.1e}" + if ( isinstance(self.expected, Complex) and self.expected.imag @@ -421,22 +432,34 @@ def __repr__(self) -> str: def __eq__(self, actual) -> bool: """Return whether the given value is equal to the expected value within the pre-specified tolerance.""" + + def is_bool(val: Any) -> bool: + # Check if `val` is a native bool or numpy bool. + if isinstance(val, bool): + return True + if np := sys.modules.get("numpy"): + return isinstance(val, np.bool_) + return False + asarray = _as_numpy_array(actual) if asarray is not None: # Call ``__eq__()`` manually to prevent infinite-recursion with # numpy<1.13. See #3748. return all(self.__eq__(a) for a in asarray.flat) - # Short-circuit exact equality. - if actual == self.expected: + # Short-circuit exact equality, except for bool and np.bool_ + if is_bool(self.expected) and not is_bool(actual): + return False + elif actual == self.expected: return True # If either type is non-numeric, fall back to strict equality. # NB: we need Complex, rather than just Number, to ensure that __abs__, - # __sub__, and __float__ are defined. - if not ( - isinstance(self.expected, (Complex, Decimal)) - and isinstance(actual, (Complex, Decimal)) + # __sub__, and __float__ are defined. Also, consider bool to be + # non-numeric, even though it has the required arithmetic. + if is_bool(self.expected) or not ( + isinstance(self.expected, Complex | Decimal) + and isinstance(actual, Complex | Decimal) ): return False @@ -459,8 +482,7 @@ def __eq__(self, actual) -> bool: result: bool = abs(self.expected - actual) <= self.tolerance return result - # Ignore type because of https://github.com/python/mypy/issues/4266. - __hash__ = None # type: ignore + __hash__ = None @property def tolerance(self): @@ -516,6 +538,25 @@ class ApproxDecimal(ApproxScalar): DEFAULT_ABSOLUTE_TOLERANCE = Decimal("1e-12") DEFAULT_RELATIVE_TOLERANCE = Decimal("1e-6") + def __repr__(self) -> str: + if isinstance(self.rel, float): + rel = Decimal.from_float(self.rel) + else: + rel = self.rel + + if isinstance(self.abs, float): + abs_ = Decimal.from_float(self.abs) + else: + abs_ = self.abs + + tol_str = "???" + if rel is not None and Decimal("1e-3") <= rel <= Decimal("1e3"): + tol_str = f"{rel:.1e}" + elif abs_ is not None: + tol_str = f"{abs_:.1e}" + + return f"{self.expected} ± {tol_str}" + def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: """Assert that two numbers (or two ordered sequences of numbers) are equal to each other @@ -617,8 +658,10 @@ def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: >>> 1 + 1e-8 == approx(1, rel=1e-6, abs=1e-12) True - You can also use ``approx`` to compare nonnumeric types, or dicts and - sequences containing nonnumeric types, in which case it falls back to + **Non-numeric types** + + You can also use ``approx`` to compare non-numeric types, or dicts and + sequences containing non-numeric types, in which case it falls back to strict equality. This can be useful for comparing dicts and sequences that can contain optional values:: @@ -675,6 +718,15 @@ def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: from the `re_assert package `_. + + .. note:: + + Unlike built-in equality, this function considers + booleans unequal to numeric zero or one. For example:: + + >>> 1 == approx(True) + False + .. warning:: .. versionchanged:: 3.2 @@ -693,10 +745,10 @@ def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: .. versionchanged:: 3.7.1 ``approx`` raises ``TypeError`` when it encounters a dict value or - sequence element of nonnumeric type. + sequence element of non-numeric type. .. versionchanged:: 6.1.0 - ``approx`` falls back to strict equality for nonnumeric types instead + ``approx`` falls back to strict equality for non-numeric types instead of raising ``TypeError``. """ # Delegate the comparison to a class that knows how to deal with the type @@ -725,7 +777,7 @@ def approx(expected, rel=None, abs=None, nan_ok: bool = False) -> ApproxBase: cls = ApproxNumpy elif _is_sequence_like(expected): cls = ApproxSequenceLike - elif isinstance(expected, Collection) and not isinstance(expected, (str, bytes)): + elif isinstance(expected, Collection) and not isinstance(expected, str | bytes): msg = f"pytest.approx() only supports ordered sequences, but got: {expected!r}" raise TypeError(msg) else: @@ -738,7 +790,7 @@ def _is_sequence_like(expected: object) -> bool: return ( hasattr(expected, "__getitem__") and isinstance(expected, Sized) - and not isinstance(expected, (str, bytes)) + and not isinstance(expected, str | bytes) ) @@ -755,8 +807,6 @@ def _as_numpy_array(obj: object) -> ndarray | None: Return an ndarray if the given object is implicitly convertible to ndarray, and numpy is already imported, otherwise None. """ - import sys - np: Any = sys.modules.get("numpy") if np is not None: # avoid infinite recursion on numpy scalars, which have __array__ @@ -767,254 +817,3 @@ def _as_numpy_array(obj: object) -> ndarray | None: elif hasattr(obj, "__array__") or hasattr("obj", "__array_interface__"): return np.asarray(obj) return None - - -# builtin pytest.raises helper - -E = TypeVar("E", bound=BaseException) - - -@overload -def raises( - expected_exception: type[E] | tuple[type[E], ...], - *, - match: str | Pattern[str] | None = ..., -) -> RaisesContext[E]: ... - - -@overload -def raises( - expected_exception: type[E] | tuple[type[E], ...], - func: Callable[..., Any], - *args: Any, - **kwargs: Any, -) -> _pytest._code.ExceptionInfo[E]: ... - - -def raises( - expected_exception: type[E] | tuple[type[E], ...], *args: Any, **kwargs: Any -) -> RaisesContext[E] | _pytest._code.ExceptionInfo[E]: - r"""Assert that a code block/function call raises an exception type, or one of its subclasses. - - :param expected_exception: - The expected exception type, or a tuple if one of multiple possible - exception types are expected. Note that subclasses of the passed exceptions - will also match. - - :kwparam str | re.Pattern[str] | None match: - If specified, a string containing a regular expression, - or a regular expression object, that is tested against the string - representation of the exception and its :pep:`678` `__notes__` - using :func:`re.search`. - - To match a literal string that may contain :ref:`special characters - `, the pattern can first be escaped with :func:`re.escape`. - - (This is only used when ``pytest.raises`` is used as a context manager, - and passed through to the function otherwise. - When using ``pytest.raises`` as a function, you can use: - ``pytest.raises(Exc, func, match="passed on").match("my pattern")``.) - - Use ``pytest.raises`` as a context manager, which will capture the exception of the given - type, or any of its subclasses:: - - >>> import pytest - >>> with pytest.raises(ZeroDivisionError): - ... 1/0 - - If the code block does not raise the expected exception (:class:`ZeroDivisionError` in the example - above), or no exception at all, the check will fail instead. - - You can also use the keyword argument ``match`` to assert that the - exception matches a text or regex:: - - >>> with pytest.raises(ValueError, match='must be 0 or None'): - ... raise ValueError("value must be 0 or None") - - >>> with pytest.raises(ValueError, match=r'must be \d+$'): - ... raise ValueError("value must be 42") - - The ``match`` argument searches the formatted exception string, which includes any - `PEP-678 `__ ``__notes__``: - - >>> with pytest.raises(ValueError, match=r"had a note added"): # doctest: +SKIP - ... e = ValueError("value must be 42") - ... e.add_note("had a note added") - ... raise e - - The context manager produces an :class:`ExceptionInfo` object which can be used to inspect the - details of the captured exception:: - - >>> with pytest.raises(ValueError) as exc_info: - ... raise ValueError("value must be 42") - >>> assert exc_info.type is ValueError - >>> assert exc_info.value.args[0] == "value must be 42" - - .. warning:: - - Given that ``pytest.raises`` matches subclasses, be wary of using it to match :class:`Exception` like this:: - - with pytest.raises(Exception): # Careful, this will catch ANY exception raised. - some_function() - - Because :class:`Exception` is the base class of almost all exceptions, it is easy for this to hide - real bugs, where the user wrote this expecting a specific exception, but some other exception is being - raised due to a bug introduced during a refactoring. - - Avoid using ``pytest.raises`` to catch :class:`Exception` unless certain that you really want to catch - **any** exception raised. - - .. note:: - - When using ``pytest.raises`` as a context manager, it's worthwhile to - note that normal context manager rules apply and that the exception - raised *must* be the final line in the scope of the context manager. - Lines of code after that, within the scope of the context manager will - not be executed. For example:: - - >>> value = 15 - >>> with pytest.raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... assert exc_info.type is ValueError # This will not execute. - - Instead, the following approach must be taken (note the difference in - scope):: - - >>> with pytest.raises(ValueError) as exc_info: - ... if value > 10: - ... raise ValueError("value must be <= 10") - ... - >>> assert exc_info.type is ValueError - - **Using with** ``pytest.mark.parametrize`` - - When using :ref:`pytest.mark.parametrize ref` - it is possible to parametrize tests such that - some runs raise an exception and others do not. - - See :ref:`parametrizing_conditional_raising` for an example. - - .. seealso:: - - :ref:`assertraises` for more examples and detailed discussion. - - **Legacy form** - - It is possible to specify a callable by passing a to-be-called lambda:: - - >>> raises(ZeroDivisionError, lambda: 1/0) - - - or you can specify an arbitrary callable with arguments:: - - >>> def f(x): return 1/x - ... - >>> raises(ZeroDivisionError, f, 0) - - >>> raises(ZeroDivisionError, f, x=0) - - - The form above is fully supported but discouraged for new code because the - context manager form is regarded as more readable and less error-prone. - - .. note:: - Similar to caught exception objects in Python, explicitly clearing - local references to returned ``ExceptionInfo`` objects can - help the Python interpreter speed up its garbage collection. - - Clearing those references breaks a reference cycle - (``ExceptionInfo`` --> caught exception --> frame stack raising - the exception --> current frame stack --> local variables --> - ``ExceptionInfo``) which makes Python keep all objects referenced - from that cycle (including all local variables in the current - frame) alive until the next cyclic garbage collection run. - More detailed information can be found in the official Python - documentation for :ref:`the try statement `. - """ - __tracebackhide__ = True - - if not expected_exception: - raise ValueError( - f"Expected an exception type or a tuple of exception types, but got `{expected_exception!r}`. " - f"Raising exceptions is already understood as failing the test, so you don't need " - f"any special code to say 'this should never raise an exception'." - ) - if isinstance(expected_exception, type): - expected_exceptions: tuple[type[E], ...] = (expected_exception,) - else: - expected_exceptions = expected_exception - for exc in expected_exceptions: - if not isinstance(exc, type) or not issubclass(exc, BaseException): - msg = "expected exception must be a BaseException type, not {}" # type: ignore[unreachable] - not_a = exc.__name__ if isinstance(exc, type) else type(exc).__name__ - raise TypeError(msg.format(not_a)) - - message = f"DID NOT RAISE {expected_exception}" - - if not args: - match: str | Pattern[str] | None = kwargs.pop("match", None) - if kwargs: - msg = "Unexpected keyword arguments passed to pytest.raises: " - msg += ", ".join(sorted(kwargs)) - msg += "\nUse context-manager form instead?" - raise TypeError(msg) - return RaisesContext(expected_exception, message, match) - else: - func = args[0] - if not callable(func): - raise TypeError(f"{func!r} object (type: {type(func)}) must be callable") - try: - func(*args[1:], **kwargs) - except expected_exception as e: - return _pytest._code.ExceptionInfo.from_exception(e) - fail(message) - - -# This doesn't work with mypy for now. Use fail.Exception instead. -raises.Exception = fail.Exception # type: ignore - - -@final -class RaisesContext(ContextManager[_pytest._code.ExceptionInfo[E]]): - def __init__( - self, - expected_exception: type[E] | tuple[type[E], ...], - message: str, - match_expr: str | Pattern[str] | None = None, - ) -> None: - self.expected_exception = expected_exception - self.message = message - self.match_expr = match_expr - self.excinfo: _pytest._code.ExceptionInfo[E] | None = None - if self.match_expr is not None: - re_error = None - try: - re.compile(self.match_expr) - except re.error as e: - re_error = e - if re_error is not None: - fail(f"Invalid regex pattern provided to 'match': {re_error}") - - def __enter__(self) -> _pytest._code.ExceptionInfo[E]: - self.excinfo = _pytest._code.ExceptionInfo.for_later() - return self.excinfo - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - __tracebackhide__ = True - if exc_type is None: - fail(self.message) - assert self.excinfo is not None - if not issubclass(exc_type, self.expected_exception): - return False - # Cast to narrow the exception type now that it's verified. - exc_info = cast(Tuple[Type[E], E, TracebackType], (exc_type, exc_val, exc_tb)) - self.excinfo.fill_unfilled(exc_info) - if self.match_expr is not None: - self.excinfo.match(self.match_expr) - return True diff --git a/src/_pytest/python_path.py b/src/_pytest/python_path.py deleted file mode 100644 index 6e33c8a39f2..00000000000 --- a/src/_pytest/python_path.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import annotations - -import sys - -import pytest -from pytest import Config -from pytest import Parser - - -def pytest_addoption(parser: Parser) -> None: - parser.addini("pythonpath", type="paths", help="Add paths to sys.path", default=[]) - - -@pytest.hookimpl(tryfirst=True) -def pytest_load_initial_conftests(early_config: Config) -> None: - # `pythonpath = a b` will set `sys.path` to `[a, b, x, y, z, ...]` - for path in reversed(early_config.getini("pythonpath")): - sys.path.insert(0, str(path)) - - -@pytest.hookimpl(trylast=True) -def pytest_unconfigure(config: Config) -> None: - for path in config.getini("pythonpath"): - path_str = str(path) - if path_str in sys.path: - sys.path.remove(path_str) diff --git a/src/_pytest/raises.py b/src/_pytest/raises.py new file mode 100644 index 00000000000..7c246fde280 --- /dev/null +++ b/src/_pytest/raises.py @@ -0,0 +1,1517 @@ +from __future__ import annotations + +from abc import ABC +from abc import abstractmethod +import re +from re import Pattern +import sys +from textwrap import indent +from typing import Any +from typing import cast +from typing import final +from typing import Generic +from typing import get_args +from typing import get_origin +from typing import Literal +from typing import overload +from typing import TYPE_CHECKING +import warnings + +from _pytest._code import ExceptionInfo +from _pytest._code.code import stringify_exception +from _pytest.outcomes import fail +from _pytest.warning_types import PytestWarning + + +if TYPE_CHECKING: + from collections.abc import Callable + from collections.abc import Sequence + + # for some reason Sphinx does not play well with 'from types import TracebackType' + import types + from typing import TypeGuard + + from typing_extensions import ParamSpec + from typing_extensions import TypeVar + + P = ParamSpec("P") + + # this conditional definition is because we want to allow a TypeVar default + BaseExcT_co_default = TypeVar( + "BaseExcT_co_default", + bound=BaseException, + default=BaseException, + covariant=True, + ) + + # Use short name because it shows up in docs. + E = TypeVar("E", bound=BaseException, default=BaseException) +else: + from typing import TypeVar + + BaseExcT_co_default = TypeVar( + "BaseExcT_co_default", bound=BaseException, covariant=True + ) + +# RaisesGroup doesn't work with a default. +BaseExcT_co = TypeVar("BaseExcT_co", bound=BaseException, covariant=True) +BaseExcT_1 = TypeVar("BaseExcT_1", bound=BaseException) +BaseExcT_2 = TypeVar("BaseExcT_2", bound=BaseException) +ExcT_1 = TypeVar("ExcT_1", bound=Exception) +ExcT_2 = TypeVar("ExcT_2", bound=Exception) + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + + +# String patterns default to including the unicode flag. +_REGEX_NO_FLAGS = re.compile(r"").flags + + +# pytest.raises helper +@overload +def raises( + expected_exception: type[E] | tuple[type[E], ...], + *, + match: str | re.Pattern[str] | None = ..., + check: Callable[[E], bool] = ..., +) -> RaisesExc[E]: ... + + +@overload +def raises( + *, + match: str | re.Pattern[str], + # If exception_type is not provided, check() must do any typechecks itself. + check: Callable[[BaseException], bool] = ..., +) -> RaisesExc[BaseException]: ... + + +@overload +def raises(*, check: Callable[[BaseException], bool]) -> RaisesExc[BaseException]: ... + + +@overload +def raises( + expected_exception: type[E] | tuple[type[E], ...], + func: Callable[..., Any], + *args: Any, + **kwargs: Any, +) -> ExceptionInfo[E]: ... + + +def raises( + expected_exception: type[E] | tuple[type[E], ...] | None = None, + *args: Any, + **kwargs: Any, +) -> RaisesExc[BaseException] | ExceptionInfo[E]: + r"""Assert that a code block/function call raises an exception type, or one of its subclasses. + + :param expected_exception: + The expected exception type, or a tuple if one of multiple possible + exception types are expected. Note that subclasses of the passed exceptions + will also match. + + This is not a required parameter, you may opt to only use ``match`` and/or + ``check`` for verifying the raised exception. + + :kwparam str | re.Pattern[str] | None match: + If specified, a string containing a regular expression, + or a regular expression object, that is tested against the string + representation of the exception and its :pep:`678` `__notes__` + using :func:`re.search`. + + To match a literal string that may contain :ref:`special characters + `, the pattern can first be escaped with :func:`re.escape`. + + (This is only used when ``pytest.raises`` is used as a context manager, + and passed through to the function otherwise. + When using ``pytest.raises`` as a function, you can use: + ``pytest.raises(Exc, func, match="passed on").match("my pattern")``.) + + :kwparam Callable[[BaseException], bool] check: + + .. versionadded:: 8.4 + + If specified, a callable that will be called with the exception as a parameter + after checking the type and the match regex if specified. + If it returns ``True`` it will be considered a match, if not it will + be considered a failed match. + + + Use ``pytest.raises`` as a context manager, which will capture the exception of the given + type, or any of its subclasses:: + + >>> import pytest + >>> with pytest.raises(ZeroDivisionError): + ... 1/0 + + If the code block does not raise the expected exception (:class:`ZeroDivisionError` in the example + above), or no exception at all, the check will fail instead. + + You can also use the keyword argument ``match`` to assert that the + exception matches a text or regex:: + + >>> with pytest.raises(ValueError, match='must be 0 or None'): + ... raise ValueError("value must be 0 or None") + + >>> with pytest.raises(ValueError, match=r'must be \d+$'): + ... raise ValueError("value must be 42") + + The ``match`` argument searches the formatted exception string, which includes any + `PEP-678 `__ ``__notes__``: + + >>> with pytest.raises(ValueError, match=r"had a note added"): # doctest: +SKIP + ... e = ValueError("value must be 42") + ... e.add_note("had a note added") + ... raise e + + The ``check`` argument, if provided, must return True when passed the raised exception + for the match to be successful, otherwise an :exc:`AssertionError` is raised. + + >>> import errno + >>> with pytest.raises(OSError, check=lambda e: e.errno == errno.EACCES): + ... raise OSError(errno.EACCES, "no permission to view") + + The context manager produces an :class:`ExceptionInfo` object which can be used to inspect the + details of the captured exception:: + + >>> with pytest.raises(ValueError) as exc_info: + ... raise ValueError("value must be 42") + >>> assert exc_info.type is ValueError + >>> assert exc_info.value.args[0] == "value must be 42" + + .. warning:: + + Given that ``pytest.raises`` matches subclasses, be wary of using it to match :class:`Exception` like this:: + + # Careful, this will catch ANY exception raised. + with pytest.raises(Exception): + some_function() + + Because :class:`Exception` is the base class of almost all exceptions, it is easy for this to hide + real bugs, where the user wrote this expecting a specific exception, but some other exception is being + raised due to a bug introduced during a refactoring. + + Avoid using ``pytest.raises`` to catch :class:`Exception` unless certain that you really want to catch + **any** exception raised. + + .. note:: + + When using ``pytest.raises`` as a context manager, it's worthwhile to + note that normal context manager rules apply and that the exception + raised *must* be the final line in the scope of the context manager. + Lines of code after that, within the scope of the context manager will + not be executed. For example:: + + >>> value = 15 + >>> with pytest.raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... assert exc_info.type is ValueError # This will not execute. + + Instead, the following approach must be taken (note the difference in + scope):: + + >>> with pytest.raises(ValueError) as exc_info: + ... if value > 10: + ... raise ValueError("value must be <= 10") + ... + >>> assert exc_info.type is ValueError + + **Expecting exception groups** + + When expecting exceptions wrapped in :exc:`BaseExceptionGroup` or + :exc:`ExceptionGroup`, you should instead use :class:`pytest.RaisesGroup`. + + **Using with** ``pytest.mark.parametrize`` + + When using :ref:`pytest.mark.parametrize ref` + it is possible to parametrize tests such that + some runs raise an exception and others do not. + + See :ref:`parametrizing_conditional_raising` for an example. + + .. seealso:: + + :ref:`assertraises` for more examples and detailed discussion. + + **Legacy form** + + It is possible to specify a callable by passing a to-be-called lambda:: + + >>> raises(ZeroDivisionError, lambda: 1/0) + + + or you can specify an arbitrary callable with arguments:: + + >>> def f(x): return 1/x + ... + >>> raises(ZeroDivisionError, f, 0) + + >>> raises(ZeroDivisionError, f, x=0) + + + The form above is fully supported but discouraged for new code because the + context manager form is regarded as more readable and less error-prone. + + .. note:: + Similar to caught exception objects in Python, explicitly clearing + local references to returned ``ExceptionInfo`` objects can + help the Python interpreter speed up its garbage collection. + + Clearing those references breaks a reference cycle + (``ExceptionInfo`` --> caught exception --> frame stack raising + the exception --> current frame stack --> local variables --> + ``ExceptionInfo``) which makes Python keep all objects referenced + from that cycle (including all local variables in the current + frame) alive until the next cyclic garbage collection run. + More detailed information can be found in the official Python + documentation for :ref:`the try statement `. + """ + __tracebackhide__ = True + + if not args: + if set(kwargs) - {"match", "check", "expected_exception"}: + msg = "Unexpected keyword arguments passed to pytest.raises: " + msg += ", ".join(sorted(kwargs)) + msg += "\nUse context-manager form instead?" + raise TypeError(msg) + + if expected_exception is None: + return RaisesExc(**kwargs) + return RaisesExc(expected_exception, **kwargs) + + if not expected_exception: + raise ValueError( + f"Expected an exception type or a tuple of exception types, but got `{expected_exception!r}`. " + f"Raising exceptions is already understood as failing the test, so you don't need " + f"any special code to say 'this should never raise an exception'." + ) + func = args[0] + if not callable(func): + raise TypeError(f"{func!r} object (type: {type(func)}) must be callable") + with RaisesExc(expected_exception) as excinfo: + func(*args[1:], **kwargs) + try: + return excinfo + finally: + del excinfo + + +# note: RaisesExc/RaisesGroup uses fail() internally, so this alias +# indicates (to [internal] plugins?) that `pytest.raises` will +# raise `_pytest.outcomes.Failed`, where +# `outcomes.Failed is outcomes.fail.Exception is raises.Exception` +# note: this is *not* the same as `_pytest.main.Failed` +# note: mypy does not recognize this attribute, and it's not possible +# to use a protocol/decorator like the others in outcomes due to +# https://github.com/python/mypy/issues/18715 +raises.Exception = fail.Exception # type: ignore[attr-defined] + + +def _match_pattern(match: Pattern[str]) -> str | Pattern[str]: + """Helper function to remove redundant `re.compile` calls when printing regex""" + return match.pattern if match.flags == _REGEX_NO_FLAGS else match + + +def repr_callable(fun: Callable[[BaseExcT_1], bool]) -> str: + """Get the repr of a ``check`` parameter. + + Split out so it can be monkeypatched (e.g. by hypothesis) + """ + return repr(fun) + + +def backquote(s: str) -> str: + return "`" + s + "`" + + +def _exception_type_name( + e: type[BaseException] | tuple[type[BaseException], ...], +) -> str: + if isinstance(e, type): + return e.__name__ + if len(e) == 1: + return e[0].__name__ + return "(" + ", ".join(ee.__name__ for ee in e) + ")" + + +def _check_raw_type( + expected_type: type[BaseException] | tuple[type[BaseException], ...] | None, + exception: BaseException, +) -> str | None: + if expected_type is None or expected_type == (): + return None + + if not isinstance( + exception, + expected_type, + ): + actual_type_str = backquote(_exception_type_name(type(exception)) + "()") + expected_type_str = backquote(_exception_type_name(expected_type)) + if ( + isinstance(exception, BaseExceptionGroup) + and isinstance(expected_type, type) + and not issubclass(expected_type, BaseExceptionGroup) + ): + return f"Unexpected nested {actual_type_str}, expected {expected_type_str}" + return f"{actual_type_str} is not an instance of {expected_type_str}" + return None + + +def is_fully_escaped(s: str) -> bool: + # we know we won't compile with re.VERBOSE, so whitespace doesn't need to be escaped + metacharacters = "{}()+.*?^$[]" + return not any( + c in metacharacters and (i == 0 or s[i - 1] != "\\") for (i, c) in enumerate(s) + ) + + +def unescape(s: str) -> str: + return re.sub(r"\\([{}()+-.*?^$\[\]\s\\])", r"\1", s) + + +# These classes conceptually differ from ExceptionInfo in that ExceptionInfo is tied, and +# constructed from, a particular exception - whereas these are constructed with expected +# exceptions, and later allow matching towards particular exceptions. +# But there's overlap in `ExceptionInfo.match` and `AbstractRaises._check_match`, as with +# `AbstractRaises.matches` and `ExceptionInfo.errisinstance`+`ExceptionInfo.group_contains`. +# The interaction between these classes should perhaps be improved. +class AbstractRaises(ABC, Generic[BaseExcT_co]): + """ABC with common functionality shared between RaisesExc and RaisesGroup""" + + def __init__( + self, + *, + match: str | Pattern[str] | None, + check: Callable[[BaseExcT_co], bool] | None, + ) -> None: + if isinstance(match, str): + # juggle error in order to avoid context to fail (necessary?) + re_error = None + try: + self.match: Pattern[str] | None = re.compile(match) + except re.error as e: + re_error = e + if re_error is not None: + fail(f"Invalid regex pattern provided to 'match': {re_error}") + if match == "": + warnings.warn( + PytestWarning( + "matching against an empty string will *always* pass. If you want " + "to check for an empty message you need to pass '^$'. If you don't " + "want to match you should pass `None` or leave out the parameter." + ), + stacklevel=2, + ) + else: + self.match = match + + # check if this is a fully escaped regex and has ^$ to match fully + # in which case we can do a proper diff on error + self.rawmatch: str | None = None + if isinstance(match, str) or ( + isinstance(match, Pattern) and match.flags == _REGEX_NO_FLAGS + ): + if isinstance(match, Pattern): + match = match.pattern + if ( + match + and match[0] == "^" + and match[-1] == "$" + and is_fully_escaped(match[1:-1]) + ): + self.rawmatch = unescape(match[1:-1]) + + self.check = check + self._fail_reason: str | None = None + + # used to suppress repeated printing of `repr(self.check)` + self._nested: bool = False + + # set in self._parse_exc + self.is_baseexception = False + + def _parse_exc( + self, exc: type[BaseExcT_1] | types.GenericAlias, expected: str + ) -> type[BaseExcT_1]: + if isinstance(exc, type) and issubclass(exc, BaseException): + if not issubclass(exc, Exception): + self.is_baseexception = True + return exc + # because RaisesGroup does not support variable number of exceptions there's + # still a use for RaisesExc(ExceptionGroup[Exception]). + origin_exc: type[BaseException] | None = get_origin(exc) + if origin_exc and issubclass(origin_exc, BaseExceptionGroup): + exc_type = get_args(exc)[0] + if ( + issubclass(origin_exc, ExceptionGroup) and exc_type in (Exception, Any) + ) or ( + issubclass(origin_exc, BaseExceptionGroup) + and exc_type in (BaseException, Any) + ): + if not issubclass(origin_exc, ExceptionGroup): + self.is_baseexception = True + return cast(type[BaseExcT_1], origin_exc) + else: + raise ValueError( + f"Only `ExceptionGroup[Exception]` or `BaseExceptionGroup[BaseException]` " + f"are accepted as generic types but got `{exc}`. " + f"As `raises` will catch all instances of the specified group regardless of the " + f"generic argument specific nested exceptions has to be checked " + f"with `RaisesGroup`." + ) + # unclear if the Type/ValueError distinction is even helpful here + msg = f"Expected {expected}, but got " + if isinstance(exc, type): # type: ignore[unreachable] + raise ValueError(msg + f"{exc.__name__!r}") + if isinstance(exc, BaseException): # type: ignore[unreachable] + raise TypeError(msg + f"an exception instance: {type(exc).__name__}") + raise TypeError(msg + repr(type(exc).__name__)) + + @property + def fail_reason(self) -> str | None: + """Set after a call to :meth:`matches` to give a human-readable reason for why the match failed. + When used as a context manager the string will be printed as the reason for the + test failing.""" + return self._fail_reason + + def _check_check( + self: AbstractRaises[BaseExcT_1], + exception: BaseExcT_1, + ) -> bool: + if self.check is None: + return True + + if self.check(exception): + return True + + check_repr = "" if self._nested else " " + repr_callable(self.check) + self._fail_reason = f"check{check_repr} did not return True" + return False + + # TODO: harmonize with ExceptionInfo.match + def _check_match(self, e: BaseException) -> bool: + if self.match is None or re.search( + self.match, + stringified_exception := stringify_exception( + e, include_subexception_msg=False + ), + ): + return True + + # if we're matching a group, make sure we're explicit to reduce confusion + # if they're trying to match an exception contained within the group + maybe_specify_type = ( + f" the `{_exception_type_name(type(e))}()`" + if isinstance(e, BaseExceptionGroup) + else "" + ) + if isinstance(self.rawmatch, str): + # TODO: it instructs to use `-v` to print leading text, but that doesn't work + # I also don't know if this is the proper entry point, or tool to use at all + from _pytest.assertion.util import _diff_text + from _pytest.assertion.util import dummy_highlighter + + diff = _diff_text(self.rawmatch, stringified_exception, dummy_highlighter) + self._fail_reason = ("\n" if diff[0][0] == "-" else "") + "\n".join(diff) + return False + + self._fail_reason = ( + f"Regex pattern did not match{maybe_specify_type}.\n" + f" Expected regex: {_match_pattern(self.match)!r}\n" + f" Actual message: {stringified_exception!r}" + ) + if _match_pattern(self.match) == stringified_exception: + self._fail_reason += "\n Did you mean to `re.escape()` the regex?" + return False + + @abstractmethod + def matches( + self: AbstractRaises[BaseExcT_1], exception: BaseException + ) -> TypeGuard[BaseExcT_1]: + """Check if an exception matches the requirements of this AbstractRaises. + If it fails, :meth:`AbstractRaises.fail_reason` should be set. + """ + + +@final +class RaisesExc(AbstractRaises[BaseExcT_co_default]): + """ + .. versionadded:: 8.4 + + + This is the class constructed when calling :func:`pytest.raises`, but may be used + directly as a helper class with :class:`RaisesGroup` when you want to specify + requirements on sub-exceptions. + + You don't need this if you only want to specify the type, since :class:`RaisesGroup` + accepts ``type[BaseException]``. + + :param type[BaseException] | tuple[type[BaseException]] | None expected_exception: + The expected type, or one of several possible types. + May be ``None`` in order to only make use of ``match`` and/or ``check`` + + The type is checked with :func:`isinstance`, and does not need to be an exact match. + If that is wanted you can use the ``check`` parameter. + + :kwparam str | Pattern[str] match: + A regex to match. + + :kwparam Callable[[BaseException], bool] check: + If specified, a callable that will be called with the exception as a parameter + after checking the type and the match regex if specified. + If it returns ``True`` it will be considered a match, if not it will + be considered a failed match. + + :meth:`RaisesExc.matches` can also be used standalone to check individual exceptions. + + Examples:: + + with RaisesGroup(RaisesExc(ValueError, match="string")) + ... + with RaisesGroup(RaisesExc(check=lambda x: x.args == (3, "hello"))): + ... + with RaisesGroup(RaisesExc(check=lambda x: type(x) is ValueError)): + ... + """ + + # Trio bundled hypothesis monkeypatching, we will probably instead assume that + # hypothesis will handle that in their pytest plugin by the time this is released. + # Alternatively we could add a version of get_pretty_function_description ourselves + # https://github.com/HypothesisWorks/hypothesis/blob/8ced2f59f5c7bea3344e35d2d53e1f8f8eb9fcd8/hypothesis-python/src/hypothesis/internal/reflection.py#L439 + + # At least one of the three parameters must be passed. + @overload + def __init__( + self, + expected_exception: ( + type[BaseExcT_co_default] | tuple[type[BaseExcT_co_default], ...] + ), + /, + *, + match: str | Pattern[str] | None = ..., + check: Callable[[BaseExcT_co_default], bool] | None = ..., + ) -> None: ... + + @overload + def __init__( + self: RaisesExc[BaseException], # Give E a value. + /, + *, + match: str | Pattern[str] | None, + # If exception_type is not provided, check() must do any typechecks itself. + check: Callable[[BaseException], bool] | None = ..., + ) -> None: ... + + @overload + def __init__(self, /, *, check: Callable[[BaseException], bool]) -> None: ... + + def __init__( + self, + expected_exception: ( + type[BaseExcT_co_default] | tuple[type[BaseExcT_co_default], ...] | None + ) = None, + /, + *, + match: str | Pattern[str] | None = None, + check: Callable[[BaseExcT_co_default], bool] | None = None, + ): + super().__init__(match=match, check=check) + if isinstance(expected_exception, tuple): + expected_exceptions = expected_exception + elif expected_exception is None: + expected_exceptions = () + else: + expected_exceptions = (expected_exception,) + + if (expected_exceptions == ()) and match is None and check is None: + raise ValueError("You must specify at least one parameter to match on.") + + self.expected_exceptions = tuple( + self._parse_exc(e, expected="a BaseException type") + for e in expected_exceptions + ) + + self._just_propagate = False + + def matches( + self, + exception: BaseException | None, + ) -> TypeGuard[BaseExcT_co_default]: + """Check if an exception matches the requirements of this :class:`RaisesExc`. + If it fails, :attr:`RaisesExc.fail_reason` will be set. + + Examples:: + + assert RaisesExc(ValueError).matches(my_exception): + # is equivalent to + assert isinstance(my_exception, ValueError) + + # this can be useful when checking e.g. the ``__cause__`` of an exception. + with pytest.raises(ValueError) as excinfo: + ... + assert RaisesExc(SyntaxError, match="foo").matches(excinfo.value.__cause__) + # above line is equivalent to + assert isinstance(excinfo.value.__cause__, SyntaxError) + assert re.search("foo", str(excinfo.value.__cause__) + + """ + self._just_propagate = False + if exception is None: + self._fail_reason = "exception is None" + return False + if not self._check_type(exception): + self._just_propagate = True + return False + + if not self._check_match(exception): + return False + + return self._check_check(exception) + + def __repr__(self) -> str: + parameters = [] + if self.expected_exceptions: + parameters.append(_exception_type_name(self.expected_exceptions)) + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + parameters.append( + f"match={_match_pattern(self.match)!r}", + ) + if self.check is not None: + parameters.append(f"check={repr_callable(self.check)}") + return f"RaisesExc({', '.join(parameters)})" + + def _check_type(self, exception: BaseException) -> TypeGuard[BaseExcT_co_default]: + self._fail_reason = _check_raw_type(self.expected_exceptions, exception) + return self._fail_reason is None + + def __enter__(self) -> ExceptionInfo[BaseExcT_co_default]: + self.excinfo: ExceptionInfo[BaseExcT_co_default] = ExceptionInfo.for_later() + return self.excinfo + + # TODO: move common code into superclass + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> bool: + __tracebackhide__ = True + if exc_type is None: + if not self.expected_exceptions: + fail("DID NOT RAISE any exception") + if len(self.expected_exceptions) > 1: + fail(f"DID NOT RAISE any of {self.expected_exceptions!r}") + + fail(f"DID NOT RAISE {self.expected_exceptions[0]!r}") + + assert self.excinfo is not None, ( + "Internal error - should have been constructed in __enter__" + ) + + if not self.matches(exc_val): + if self._just_propagate: + return False + raise AssertionError(self._fail_reason) + + # Cast to narrow the exception type now that it's verified.... + # even though the TypeGuard in self.matches should be narrowing + exc_info = cast( + "tuple[type[BaseExcT_co_default], BaseExcT_co_default, types.TracebackType]", + (exc_type, exc_val, exc_tb), + ) + self.excinfo.fill_unfilled(exc_info) + return True + + +@final +class RaisesGroup(AbstractRaises[BaseExceptionGroup[BaseExcT_co]]): + """ + .. versionadded:: 8.4 + + Contextmanager for checking for an expected :exc:`ExceptionGroup`. + This works similar to :func:`pytest.raises`, but allows for specifying the structure of an :exc:`ExceptionGroup`. + :meth:`ExceptionInfo.group_contains` also tries to handle exception groups, + but it is very bad at checking that you *didn't* get unexpected exceptions. + + The catching behaviour differs from :ref:`except* `, being much + stricter about the structure by default. + By using ``allow_unwrapped=True`` and ``flatten_subgroups=True`` you can match + :ref:`except* ` fully when expecting a single exception. + + :param args: + Any number of exception types, :class:`RaisesGroup` or :class:`RaisesExc` + to specify the exceptions contained in this exception. + All specified exceptions must be present in the raised group, *and no others*. + + If you expect a variable number of exceptions you need to use + :func:`pytest.raises(ExceptionGroup) ` and manually check + the contained exceptions. Consider making use of :meth:`RaisesExc.matches`. + + It does not care about the order of the exceptions, so + ``RaisesGroup(ValueError, TypeError)`` + is equivalent to + ``RaisesGroup(TypeError, ValueError)``. + :kwparam str | re.Pattern[str] | None match: + If specified, a string containing a regular expression, + or a regular expression object, that is tested against the string + representation of the exception group and its :pep:`678` `__notes__` + using :func:`re.search`. + + To match a literal string that may contain :ref:`special characters + `, the pattern can first be escaped with :func:`re.escape`. + + Note that " (5 subgroups)" will be stripped from the ``repr`` before matching. + :kwparam Callable[[E], bool] check: + If specified, a callable that will be called with the group as a parameter + after successfully matching the expected exceptions. If it returns ``True`` + it will be considered a match, if not it will be considered a failed match. + :kwparam bool allow_unwrapped: + If expecting a single exception or :class:`RaisesExc` it will match even + if the exception is not inside an exceptiongroup. + + Using this together with ``match``, ``check`` or expecting multiple exceptions + will raise an error. + :kwparam bool flatten_subgroups: + "flatten" any groups inside the raised exception group, extracting all exceptions + inside any nested groups, before matching. Without this it expects you to + fully specify the nesting structure by passing :class:`RaisesGroup` as expected + parameter. + + Examples:: + + with RaisesGroup(ValueError): + raise ExceptionGroup("", (ValueError(),)) + # match + with RaisesGroup( + ValueError, + ValueError, + RaisesExc(TypeError, match="^expected int$"), + match="^my group$", + ): + raise ExceptionGroup( + "my group", + [ + ValueError(), + TypeError("expected int"), + ValueError(), + ], + ) + # check + with RaisesGroup( + KeyboardInterrupt, + match="^hello$", + check=lambda x: isinstance(x.__cause__, ValueError), + ): + raise BaseExceptionGroup("hello", [KeyboardInterrupt()]) from ValueError + # nested groups + with RaisesGroup(RaisesGroup(ValueError)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # flatten_subgroups + with RaisesGroup(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + # allow_unwrapped + with RaisesGroup(ValueError, allow_unwrapped=True): + raise ValueError + + + :meth:`RaisesGroup.matches` can also be used directly to check a standalone exception group. + + + The matching algorithm is greedy, which means cases such as this may fail:: + + with RaisesGroup(ValueError, RaisesExc(ValueError, match="hello")): + raise ExceptionGroup("", (ValueError("hello"), ValueError("goodbye"))) + + even though it generally does not care about the order of the exceptions in the group. + To avoid the above you should specify the first :exc:`ValueError` with a :class:`RaisesExc` as well. + + .. note:: + When raised exceptions don't match the expected ones, you'll get a detailed error + message explaining why. This includes ``repr(check)`` if set, which in Python can be + overly verbose, showing memory locations etc etc. + + If installed and imported (in e.g. ``conftest.py``), the ``hypothesis`` library will + monkeypatch this output to provide shorter & more readable repr's. + """ + + # allow_unwrapped=True requires: singular exception, exception not being + # RaisesGroup instance, match is None, check is None + @overload + def __init__( + self, + expected_exception: type[BaseExcT_co] | RaisesExc[BaseExcT_co], + /, + *, + allow_unwrapped: Literal[True], + flatten_subgroups: bool = False, + ) -> None: ... + + # flatten_subgroups = True also requires no nested RaisesGroup + @overload + def __init__( + self, + expected_exception: type[BaseExcT_co] | RaisesExc[BaseExcT_co], + /, + *other_exceptions: type[BaseExcT_co] | RaisesExc[BaseExcT_co], + flatten_subgroups: Literal[True], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_co]], bool] | None = None, + ) -> None: ... + + # simplify the typevars if possible (the following 3 are equivalent but go simpler->complicated) + # ... the first handles RaisesGroup[ValueError], the second RaisesGroup[ExceptionGroup[ValueError]], + # the third RaisesGroup[ValueError | ExceptionGroup[ValueError]]. + # ... otherwise, we will get results like RaisesGroup[ValueError | ExceptionGroup[Never]] (I think) + # (technically correct but misleading) + @overload + def __init__( + self: RaisesGroup[ExcT_1], + expected_exception: type[ExcT_1] | RaisesExc[ExcT_1], + /, + *other_exceptions: type[ExcT_1] | RaisesExc[ExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExceptionGroup[ExcT_2]], + expected_exception: RaisesGroup[ExcT_2], + /, + *other_exceptions: RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: Callable[[ExceptionGroup[ExceptionGroup[ExcT_2]]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[ExcT_1 | ExceptionGroup[ExcT_2]], + expected_exception: type[ExcT_1] | RaisesExc[ExcT_1] | RaisesGroup[ExcT_2], + /, + *other_exceptions: type[ExcT_1] | RaisesExc[ExcT_1] | RaisesGroup[ExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[ExceptionGroup[ExcT_1 | ExceptionGroup[ExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + # same as the above 3 but handling BaseException + @overload + def __init__( + self: RaisesGroup[BaseExcT_1], + expected_exception: type[BaseExcT_1] | RaisesExc[BaseExcT_1], + /, + *other_exceptions: type[BaseExcT_1] | RaisesExc[BaseExcT_1], + match: str | Pattern[str] | None = None, + check: Callable[[BaseExceptionGroup[BaseExcT_1]], bool] | None = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExceptionGroup[BaseExcT_2]], + expected_exception: RaisesGroup[BaseExcT_2], + /, + *other_exceptions: RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExceptionGroup[BaseExcT_2]]], bool] | None + ) = None, + ) -> None: ... + + @overload + def __init__( + self: RaisesGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + expected_exception: type[BaseExcT_1] + | RaisesExc[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + /, + *other_exceptions: type[BaseExcT_1] + | RaisesExc[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + match: str | Pattern[str] | None = None, + check: ( + Callable[ + [BaseExceptionGroup[BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]]], + bool, + ] + | None + ) = None, + ) -> None: ... + + def __init__( + self: RaisesGroup[ExcT_1 | BaseExcT_1 | BaseExceptionGroup[BaseExcT_2]], + expected_exception: type[BaseExcT_1] + | RaisesExc[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + /, + *other_exceptions: type[BaseExcT_1] + | RaisesExc[BaseExcT_1] + | RaisesGroup[BaseExcT_2], + allow_unwrapped: bool = False, + flatten_subgroups: bool = False, + match: str | Pattern[str] | None = None, + check: ( + Callable[[BaseExceptionGroup[BaseExcT_1]], bool] + | Callable[[ExceptionGroup[ExcT_1]], bool] + | None + ) = None, + ): + # The type hint on the `self` and `check` parameters uses different formats + # that are *very* hard to reconcile while adhering to the overloads, so we cast + # it to avoid an error when passing it to super().__init__ + check = cast( + "Callable[[BaseExceptionGroup[ExcT_1|BaseExcT_1|BaseExceptionGroup[BaseExcT_2]]], bool]", + check, + ) + super().__init__(match=match, check=check) + self.allow_unwrapped = allow_unwrapped + self.flatten_subgroups: bool = flatten_subgroups + self.is_baseexception = False + + if allow_unwrapped and other_exceptions: + raise ValueError( + "You cannot specify multiple exceptions with `allow_unwrapped=True.`" + " If you want to match one of multiple possible exceptions you should" + " use a `RaisesExc`." + " E.g. `RaisesExc(check=lambda e: isinstance(e, (...)))`", + ) + if allow_unwrapped and isinstance(expected_exception, RaisesGroup): + raise ValueError( + "`allow_unwrapped=True` has no effect when expecting a `RaisesGroup`." + " You might want it in the expected `RaisesGroup`, or" + " `flatten_subgroups=True` if you don't care about the structure.", + ) + if allow_unwrapped and (match is not None or check is not None): + raise ValueError( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `RaisesExc` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards.", + ) + + self.expected_exceptions: tuple[ + type[BaseExcT_co] | RaisesExc[BaseExcT_co] | RaisesGroup[BaseException], ... + ] = tuple( + self._parse_excgroup(e, "a BaseException type, RaisesExc, or RaisesGroup") + for e in ( + expected_exception, + *other_exceptions, + ) + ) + + def _parse_excgroup( + self, + exc: ( + type[BaseExcT_co] + | types.GenericAlias + | RaisesExc[BaseExcT_1] + | RaisesGroup[BaseExcT_2] + ), + expected: str, + ) -> type[BaseExcT_co] | RaisesExc[BaseExcT_1] | RaisesGroup[BaseExcT_2]: + # verify exception type and set `self.is_baseexception` + if isinstance(exc, RaisesGroup): + if self.flatten_subgroups: + raise ValueError( + "You cannot specify a nested structure inside a RaisesGroup with" + " `flatten_subgroups=True`. The parameter will flatten subgroups" + " in the raised exceptiongroup before matching, which would never" + " match a nested structure.", + ) + self.is_baseexception |= exc.is_baseexception + exc._nested = True + return exc + elif isinstance(exc, RaisesExc): + self.is_baseexception |= exc.is_baseexception + exc._nested = True + return exc + elif isinstance(exc, tuple): + raise TypeError( + f"Expected {expected}, but got {type(exc).__name__!r}.\n" + "RaisesGroup does not support tuples of exception types when expecting one of " + "several possible exception types like RaisesExc.\n" + "If you meant to expect a group with multiple exceptions, list them as separate arguments." + ) + else: + return super()._parse_exc(exc, expected) + + @overload + def __enter__( + self: RaisesGroup[ExcT_1], + ) -> ExceptionInfo[ExceptionGroup[ExcT_1]]: ... + @overload + def __enter__( + self: RaisesGroup[BaseExcT_1], + ) -> ExceptionInfo[BaseExceptionGroup[BaseExcT_1]]: ... + + def __enter__(self) -> ExceptionInfo[BaseExceptionGroup[BaseException]]: + self.excinfo: ExceptionInfo[BaseExceptionGroup[BaseExcT_co]] = ( + ExceptionInfo.for_later() + ) + return self.excinfo + + def __repr__(self) -> str: + reqs = [ + e.__name__ if isinstance(e, type) else repr(e) + for e in self.expected_exceptions + ] + if self.allow_unwrapped: + reqs.append(f"allow_unwrapped={self.allow_unwrapped}") + if self.flatten_subgroups: + reqs.append(f"flatten_subgroups={self.flatten_subgroups}") + if self.match is not None: + # If no flags were specified, discard the redundant re.compile() here. + reqs.append(f"match={_match_pattern(self.match)!r}") + if self.check is not None: + reqs.append(f"check={repr_callable(self.check)}") + return f"RaisesGroup({', '.join(reqs)})" + + def _unroll_exceptions( + self, + exceptions: Sequence[BaseException], + ) -> Sequence[BaseException]: + """Used if `flatten_subgroups=True`.""" + res: list[BaseException] = [] + for exc in exceptions: + if isinstance(exc, BaseExceptionGroup): + res.extend(self._unroll_exceptions(exc.exceptions)) + + else: + res.append(exc) + return res + + @overload + def matches( + self: RaisesGroup[ExcT_1], + exception: BaseException | None, + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def matches( + self: RaisesGroup[BaseExcT_1], + exception: BaseException | None, + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def matches( + self, + exception: BaseException | None, + ) -> bool: + """Check if an exception matches the requirements of this RaisesGroup. + If it fails, `RaisesGroup.fail_reason` will be set. + + Example:: + + with pytest.raises(TypeError) as excinfo: + ... + assert RaisesGroup(ValueError).matches(excinfo.value.__cause__) + # the above line is equivalent to + myexc = excinfo.value.__cause + assert isinstance(myexc, BaseExceptionGroup) + assert len(myexc.exceptions) == 1 + assert isinstance(myexc.exceptions[0], ValueError) + """ + self._fail_reason = None + if exception is None: + self._fail_reason = "exception is None" + return False + if not isinstance(exception, BaseExceptionGroup): + # we opt to only print type of the exception here, as the repr would + # likely be quite long + not_group_msg = f"`{type(exception).__name__}()` is not an exception group" + if len(self.expected_exceptions) > 1: + self._fail_reason = not_group_msg + return False + # if we have 1 expected exception, check if it would work even if + # allow_unwrapped is not set + res = self._check_expected(self.expected_exceptions[0], exception) + if res is None and self.allow_unwrapped: + return True + + if res is None: + self._fail_reason = ( + f"{not_group_msg}, but would match with `allow_unwrapped=True`" + ) + elif self.allow_unwrapped: + self._fail_reason = res + else: + self._fail_reason = not_group_msg + return False + + actual_exceptions: Sequence[BaseException] = exception.exceptions + if self.flatten_subgroups: + actual_exceptions = self._unroll_exceptions(actual_exceptions) + + if not self._check_match(exception): + self._fail_reason = cast(str, self._fail_reason) + old_reason = self._fail_reason + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + and isinstance(actual := actual_exceptions[0], expected) + and self._check_match(actual) + ): + assert self.match is not None, "can't be None if _check_match failed" + assert self._fail_reason is old_reason is not None + self._fail_reason += ( + f"\n" + f" but matched the expected `{self._repr_expected(expected)}`.\n" + f" You might want " + f"`RaisesGroup(RaisesExc({expected.__name__}, match={_match_pattern(self.match)!r}))`" + ) + else: + self._fail_reason = old_reason + return False + + # do the full check on expected exceptions + if not self._check_exceptions( + exception, + actual_exceptions, + ): + self._fail_reason = cast(str, self._fail_reason) + assert self._fail_reason is not None + old_reason = self._fail_reason + # if we're not expecting a nested structure, and there is one, do a second + # pass where we try flattening it + if ( + not self.flatten_subgroups + and not any( + isinstance(e, RaisesGroup) for e in self.expected_exceptions + ) + and any(isinstance(e, BaseExceptionGroup) for e in actual_exceptions) + and self._check_exceptions( + exception, + self._unroll_exceptions(exception.exceptions), + ) + ): + # only indent if it's a single-line reason. In a multi-line there's already + # indented lines that this does not belong to. + indent = " " if "\n" not in self._fail_reason else "" + self._fail_reason = ( + old_reason + + f"\n{indent}Did you mean to use `flatten_subgroups=True`?" + ) + else: + self._fail_reason = old_reason + return False + + # Only run `self.check` once we know `exception` is of the correct type. + if not self._check_check(exception): + reason = ( + cast(str, self._fail_reason) + f" on the {type(exception).__name__}" + ) + if ( + len(actual_exceptions) == len(self.expected_exceptions) == 1 + and isinstance(expected := self.expected_exceptions[0], type) + # we explicitly break typing here :) + and self._check_check(actual_exceptions[0]) # type: ignore[arg-type] + ): + self._fail_reason = reason + ( + f", but did return True for the expected {self._repr_expected(expected)}." + f" You might want RaisesGroup(RaisesExc({expected.__name__}, check=<...>))" + ) + else: + self._fail_reason = reason + return False + + return True + + @staticmethod + def _check_expected( + expected_type: ( + type[BaseException] | RaisesExc[BaseException] | RaisesGroup[BaseException] + ), + exception: BaseException, + ) -> str | None: + """Helper method for `RaisesGroup.matches` and `RaisesGroup._check_exceptions` + to check one of potentially several expected exceptions.""" + if isinstance(expected_type, type): + return _check_raw_type(expected_type, exception) + res = expected_type.matches(exception) + if res: + return None + assert expected_type.fail_reason is not None + if expected_type.fail_reason.startswith("\n"): + return f"\n{expected_type!r}: {indent(expected_type.fail_reason, ' ')}" + return f"{expected_type!r}: {expected_type.fail_reason}" + + @staticmethod + def _repr_expected(e: type[BaseException] | AbstractRaises[BaseException]) -> str: + """Get the repr of an expected type/RaisesExc/RaisesGroup, but we only want + the name if it's a type""" + if isinstance(e, type): + return _exception_type_name(e) + return repr(e) + + @overload + def _check_exceptions( + self: RaisesGroup[ExcT_1], + _exception: Exception, + actual_exceptions: Sequence[Exception], + ) -> TypeGuard[ExceptionGroup[ExcT_1]]: ... + @overload + def _check_exceptions( + self: RaisesGroup[BaseExcT_1], + _exception: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> TypeGuard[BaseExceptionGroup[BaseExcT_1]]: ... + + def _check_exceptions( + self, + _exception: BaseException, + actual_exceptions: Sequence[BaseException], + ) -> bool: + """Helper method for RaisesGroup.matches that attempts to pair up expected and actual exceptions""" + # The _exception parameter is not used, but necessary for the TypeGuard + + # full table with all results + results = ResultHolder(self.expected_exceptions, actual_exceptions) + + # (indexes of) raised exceptions that haven't (yet) found an expected + remaining_actual = list(range(len(actual_exceptions))) + # (indexes of) expected exceptions that haven't found a matching raised + failed_expected: list[int] = [] + # successful greedy matches + matches: dict[int, int] = {} + + # loop over expected exceptions first to get a more predictable result + for i_exp, expected in enumerate(self.expected_exceptions): + for i_rem in remaining_actual: + res = self._check_expected(expected, actual_exceptions[i_rem]) + results.set_result(i_exp, i_rem, res) + if res is None: + remaining_actual.remove(i_rem) + matches[i_exp] = i_rem + break + else: + failed_expected.append(i_exp) + + # All exceptions matched up successfully + if not remaining_actual and not failed_expected: + return True + + # in case of a single expected and single raised we simplify the output + if 1 == len(actual_exceptions) == len(self.expected_exceptions): + assert not matches + self._fail_reason = res + return False + + # The test case is failing, so we can do a slow and exhaustive check to find + # duplicate matches etc that will be helpful in debugging + for i_exp, expected in enumerate(self.expected_exceptions): + for i_actual, actual in enumerate(actual_exceptions): + if results.has_result(i_exp, i_actual): + continue + results.set_result( + i_exp, i_actual, self._check_expected(expected, actual) + ) + + successful_str = ( + f"{len(matches)} matched exception{'s' if len(matches) > 1 else ''}. " + if matches + else "" + ) + + # all expected were found + if not failed_expected and results.no_match_for_actual(remaining_actual): + self._fail_reason = ( + f"{successful_str}Unexpected exception(s):" + f" {[actual_exceptions[i] for i in remaining_actual]!r}" + ) + return False + # all raised exceptions were expected + if not remaining_actual and results.no_match_for_expected(failed_expected): + no_match_for_str = ", ".join( + self._repr_expected(self.expected_exceptions[i]) + for i in failed_expected + ) + self._fail_reason = f"{successful_str}Too few exceptions raised, found no match for: [{no_match_for_str}]" + return False + + # if there's only one remaining and one failed, and the unmatched didn't match anything else, + # we elect to only print why the remaining and the failed didn't match. + if ( + 1 == len(remaining_actual) == len(failed_expected) + and results.no_match_for_actual(remaining_actual) + and results.no_match_for_expected(failed_expected) + ): + self._fail_reason = f"{successful_str}{results.get_result(failed_expected[0], remaining_actual[0])}" + return False + + # there's both expected and raised exceptions without matches + s = "" + if matches: + s += f"\n{successful_str}" + indent_1 = " " * 2 + indent_2 = " " * 4 + + if not remaining_actual: + s += "\nToo few exceptions raised!" + elif not failed_expected: + s += "\nUnexpected exception(s)!" + + if failed_expected: + s += "\nThe following expected exceptions did not find a match:" + rev_matches = {v: k for k, v in matches.items()} + for i_failed in failed_expected: + s += ( + f"\n{indent_1}{self._repr_expected(self.expected_exceptions[i_failed])}" + ) + for i_actual, actual in enumerate(actual_exceptions): + if results.get_result(i_exp, i_actual) is None: + # we print full repr of match target + s += ( + f"\n{indent_2}It matches {backquote(repr(actual))} which was paired with " + + backquote( + self._repr_expected( + self.expected_exceptions[rev_matches[i_actual]] + ) + ) + ) + + if remaining_actual: + s += "\nThe following raised exceptions did not find a match" + for i_actual in remaining_actual: + s += f"\n{indent_1}{actual_exceptions[i_actual]!r}:" + for i_exp, expected in enumerate(self.expected_exceptions): + res = results.get_result(i_exp, i_actual) + if i_exp in failed_expected: + assert res is not None + if res[0] != "\n": + s += "\n" + s += indent(res, indent_2) + if res is None: + # we print full repr of match target + s += ( + f"\n{indent_2}It matches {backquote(self._repr_expected(expected))} " + f"which was paired with {backquote(repr(actual_exceptions[matches[i_exp]]))}" + ) + + if len(self.expected_exceptions) == len(actual_exceptions) and possible_match( + results + ): + s += ( + "\nThere exist a possible match when attempting an exhaustive check," + " but RaisesGroup uses a greedy algorithm. " + "Please make your expected exceptions more stringent with `RaisesExc` etc" + " so the greedy algorithm can function." + ) + self._fail_reason = s + return False + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: types.TracebackType | None, + ) -> bool: + __tracebackhide__ = True + if exc_type is None: + fail(f"DID NOT RAISE any exception, expected `{self.expected_type()}`") + + assert self.excinfo is not None, ( + "Internal error - should have been constructed in __enter__" + ) + + # group_str is the only thing that differs between RaisesExc and RaisesGroup... + # I might just scrap it? Or make it part of fail_reason + group_str = ( + "(group)" + if self.allow_unwrapped and not issubclass(exc_type, BaseExceptionGroup) + else "group" + ) + + if not self.matches(exc_val): + fail(f"Raised exception {group_str} did not match: {self._fail_reason}") + + # Cast to narrow the exception type now that it's verified.... + # even though the TypeGuard in self.matches should be narrowing + exc_info = cast( + "tuple[type[BaseExceptionGroup[BaseExcT_co]], BaseExceptionGroup[BaseExcT_co], types.TracebackType]", + (exc_type, exc_val, exc_tb), + ) + self.excinfo.fill_unfilled(exc_info) + return True + + def expected_type(self) -> str: + subexcs = [] + for e in self.expected_exceptions: + if isinstance(e, RaisesExc): + subexcs.append(repr(e)) + elif isinstance(e, RaisesGroup): + subexcs.append(e.expected_type()) + elif isinstance(e, type): + subexcs.append(e.__name__) + else: # pragma: no cover + raise AssertionError("unknown type") + group_type = "Base" if self.is_baseexception else "" + return f"{group_type}ExceptionGroup({', '.join(subexcs)})" + + +@final +class NotChecked: + """Singleton for unchecked values in ResultHolder""" + + +class ResultHolder: + """Container for results of checking exceptions. + Used in RaisesGroup._check_exceptions and possible_match. + """ + + def __init__( + self, + expected_exceptions: tuple[ + type[BaseException] | AbstractRaises[BaseException], ... + ], + actual_exceptions: Sequence[BaseException], + ) -> None: + self.results: list[list[str | type[NotChecked] | None]] = [ + [NotChecked for _ in expected_exceptions] for _ in actual_exceptions + ] + + def set_result(self, expected: int, actual: int, result: str | None) -> None: + self.results[actual][expected] = result + + def get_result(self, expected: int, actual: int) -> str | None: + res = self.results[actual][expected] + assert res is not NotChecked + # mypy doesn't support identity checking against anything but None + return res # type: ignore[return-value] + + def has_result(self, expected: int, actual: int) -> bool: + return self.results[actual][expected] is not NotChecked + + def no_match_for_expected(self, expected: list[int]) -> bool: + for i in expected: + for actual_results in self.results: + assert actual_results[i] is not NotChecked + if actual_results[i] is None: + return False + return True + + def no_match_for_actual(self, actual: list[int]) -> bool: + for i in actual: + for res in self.results[i]: + assert res is not NotChecked + if res is None: + return False + return True + + +def possible_match(results: ResultHolder, used: set[int] | None = None) -> bool: + if used is None: + used = set() + curr_row = len(used) + if curr_row == len(results.results): + return True + return any( + val is None and i not in used and possible_match(results, used | {i}) + for (i, val) in enumerate(results.results[curr_row]) + ) diff --git a/src/_pytest/recwarn.py b/src/_pytest/recwarn.py index 3fc00d94736..e3db717bfe4 100644 --- a/src/_pytest/recwarn.py +++ b/src/_pytest/recwarn.py @@ -3,16 +3,15 @@ from __future__ import annotations +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterator from pprint import pformat import re from types import TracebackType from typing import Any -from typing import Callable from typing import final -from typing import Generator -from typing import Iterator from typing import overload -from typing import Pattern from typing import TYPE_CHECKING from typing import TypeVar @@ -32,11 +31,10 @@ @fixture -def recwarn() -> Generator[WarningsRecorder, None, None]: +def recwarn() -> Generator[WarningsRecorder]: """Return a :class:`WarningsRecorder` instance that records all warnings emitted by test functions. - See https://docs.pytest.org/en/latest/how-to/capture-warnings.html for information - on warning categories. + See :ref:`warnings` for information on warning categories. """ wrec = WarningsRecorder(_ispytest=True) with wrec: @@ -45,7 +43,9 @@ def recwarn() -> Generator[WarningsRecorder, None, None]: @overload -def deprecated_call(*, match: str | Pattern[str] | None = ...) -> WarningsRecorder: ... +def deprecated_call( + *, match: str | re.Pattern[str] | None = ... +) -> WarningsRecorder: ... @overload @@ -90,7 +90,7 @@ def deprecated_call( def warns( expected_warning: type[Warning] | tuple[type[Warning], ...] = ..., *, - match: str | Pattern[str] | None = ..., + match: str | re.Pattern[str] | None = ..., ) -> WarningsChecker: ... @@ -106,7 +106,7 @@ def warns( def warns( expected_warning: type[Warning] | tuple[type[Warning], ...] = Warning, *args: Any, - match: str | Pattern[str] | None = None, + match: str | re.Pattern[str] | None = None, **kwargs: Any, ) -> WarningsChecker | Any: r"""Assert that code raises a particular class of warning. @@ -167,7 +167,7 @@ def warns( return func(*args[1:], **kwargs) -class WarningsRecorder(warnings.catch_warnings): # type:ignore[type-arg] +class WarningsRecorder(warnings.catch_warnings): """A context manager to record raised warnings. Each recorded warning is an instance of :class:`warnings.WarningMessage`. @@ -226,7 +226,9 @@ def clear(self) -> None: """Clear the list of recorded warnings.""" self._list[:] = [] - def __enter__(self) -> Self: + # Type ignored because we basically want the `catch_warnings` generic type + # parameter to be ourselves but that is not possible(?). + def __enter__(self) -> Self: # type: ignore[override] if self._entered: __tracebackhide__ = True raise RuntimeError(f"Cannot enter {self!r} twice") @@ -259,7 +261,7 @@ class WarningsChecker(WarningsRecorder): def __init__( self, expected_warning: type[Warning] | tuple[type[Warning], ...] = Warning, - match_expr: str | Pattern[str] | None = None, + match_expr: str | re.Pattern[str] | None = None, *, _ispytest: bool = False, ) -> None: diff --git a/src/_pytest/reports.py b/src/_pytest/reports.py index 77cbf773e23..011a69db001 100644 --- a/src/_pytest/reports.py +++ b/src/_pytest/reports.py @@ -1,19 +1,20 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Iterable +from collections.abc import Iterator +from collections.abc import Mapping +from collections.abc import Sequence import dataclasses from io import StringIO import os from pprint import pprint +import sys from typing import Any from typing import cast from typing import final -from typing import Iterable -from typing import Iterator from typing import Literal -from typing import Mapping from typing import NoReturn -from typing import Sequence from typing import TYPE_CHECKING from _pytest._code.code import ExceptionChainRepr @@ -35,6 +36,10 @@ from _pytest.outcomes import skip +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + + if TYPE_CHECKING: from typing_extensions import Self @@ -188,7 +193,7 @@ def head_line(self) -> str | None: even in patch releases. """ if self.location is not None: - fspath, lineno, domain = self.location + _fspath, _lineno, domain = self.location return domain return None @@ -251,7 +256,52 @@ def _report_unserialization_failure( raise RuntimeError(stream.getvalue()) -@final +def _format_failed_longrepr( + item: Item, call: CallInfo[None], excinfo: ExceptionInfo[BaseException] +): + if call.when == "call": + longrepr = item.repr_failure(excinfo) + else: + # Exception in setup or teardown. + longrepr = item._repr_failure_py( + excinfo, style=item.config.getoption("tbstyle", "auto") + ) + return longrepr + + +def _format_exception_group_all_skipped_longrepr( + item: Item, + excinfo: ExceptionInfo[BaseExceptionGroup[BaseException | BaseExceptionGroup]], +) -> tuple[str, int, str]: + r = excinfo._getreprcrash() + assert r is not None, ( + "There should always be a traceback entry for skipping a test." + ) + if all( + getattr(skip, "_use_item_location", False) for skip in excinfo.value.exceptions + ): + path, line = item.reportinfo()[:2] + assert line is not None + loc = (os.fspath(path), line + 1) + default_msg = "skipped" + else: + loc = (str(r.path), r.lineno) + default_msg = r.message + + # Get all unique skip messages. + msgs: list[str] = [] + for exception in excinfo.value.exceptions: + m = getattr(exception, "msg", None) or ( + exception.args[0] if exception.args else None + ) + if m and m not in msgs: + msgs.append(m) + + reason = "; ".join(msgs) if msgs else default_msg + longrepr = (*loc, reason) + return longrepr + + class TestReport(BaseReport): """Basic test report object (also used for setup and teardown calls if they fail). @@ -260,6 +310,7 @@ class TestReport(BaseReport): """ __test__ = False + # Defined by skipping plugin. # xfail reason if xfailed, otherwise not defined. Use hasattr to distinguish. wasxfail: str @@ -304,7 +355,7 @@ def __init__( self.longrepr = longrepr #: One of 'setup', 'call', 'teardown' to indicate runtest phase. - self.when = when + self.when: Literal["setup", "call", "teardown"] = when #: User properties is a list of tuples (name, value) that holds user #: defined properties of the test. @@ -361,23 +412,30 @@ def from_item_and_call(cls, item: Item, call: CallInfo[None]) -> TestReport: elif isinstance(excinfo.value, skip.Exception): outcome = "skipped" r = excinfo._getreprcrash() - assert ( - r is not None - ), "There should always be a traceback entry for skipping a test." + assert r is not None, ( + "There should always be a traceback entry for skipping a test." + ) if excinfo.value._use_item_location: path, line = item.reportinfo()[:2] assert line is not None - longrepr = os.fspath(path), line + 1, r.message + longrepr = (os.fspath(path), line + 1, r.message) else: longrepr = (str(r.path), r.lineno, r.message) + elif isinstance(excinfo.value, BaseExceptionGroup) and ( + excinfo.value.split(skip.Exception)[1] is None + ): + # All exceptions in the group are skip exceptions. + outcome = "skipped" + excinfo = cast( + ExceptionInfo[ + BaseExceptionGroup[BaseException | BaseExceptionGroup] + ], + excinfo, + ) + longrepr = _format_exception_group_all_skipped_longrepr(item, excinfo) else: outcome = "failed" - if call.when == "call": - longrepr = item.repr_failure(excinfo) - else: # exception in setup or teardown - longrepr = item._repr_failure_py( - excinfo, style=item.config.getoption("tbstyle", "auto") - ) + longrepr = _format_failed_longrepr(item, call, excinfo) for rwhen, key, content in item._report_sections: sections.append((f"Captured {key} {rwhen}", content)) return cls( @@ -458,7 +516,7 @@ def toterminal(self, out: TerminalWriter) -> None: def pytest_report_to_serializable( report: CollectReport | TestReport, ) -> dict[str, Any] | None: - if isinstance(report, (TestReport, CollectReport)): + if isinstance(report, TestReport | CollectReport): data = report._to_json() data["$report_type"] = report.__class__.__name__ return data diff --git a/src/_pytest/runner.py b/src/_pytest/runner.py index 716c4948f4a..9c20ff9e638 100644 --- a/src/_pytest/runner.py +++ b/src/_pytest/runner.py @@ -4,11 +4,11 @@ from __future__ import annotations import bdb +from collections.abc import Callable import dataclasses import os import sys import types -from typing import Callable from typing import cast from typing import final from typing import Generic @@ -16,6 +16,7 @@ from typing import TYPE_CHECKING from typing import TypeVar +from .config import Config from .reports import BaseReport from .reports import CollectErrorRepr from .reports import CollectReport @@ -61,19 +62,21 @@ def pytest_addoption(parser: Parser) -> None: "--durations-min", action="store", type=float, - default=0.005, + default=None, metavar="N", help="Minimal duration in seconds for inclusion in slowest list. " - "Default: 0.005.", + "Default: 0.005 (or 0.0 if -vv is given).", ) def pytest_terminal_summary(terminalreporter: TerminalReporter) -> None: durations = terminalreporter.config.option.durations durations_min = terminalreporter.config.option.durations_min - verbose = terminalreporter.config.getvalue("verbose") + verbose = terminalreporter.config.get_verbosity() if durations is None: return + if durations_min is None: + durations_min = 0.005 if verbose < 2 else 0.0 tr = terminalreporter dlist = [] for replist in tr.stats.values(): @@ -90,11 +93,13 @@ def pytest_terminal_summary(terminalreporter: TerminalReporter) -> None: dlist = dlist[:durations] for i, rep in enumerate(dlist): - if verbose < 2 and rep.duration < durations_min: + if rep.duration < durations_min: tr.write_line("") - tr.write_line( - f"({len(dlist) - i} durations < {durations_min:g}s hidden. Use -vv to show these durations.)" - ) + message = f"({len(dlist) - i} durations < {durations_min:g}s hidden." + if terminalreporter.config.option.durations_min is None: + message += " Use -vv to show these durations." + message += ")" + tr.write_line(message) break tr.write_line(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}") @@ -167,7 +172,7 @@ def pytest_runtest_call(item: Item) -> None: del sys.last_value del sys.last_traceback if sys.version_info >= (3, 12, 0): - del sys.last_exc + del sys.last_exc # type:ignore[attr-defined] except AttributeError: pass try: @@ -177,7 +182,7 @@ def pytest_runtest_call(item: Item) -> None: sys.last_type = type(e) sys.last_value = e if sys.version_info >= (3, 12, 0): - sys.last_exc = e + sys.last_exc = e # type:ignore[attr-defined] assert e.__traceback__ is not None # Skip *this* frame sys.last_traceback = e.__traceback__.tb_next @@ -235,11 +240,11 @@ def call_and_report( runtest_hook = ihook.pytest_runtest_teardown else: assert False, f"Unhandled runtest hook case: {when}" - reraise: tuple[type[BaseException], ...] = (Exit,) - if not item.config.getoption("usepdb", False): - reraise += (KeyboardInterrupt,) + call = CallInfo.from_call( - lambda: runtest_hook(item=item, **kwds), when=when, reraise=reraise + lambda: runtest_hook(item=item, **kwds), + when=when, + reraise=get_reraise_exceptions(item.config), ) report: TestReport = ihook.pytest_runtest_makereport(item=item, call=call) if log: @@ -249,6 +254,14 @@ def call_and_report( return report +def get_reraise_exceptions(config: Config) -> tuple[type[BaseException], ...]: + """Return exception types that should not be suppressed in general.""" + reraise: tuple[type[BaseException], ...] = (Exit,) + if not config.getoption("usepdb", False): + reraise += (KeyboardInterrupt,) + return reraise + + def check_interactive_exception(call: CallInfo[object], report: BaseReport) -> bool: """Check whether the call raised an exception that should be reported as interactive.""" @@ -258,7 +271,7 @@ def check_interactive_exception(call: CallInfo[object], report: BaseReport) -> b if hasattr(report, "wasxfail"): # Exception was expected. return False - if isinstance(call.excinfo.value, (Skipped, bdb.BdbQuit)): + if isinstance(call.excinfo.value, Skipped | bdb.BdbQuit): # Special control flow exception. return False return True @@ -335,8 +348,7 @@ def from_call( function, instead of being wrapped in the CallInfo. """ excinfo = None - start = timing.time() - precise_start = timing.perf_counter() + instant = timing.Instant() try: result: TResult | None = func() except BaseException: @@ -344,14 +356,11 @@ def from_call( if reraise is not None and isinstance(excinfo.value, reraise): raise result = None - # use the perf counter - precise_stop = timing.perf_counter() - duration = precise_stop - precise_start - stop = timing.time() + duration = instant.elapsed() return cls( - start=start, - stop=stop, - duration=duration, + start=duration.start.time, + stop=duration.stop.time, + duration=duration.seconds, when=when, result=result, excinfo=excinfo, @@ -533,7 +542,7 @@ def teardown_exact(self, nextitem: Item | None) -> None: When nextitem is None (meaning we're at the last item), the entire stack is torn down. """ - needed_collectors = nextitem and nextitem.listchain() or [] + needed_collectors = (nextitem and nextitem.listchain()) or [] exceptions: list[BaseException] = [] while self.stack: if list(self.stack.keys()) == needed_collectors[: len(self.stack)]: diff --git a/src/_pytest/scope.py b/src/_pytest/scope.py index 976a3ba242e..2b007e87893 100644 --- a/src/_pytest/scope.py +++ b/src/_pytest/scope.py @@ -33,11 +33,11 @@ class Scope(Enum): """ # Scopes need to be listed from lower to higher. - Function: _ScopeName = "function" - Class: _ScopeName = "class" - Module: _ScopeName = "module" - Package: _ScopeName = "package" - Session: _ScopeName = "session" + Function = "function" + Class = "class" + Module = "module" + Package = "package" + Session = "session" def next_lower(self) -> Scope: """Return the next lower scope.""" diff --git a/src/_pytest/setuponly.py b/src/_pytest/setuponly.py index de297f408d3..7e6b46bcdb4 100644 --- a/src/_pytest/setuponly.py +++ b/src/_pytest/setuponly.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Generator +from collections.abc import Generator from _pytest._io.saferepr import saferepr from _pytest.config import Config @@ -73,13 +73,9 @@ def _show_fixture_action( # Use smaller indentation the higher the scope: Session = 0, Package = 1, etc. scope_indent = list(reversed(Scope)).index(fixturedef._scope) tw.write(" " * 2 * scope_indent) - tw.write( - "{step} {scope} {fixture}".format( # noqa: UP032 (Readability) - step=msg.ljust(8), # align the output to TEARDOWN - scope=fixturedef.scope[0].upper(), - fixture=fixturedef.argname, - ) - ) + + scopename = fixturedef.scope[0].upper() + tw.write(f"{msg:<8} {scopename} {fixturedef.argname}") if msg == "SETUP": deps = sorted(arg for arg in fixturedef.argnames if arg != "request") diff --git a/src/_pytest/skipping.py b/src/_pytest/skipping.py index 08fcb283eb2..3b067629de0 100644 --- a/src/_pytest/skipping.py +++ b/src/_pytest/skipping.py @@ -3,14 +3,13 @@ from __future__ import annotations +from collections.abc import Generator from collections.abc import Mapping import dataclasses import os import platform import sys import traceback -from typing import Generator -from typing import Optional from _pytest.config import Config from _pytest.config import hookimpl @@ -20,6 +19,7 @@ from _pytest.outcomes import fail from _pytest.outcomes import skip from _pytest.outcomes import xfail +from _pytest.raises import AbstractRaises from _pytest.reports import BaseReport from _pytest.reports import TestReport from _pytest.runner import CallInfo @@ -37,11 +37,13 @@ def pytest_addoption(parser: Parser) -> None: ) parser.addini( - "xfail_strict", + "strict_xfail", "Default for the strict parameter of xfail " - "markers when not given explicitly (default: False)", - default=False, + "markers when not given explicitly (default: False) (alias: xfail_strict)", type="bool", + # None => fallback to `strict`. + default=None, + aliases=["xfail_strict"], ) @@ -74,7 +76,7 @@ def nop(*args, **kwargs): ) config.addinivalue_line( "markers", - "xfail(condition, ..., *, reason=..., run=True, raises=None, strict=xfail_strict): " + "xfail(condition, ..., *, reason=..., run=True, raises=None, strict=strict_xfail): " "mark the test function as an expected failure if any of the conditions " "evaluate to True. Optionally specify a reason for better reporting " "and run=False if you don't even want to execute the test function. " @@ -196,19 +198,28 @@ def evaluate_skip_marks(item: Item) -> Skip | None: class Xfail: """The result of evaluate_xfail_marks().""" - __slots__ = ("reason", "run", "strict", "raises") + __slots__ = ("raises", "reason", "run", "strict") reason: str run: bool strict: bool - raises: tuple[type[BaseException], ...] | None + raises: ( + type[BaseException] + | tuple[type[BaseException], ...] + | AbstractRaises[BaseException] + | None + ) def evaluate_xfail_marks(item: Item) -> Xfail | None: """Evaluate xfail marks on item, returning Xfail if triggered.""" for mark in item.iter_markers(name="xfail"): run = mark.kwargs.get("run", True) - strict = mark.kwargs.get("strict", item.config.getini("xfail_strict")) + strict = mark.kwargs.get("strict") + if strict is None: + strict = item.config.getini("strict_xfail") + if strict is None: + strict = item.config.getini("strict") raises = mark.kwargs.get("raises", None) if "condition" not in mark.kwargs: conditions = mark.args @@ -230,7 +241,7 @@ def evaluate_xfail_marks(item: Item) -> Xfail | None: # Saves the xfail mark evaluation. Can be refreshed during call if None. -xfailed_key = StashKey[Optional[Xfail]]() +xfailed_key = StashKey[Xfail | None]() @hookimpl(tryfirst=True) @@ -245,7 +256,7 @@ def pytest_runtest_setup(item: Item) -> None: @hookimpl(wrapper=True) -def pytest_runtest_call(item: Item) -> Generator[None, None, None]: +def pytest_runtest_call(item: Item) -> Generator[None]: xfailed = item.stash.get(xfailed_key, None) if xfailed is None: item.stash[xfailed_key] = xfailed = evaluate_xfail_marks(item) @@ -272,16 +283,25 @@ def pytest_runtest_makereport( pass # don't interfere elif call.excinfo and isinstance(call.excinfo.value, xfail.Exception): assert call.excinfo.value.msg is not None - rep.wasxfail = "reason: " + call.excinfo.value.msg + rep.wasxfail = call.excinfo.value.msg rep.outcome = "skipped" elif not rep.skipped and xfailed: if call.excinfo: raises = xfailed.raises - if raises is not None and not isinstance(call.excinfo.value, raises): - rep.outcome = "failed" - else: + if raises is None or ( + ( + isinstance(raises, type | tuple) + and isinstance(call.excinfo.value, raises) + ) + or ( + isinstance(raises, AbstractRaises) + and raises.matches(call.excinfo.value) + ) + ): rep.outcome = "skipped" rep.wasxfail = xfailed.reason + else: + rep.outcome = "failed" elif call.when == "call": if xfailed.strict: rep.outcome = "failed" diff --git a/src/_pytest/stepwise.py b/src/_pytest/stepwise.py index bd906ce63c1..8901540eb59 100644 --- a/src/_pytest/stepwise.py +++ b/src/_pytest/stepwise.py @@ -1,5 +1,11 @@ from __future__ import annotations +import dataclasses +from datetime import datetime +from datetime import timedelta +from typing import Any +from typing import TYPE_CHECKING + from _pytest import nodes from _pytest.cacheprovider import Cache from _pytest.config import Config @@ -8,6 +14,9 @@ from _pytest.reports import TestReport +if TYPE_CHECKING: + from typing_extensions import Self + STEPWISE_CACHE_DIR = "cache/stepwise" @@ -30,11 +39,20 @@ def pytest_addoption(parser: Parser) -> None: help="Ignore the first failing test but stop on the next failing test. " "Implicitly enables --stepwise.", ) + group.addoption( + "--sw-reset", + "--stepwise-reset", + action="store_true", + default=False, + dest="stepwise_reset", + help="Resets stepwise state, restarting the stepwise workflow. " + "Implicitly enables --stepwise.", + ) def pytest_configure(config: Config) -> None: - if config.option.stepwise_skip: - # allow --stepwise-skip to work on its own merits. + # --stepwise-skip/--stepwise-reset implies stepwise. + if config.option.stepwise_skip or config.option.stepwise_reset: config.option.stepwise = True if config.getoption("stepwise"): config.pluginmanager.register(StepwisePlugin(config), "stepwiseplugin") @@ -47,19 +65,63 @@ def pytest_sessionfinish(session: Session) -> None: # Do not update cache if this process is a xdist worker to prevent # race conditions (#10641). return - # Clear the list of failing tests if the plugin is not active. - session.config.cache.set(STEPWISE_CACHE_DIR, []) + + +@dataclasses.dataclass +class StepwiseCacheInfo: + # The nodeid of the last failed test. + last_failed: str | None + + # The number of tests in the last time --stepwise was run. + # We use this information as a simple way to invalidate the cache information, avoiding + # confusing behavior in case the cache is stale. + last_test_count: int | None + + # The date when the cache was last updated, for information purposes only. + last_cache_date_str: str + + @property + def last_cache_date(self) -> datetime: + return datetime.fromisoformat(self.last_cache_date_str) + + @classmethod + def empty(cls) -> Self: + return cls( + last_failed=None, + last_test_count=None, + last_cache_date_str=datetime.now().isoformat(), + ) + + def update_date_to_now(self) -> None: + self.last_cache_date_str = datetime.now().isoformat() class StepwisePlugin: def __init__(self, config: Config) -> None: self.config = config self.session: Session | None = None - self.report_status = "" + self.report_status: list[str] = [] assert config.cache is not None self.cache: Cache = config.cache - self.lastfailed: str | None = self.cache.get(STEPWISE_CACHE_DIR, None) self.skip: bool = config.getoption("stepwise_skip") + self.reset: bool = config.getoption("stepwise_reset") + self.cached_info = self._load_cached_info() + + def _load_cached_info(self) -> StepwiseCacheInfo: + cached_dict: dict[str, Any] | None = self.cache.get(STEPWISE_CACHE_DIR, None) + if cached_dict: + try: + return StepwiseCacheInfo( + cached_dict["last_failed"], + cached_dict["last_test_count"], + cached_dict["last_cache_date_str"], + ) + except (KeyError, TypeError) as e: + error = f"{type(e).__name__}: {e}" + self.report_status.append(f"error reading cache, discarding ({error})") + + # Cache not found or error during load, return a new cache. + return StepwiseCacheInfo.empty() def pytest_sessionstart(self, session: Session) -> None: self.session = session @@ -67,23 +129,44 @@ def pytest_sessionstart(self, session: Session) -> None: def pytest_collection_modifyitems( self, config: Config, items: list[nodes.Item] ) -> None: - if not self.lastfailed: - self.report_status = "no previously failed tests, not skipping." + last_test_count = self.cached_info.last_test_count + self.cached_info.last_test_count = len(items) + + if self.reset: + self.report_status.append("resetting state, not skipping.") + self.cached_info.last_failed = None + return + + if not self.cached_info.last_failed: + self.report_status.append("no previously failed tests, not skipping.") + return + + if last_test_count is not None and last_test_count != len(items): + self.report_status.append( + f"test count changed, not skipping (now {len(items)} tests, previously {last_test_count})." + ) + self.cached_info.last_failed = None return - # check all item nodes until we find a match on last failed + # Check all item nodes until we find a match on last failed. failed_index = None for index, item in enumerate(items): - if item.nodeid == self.lastfailed: + if item.nodeid == self.cached_info.last_failed: failed_index = index break # If the previously failed test was not found among the test items, # do not skip any tests. if failed_index is None: - self.report_status = "previously failed test not found, not skipping." + self.report_status.append("previously failed test not found, not skipping.") else: - self.report_status = f"skipping {failed_index} already passed items." + cache_age = datetime.now() - self.cached_info.last_cache_date + # Round up to avoid showing microseconds. + cache_age = timedelta(seconds=int(cache_age.total_seconds())) + self.report_status.append( + f"skipping {failed_index} already passed items (cache from {cache_age} ago," + f" use --sw-reset to discard)." + ) deselected = items[:failed_index] del items[:failed_index] config.hook.pytest_deselected(items=deselected) @@ -93,13 +176,13 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: if self.skip: # Remove test from the failed ones (if it exists) and unset the skip option # to make sure the following tests will not be skipped. - if report.nodeid == self.lastfailed: - self.lastfailed = None + if report.nodeid == self.cached_info.last_failed: + self.cached_info.last_failed = None self.skip = False else: # Mark test as the last failing and interrupt the test session. - self.lastfailed = report.nodeid + self.cached_info.last_failed = report.nodeid assert self.session is not None self.session.shouldstop = ( "Test failed, continuing from this test next run." @@ -109,12 +192,12 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: # If the test was actually run and did pass. if report.when == "call": # Remove test from the failed ones, if exists. - if report.nodeid == self.lastfailed: - self.lastfailed = None + if report.nodeid == self.cached_info.last_failed: + self.cached_info.last_failed = None - def pytest_report_collectionfinish(self) -> str | None: - if self.config.getoption("verbose") >= 0 and self.report_status: - return f"stepwise: {self.report_status}" + def pytest_report_collectionfinish(self) -> list[str] | None: + if self.config.get_verbosity() >= 0 and self.report_status: + return [f"stepwise: {x}" for x in self.report_status] return None def pytest_sessionfinish(self) -> None: @@ -122,4 +205,5 @@ def pytest_sessionfinish(self) -> None: # Do not update cache if this process is a xdist worker to prevent # race conditions (#10641). return - self.cache.set(STEPWISE_CACHE_DIR, self.lastfailed) + self.cached_info.update_date_to_now() + self.cache.set(STEPWISE_CACHE_DIR, dataclasses.asdict(self.cached_info)) diff --git a/src/_pytest/subtests.py b/src/_pytest/subtests.py new file mode 100644 index 00000000000..e0ceb27f4b1 --- /dev/null +++ b/src/_pytest/subtests.py @@ -0,0 +1,411 @@ +"""Builtin plugin that adds subtests support.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Callable +from collections.abc import Iterator +from collections.abc import Mapping +from contextlib import AbstractContextManager +from contextlib import contextmanager +from contextlib import ExitStack +from contextlib import nullcontext +import dataclasses +import time +from types import TracebackType +from typing import Any +from typing import TYPE_CHECKING + +import pluggy + +from _pytest._code import ExceptionInfo +from _pytest._io.saferepr import saferepr +from _pytest.capture import CaptureFixture +from _pytest.capture import FDCapture +from _pytest.capture import SysCapture +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.config.argparsing import Parser +from _pytest.deprecated import check_ispytest +from _pytest.fixtures import fixture +from _pytest.fixtures import SubRequest +from _pytest.logging import catching_logs +from _pytest.logging import LogCaptureHandler +from _pytest.logging import LoggingPlugin +from _pytest.reports import TestReport +from _pytest.runner import CallInfo +from _pytest.runner import check_interactive_exception +from _pytest.runner import get_reraise_exceptions +from _pytest.stash import StashKey + + +if TYPE_CHECKING: + from typing_extensions import Self + + +def pytest_addoption(parser: Parser) -> None: + Config._add_verbosity_ini( + parser, + Config.VERBOSITY_SUBTESTS, + help=( + "Specify verbosity level for subtests. " + "Higher levels will generate output for passed subtests. Failed subtests are always reported." + ), + ) + + +@dataclasses.dataclass(frozen=True, slots=True, kw_only=True) +class SubtestContext: + """The values passed to Subtests.test() that are included in the test report.""" + + msg: str | None + kwargs: Mapping[str, Any] + + def _to_json(self) -> dict[str, Any]: + return dataclasses.asdict(self) + + @classmethod + def _from_json(cls, d: dict[str, Any]) -> Self: + return cls(msg=d["msg"], kwargs=d["kwargs"]) + + +@dataclasses.dataclass(init=False) +class SubtestReport(TestReport): + context: SubtestContext + + @property + def head_line(self) -> str: + _, _, domain = self.location + return f"{domain} {self._sub_test_description()}" + + def _sub_test_description(self) -> str: + parts = [] + if self.context.msg is not None: + parts.append(f"[{self.context.msg}]") + if self.context.kwargs: + params_desc = ", ".join( + f"{k}={saferepr(v)}" for (k, v) in self.context.kwargs.items() + ) + parts.append(f"({params_desc})") + return " ".join(parts) or "()" + + def _to_json(self) -> dict[str, Any]: + data = super()._to_json() + del data["context"] + data["_report_type"] = "SubTestReport" + data["_subtest.context"] = self.context._to_json() + return data + + @classmethod + def _from_json(cls, reportdict: dict[str, Any]) -> SubtestReport: + report = super()._from_json(reportdict) + report.context = SubtestContext._from_json(reportdict["_subtest.context"]) + return report + + @classmethod + def _new( + cls, + test_report: TestReport, + context: SubtestContext, + captured_output: Captured | None, + captured_logs: CapturedLogs | None, + ) -> Self: + result = super()._from_json(test_report._to_json()) + result.context = context + + if captured_output: + if captured_output.out: + result.sections.append(("Captured stdout call", captured_output.out)) + if captured_output.err: + result.sections.append(("Captured stderr call", captured_output.err)) + + if captured_logs and (log := captured_logs.handler.stream.getvalue()): + result.sections.append(("Captured log call", log)) + + return result + + +@fixture +def subtests(request: SubRequest) -> Subtests: + """Provides subtests functionality.""" + capmam = request.node.config.pluginmanager.get_plugin("capturemanager") + suspend_capture_ctx = ( + capmam.global_and_fixture_disabled if capmam is not None else nullcontext + ) + return Subtests(request.node.ihook, suspend_capture_ctx, request, _ispytest=True) + + +class Subtests: + """Subtests fixture, enables declaring subtests inside test functions via the :meth:`test` method.""" + + def __init__( + self, + ihook: pluggy.HookRelay, + suspend_capture_ctx: Callable[[], AbstractContextManager[None]], + request: SubRequest, + *, + _ispytest: bool = False, + ) -> None: + check_ispytest(_ispytest) + self._ihook = ihook + self._suspend_capture_ctx = suspend_capture_ctx + self._request = request + + def test( + self, + msg: str | None = None, + **kwargs: Any, + ) -> _SubTestContextManager: + """ + Context manager for subtests, capturing exceptions raised inside the subtest scope and + reporting assertion failures and errors individually. + + Usage + ----- + + .. code-block:: python + + def test(subtests): + for i in range(5): + with subtests.test("custom message", i=i): + assert i % 2 == 0 + + :param msg: + If given, the message will be shown in the test report in case of subtest failure. + + :param kwargs: + Arbitrary values that are also added to the subtest report. + """ + return _SubTestContextManager( + self._ihook, + msg, + kwargs, + request=self._request, + suspend_capture_ctx=self._suspend_capture_ctx, + config=self._request.config, + ) + + +@dataclasses.dataclass +class _SubTestContextManager: + """ + Context manager for subtests, capturing exceptions raised inside the subtest scope and handling + them through the pytest machinery. + """ + + # Note: initially the logic for this context manager was implemented directly + # in Subtests.test() as a @contextmanager, however, it is not possible to control the output fully when + # exiting from it due to an exception when in `--exitfirst` mode, so this was refactored into an + # explicit context manager class (pytest-dev/pytest-subtests#134). + + ihook: pluggy.HookRelay + msg: str | None + kwargs: dict[str, Any] + suspend_capture_ctx: Callable[[], AbstractContextManager[None]] + request: SubRequest + config: Config + + def __enter__(self) -> None: + __tracebackhide__ = True + + self._start = time.time() + self._precise_start = time.perf_counter() + self._exc_info = None + + self._exit_stack = ExitStack() + self._captured_output = self._exit_stack.enter_context( + capturing_output(self.request) + ) + self._captured_logs = self._exit_stack.enter_context( + capturing_logs(self.request) + ) + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> bool: + __tracebackhide__ = True + if exc_val is not None: + exc_info = ExceptionInfo.from_exception(exc_val) + else: + exc_info = None + + self._exit_stack.close() + + precise_stop = time.perf_counter() + duration = precise_stop - self._precise_start + stop = time.time() + + call_info = CallInfo[None]( + None, + exc_info, + start=self._start, + stop=stop, + duration=duration, + when="call", + _ispytest=True, + ) + report = self.ihook.pytest_runtest_makereport( + item=self.request.node, call=call_info + ) + sub_report = SubtestReport._new( + report, + SubtestContext(msg=self.msg, kwargs=self.kwargs), + captured_output=self._captured_output, + captured_logs=self._captured_logs, + ) + + if sub_report.failed: + failed_subtests = self.config.stash[failed_subtests_key] + failed_subtests[self.request.node.nodeid] += 1 + + with self.suspend_capture_ctx(): + self.ihook.pytest_runtest_logreport(report=sub_report) + + if check_interactive_exception(call_info, sub_report): + self.ihook.pytest_exception_interact( + node=self.request.node, call=call_info, report=sub_report + ) + + if exc_val is not None: + if isinstance(exc_val, get_reraise_exceptions(self.config)): + return False + if self.request.session.shouldfail: + return False + return True + + +@contextmanager +def capturing_output(request: SubRequest) -> Iterator[Captured]: + option = request.config.getoption("capture", None) + + capman = request.config.pluginmanager.getplugin("capturemanager") + if getattr(capman, "_capture_fixture", None): + # capsys or capfd are active, subtest should not capture. + fixture = None + elif option == "sys": + fixture = CaptureFixture(SysCapture, request, _ispytest=True) + elif option == "fd": + fixture = CaptureFixture(FDCapture, request, _ispytest=True) + else: + fixture = None + + if fixture is not None: + fixture._start() + + captured = Captured() + try: + yield captured + finally: + if fixture is not None: + out, err = fixture.readouterr() + fixture.close() + captured.out = out + captured.err = err + + +@contextmanager +def capturing_logs( + request: SubRequest, +) -> Iterator[CapturedLogs | None]: + logging_plugin: LoggingPlugin | None = request.config.pluginmanager.getplugin( + "logging-plugin" + ) + if logging_plugin is None: + yield None + else: + handler = LogCaptureHandler() + handler.setFormatter(logging_plugin.formatter) + + captured_logs = CapturedLogs(handler) + with catching_logs(handler, level=logging_plugin.log_level): + yield captured_logs + + +@dataclasses.dataclass +class Captured: + out: str = "" + err: str = "" + + +@dataclasses.dataclass +class CapturedLogs: + handler: LogCaptureHandler + + +def pytest_report_to_serializable(report: TestReport) -> dict[str, Any] | None: + if isinstance(report, SubtestReport): + return report._to_json() + return None + + +def pytest_report_from_serializable(data: dict[str, Any]) -> SubtestReport | None: + if data.get("_report_type") == "SubTestReport": + return SubtestReport._from_json(data) + return None + + +# Dict of nodeid -> number of failed subtests. +# Used to fail top-level tests that passed but contain failed subtests. +failed_subtests_key = StashKey[defaultdict[str, int]]() + + +def pytest_configure(config: Config) -> None: + config.stash[failed_subtests_key] = defaultdict(lambda: 0) + + +@hookimpl(tryfirst=True) +def pytest_report_teststatus( + report: TestReport, + config: Config, +) -> tuple[str, str, str | Mapping[str, bool]] | None: + if report.when != "call": + return None + + quiet = config.get_verbosity(Config.VERBOSITY_SUBTESTS) == 0 + if isinstance(report, SubtestReport): + outcome = report.outcome + description = report._sub_test_description() + + if hasattr(report, "wasxfail"): + if quiet: + return "", "", "" + elif outcome == "skipped": + category = "xfailed" + short = "y" # x letter is used for regular xfail, y for subtest xfail + status = "SUBXFAIL" + # outcome == "passed" in an xfail is only possible via a @pytest.mark.xfail mark, which + # is not applicable to a subtest, which only handles pytest.xfail(). + else: # pragma: no cover + # This should not normally happen, unless some plugin is setting wasxfail without + # the correct outcome. Pytest expects the call outcome to be either skipped or + # passed in case of xfail. + # Let's pass this report to the next hook. + return None + return category, short, f"{status}{description}" + + if report.failed: + return outcome, "u", f"SUBFAILED{description}" + else: + if report.passed: + if quiet: + return "", "", "" + else: + return f"subtests {outcome}", "u", f"SUBPASSED{description}" + elif report.skipped: + if quiet: + return "", "", "" + else: + return outcome, "-", f"SUBSKIPPED{description}" + + else: + failed_subtests_count = config.stash[failed_subtests_key][report.nodeid] + # Top-level test, fail if it contains failed subtests and it has passed. + if report.passed and failed_subtests_count > 0: + report.outcome = "failed" + suffix = "s" if failed_subtests_count > 1 else "" + report.longrepr = f"contains {failed_subtests_count} failed subtest{suffix}" + + return None diff --git a/src/_pytest/terminal.py b/src/_pytest/terminal.py index 8c722124d04..e66e4f48dd6 100644 --- a/src/_pytest/terminal.py +++ b/src/_pytest/terminal.py @@ -8,6 +8,10 @@ import argparse from collections import Counter +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Mapping +from collections.abc import Sequence import dataclasses import datetime from functools import partial @@ -17,20 +21,17 @@ import sys import textwrap from typing import Any -from typing import Callable from typing import ClassVar from typing import final -from typing import Generator from typing import Literal -from typing import Mapping from typing import NamedTuple -from typing import Sequence from typing import TextIO from typing import TYPE_CHECKING import warnings import pluggy +from _pytest import compat from _pytest import nodes from _pytest import timing from _pytest._code import ExceptionInfo @@ -38,7 +39,7 @@ from _pytest._io import TerminalWriter from _pytest._io.wcwidth import wcswidth import _pytest._version -from _pytest.assertion.util import running_on_ci +from _pytest.compat import running_on_ci from _pytest.config import _PluggyPlugin from _pytest.config import Config from _pytest.config import ExitCode @@ -68,6 +69,9 @@ "xpassed", "warnings", "error", + "subtests passed", + "subtests failed", + "subtests skipped", ) _REPORTCHARS_DEFAULT = "fE" @@ -132,7 +136,7 @@ class TestShortLogReport(NamedTuple): def pytest_addoption(parser: Parser) -> None: group = parser.getgroup("terminal reporting", "Reporting", after="general") - group._addoption( + group._addoption( # private to use reserved lower-case short option "-v", "--verbose", action="count", @@ -140,28 +144,35 @@ def pytest_addoption(parser: Parser) -> None: dest="verbose", help="Increase verbosity", ) - group._addoption( + group.addoption( "--no-header", action="store_true", default=False, dest="no_header", help="Disable header", ) - group._addoption( + group.addoption( "--no-summary", action="store_true", default=False, dest="no_summary", help="Disable summary", ) - group._addoption( + group.addoption( "--no-fold-skipped", action="store_false", dest="fold_skipped", default=True, help="Do not fold skipped tests in short summary.", ) - group._addoption( + group.addoption( + "--force-short-summary", + action="store_true", + dest="force_short_summary", + default=False, + help="Force condensed summary output regardless of verbosity level.", + ) + group._addoption( # private to use reserved lower-case short option "-q", "--quiet", action=MoreQuietAction, @@ -169,14 +180,14 @@ def pytest_addoption(parser: Parser) -> None: dest="verbose", help="Decrease verbosity", ) - group._addoption( + group.addoption( "--verbosity", dest="verbose", type=int, default=0, help="Set verbosity. Default: 0.", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-r", action="store", dest="reportchars", @@ -188,7 +199,7 @@ def pytest_addoption(parser: Parser) -> None: "(w)arnings are enabled by default (see --disable-warnings), " "'N' can be used to reset the list. (default: 'fE').", ) - group._addoption( + group.addoption( "--disable-warnings", "--disable-pytest-warnings", default=False, @@ -196,7 +207,7 @@ def pytest_addoption(parser: Parser) -> None: action="store_true", help="Disable warnings summary", ) - group._addoption( + group._addoption( # private to use reserved lower-case short option "-l", "--showlocals", action="store_true", @@ -204,13 +215,13 @@ def pytest_addoption(parser: Parser) -> None: default=False, help="Show locals in tracebacks (disabled by default)", ) - group._addoption( + group.addoption( "--no-showlocals", action="store_false", dest="showlocals", help="Hide locals in tracebacks (negate --showlocals passed through addopts)", ) - group._addoption( + group.addoption( "--tb", metavar="style", action="store", @@ -219,14 +230,14 @@ def pytest_addoption(parser: Parser) -> None: choices=["auto", "long", "short", "no", "line", "native"], help="Traceback print mode (auto/long/short/line/native/no)", ) - group._addoption( + group.addoption( "--xfail-tb", action="store_true", dest="xfail_tb", default=False, help="Show tracebacks for xfail (as long as --tb != no)", ) - group._addoption( + group.addoption( "--show-capture", action="store", dest="showcapture", @@ -235,14 +246,14 @@ def pytest_addoption(parser: Parser) -> None: help="Controls how captured stdout/stderr/log is shown on failed tests. " "Default: all.", ) - group._addoption( + group.addoption( "--fulltrace", "--full-trace", action="store_true", default=False, help="Don't cut any tracebacks (default is to cut)", ) - group._addoption( + group.addoption( "--color", metavar="color", action="store", @@ -251,7 +262,7 @@ def pytest_addoption(parser: Parser) -> None: choices=["yes", "no", "auto"], help="Color terminal output (yes/no/auto)", ) - group._addoption( + group.addoption( "--code-highlight", default="yes", choices=["yes", "no"], @@ -287,6 +298,11 @@ def mywriter(tags, args): config.trace.root.setprocessor("pytest:config", mywriter) + # See terminalprogress.py. + # On Windows it's safe to load by default. + if sys.platform == "win32": + config.pluginmanager.import_plugin("terminalprogress") + def getreportopt(config: Config) -> str: reportchars: str = config.option.reportchars @@ -380,14 +396,19 @@ def __init__(self, config: Config, file: TextIO | None = None) -> None: self.reportchars = getreportopt(config) self.foldskipped = config.option.fold_skipped self.hasmarkup = self._tw.hasmarkup - self.isatty = file.isatty() + # isatty should be a method but was wrongly implemented as a boolean. + # We use CallableBool here to support both. + self.isatty = compat.CallableBool(file.isatty()) self._progress_nodeids_reported: set[str] = set() + self._timing_nodeids_reported: set[str] = set() self._show_progress_info = self._determine_show_progress_info() - self._collect_report_last_write: float | None = None + self._collect_report_last_write = timing.Instant() self._already_displayed_warnings: int | None = None self._keyboardinterrupt_memo: ExceptionRepr | None = None - def _determine_show_progress_info(self) -> Literal["progress", "count", False]: + def _determine_show_progress_info( + self, + ) -> Literal["progress", "count", "times", False]: """Return whether we should display progress information based on the current config.""" # do not show progress if we are not capturing output (#3038) unless explicitly # overridden by progress-even-when-capture-no @@ -405,6 +426,8 @@ def _determine_show_progress_info(self) -> Literal["progress", "count", False]: return "progress" elif cfg == "count": return "count" + elif cfg == "times": + return "times" else: return False @@ -439,6 +462,14 @@ def showfspath(self, value: bool | None) -> None: def showlongtestinfo(self) -> bool: return self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) > 0 + @property + def reported_progress(self) -> int: + """The amount of items reported in the progress so far. + + :meta private: + """ + return len(self._progress_nodeids_reported) + def hasopt(self, char: str) -> bool: char = {"xfailed": "x", "skipped": "s"}.get(char, char) return char in self.reportchars @@ -493,6 +524,9 @@ def wrap_write( def write(self, content: str, *, flush: bool = False, **markup: bool) -> None: self._tw.write(content, flush=flush, **markup) + def write_raw(self, content: str, *, flush: bool = False) -> None: + self._tw.write_raw(content, flush=flush) + def flush(self) -> None: self._tw.flush() @@ -666,7 +700,7 @@ def pytest_runtest_logreport(self, report: TestReport) -> None: @property def _is_last_item(self) -> bool: assert self._session is not None - return len(self._progress_nodeids_reported) == self._session.testscollected + return self.reported_progress == self._session.testscollected @hookimpl(wrapper=True) def pytest_runtestloop(self) -> Generator[None, object, object]: @@ -676,7 +710,7 @@ def pytest_runtestloop(self) -> Generator[None, object, object]: if ( self.config.get_verbosity(Config.VERBOSITY_TEST_CASES) <= 0 and self._show_progress_info - and self._progress_nodeids_reported + and self.reported_progress ): self._write_progress_information_filling_space() @@ -687,17 +721,45 @@ def _get_progress_information_message(self) -> str: collected = self._session.testscollected if self._show_progress_info == "count": if collected: - progress = len(self._progress_nodeids_reported) + progress = self.reported_progress counter_format = f"{{:{len(str(collected))}d}}" format_string = f" [{counter_format}/{{}}]" return format_string.format(progress, collected) return f" [ {collected} / {collected} ]" - else: - if collected: - return ( - f" [{len(self._progress_nodeids_reported) * 100 // collected:3d}%]" + if self._show_progress_info == "times": + if not collected: + return "" + all_reports = ( + self._get_reports_to_display("passed") + + self._get_reports_to_display("xpassed") + + self._get_reports_to_display("failed") + + self._get_reports_to_display("xfailed") + + self._get_reports_to_display("skipped") + + self._get_reports_to_display("error") + + self._get_reports_to_display("") + ) + current_location = all_reports[-1].location[0] + not_reported = [ + r for r in all_reports if r.nodeid not in self._timing_nodeids_reported + ] + tests_in_module = sum( + i.location[0] == current_location for i in self._session.items + ) + tests_completed = sum( + r.when == "setup" + for r in not_reported + if r.location[0] == current_location + ) + last_in_module = tests_completed == tests_in_module + if self.showlongtestinfo or last_in_module: + self._timing_nodeids_reported.update(r.nodeid for r in not_reported) + return format_node_duration( + sum(r.duration for r in not_reported if isinstance(r, TestReport)) ) - return " [100%]" + return "" + if collected: + return f" [{self.reported_progress * 100 // collected:3d}%]" + return " [100%]" def _write_progress_information_if_past_edge(self) -> None: w = self._width_of_current_line @@ -705,6 +767,8 @@ def _write_progress_information_if_past_edge(self) -> None: assert self._session num_tests = self._session.testscollected progress_length = len(f" [{num_tests}/{num_tests}]") + elif self._show_progress_info == "times": + progress_length = len(" 99h 59m") else: progress_length = len(" [100%]") past_edge = w + progress_length + 1 >= self._screen_width @@ -726,10 +790,9 @@ def _width_of_current_line(self) -> int: return self._tw.width_of_current_line def pytest_collection(self) -> None: - if self.isatty: + if self.isatty(): if self.config.option.verbose >= 0: self.write("collecting ... ", flush=True, bold=True) - self._collect_report_last_write = timing.time() elif self.config.option.verbose >= 1: self.write("collecting ... ", flush=True, bold=True) @@ -740,7 +803,7 @@ def pytest_collectreport(self, report: CollectReport) -> None: self._add_stats("skipped", [report]) items = [x for x in report.result if isinstance(x, Item)] self._numcollected += len(items) - if self.isatty: + if self.isatty(): self.report_collect() def report_collect(self, final: bool = False) -> None: @@ -748,14 +811,13 @@ def report_collect(self, final: bool = False) -> None: return if not final: - # Only write "collecting" report every 0.5s. - t = timing.time() + # Only write the "collecting" report every `REPORT_COLLECTING_RESOLUTION`. if ( - self._collect_report_last_write is not None - and self._collect_report_last_write > t - REPORT_COLLECTING_RESOLUTION + self._collect_report_last_write.elapsed().seconds + < REPORT_COLLECTING_RESOLUTION ): return - self._collect_report_last_write = t + self._collect_report_last_write = timing.Instant() errors = len(self.stats.get("error", [])) skipped = len(self.stats.get("skipped", [])) @@ -766,14 +828,14 @@ def report_collect(self, final: bool = False) -> None: str(self._numcollected) + " item" + ("" if self._numcollected == 1 else "s") ) if errors: - line += " / %d error%s" % (errors, "s" if errors != 1 else "") + line += f" / {errors} error{'s' if errors != 1 else ''}" if deselected: - line += " / %d deselected" % deselected + line += f" / {deselected} deselected" if skipped: - line += " / %d skipped" % skipped + line += f" / {skipped} skipped" if self._numcollected > selected: - line += " / %d selected" % selected - if self.isatty: + line += f" / {selected} selected" + if self.isatty(): self.rewrite(line, bold=True, erase=True) if final: self.write("\n") @@ -783,7 +845,7 @@ def report_collect(self, final: bool = False) -> None: @hookimpl(trylast=True) def pytest_sessionstart(self, session: Session) -> None: self._session = session - self._sessionstarttime = timing.time() + self._session_start = timing.Instant() if not self.showheader: return self.write_sep("=", "test session starts", bold=True) @@ -821,7 +883,12 @@ def pytest_report_header(self, config: Config) -> list[str]: result = [f"rootdir: {config.rootpath}"] if config.inipath: - result.append("configfile: " + bestrelpath(config.rootpath, config.inipath)) + warning = "" + if config._ignored_config_files: + warning = f" (WARNING: ignoring pytest config in {', '.join(config._ignored_config_files)}!)" + result.append( + "configfile: " + bestrelpath(config.rootpath, config.inipath) + warning + ) if config.args_source == Config.ArgsSource.TESTPATHS: testpaths: list[str] = config.getini("testpaths") @@ -862,7 +929,7 @@ def _printcollecteditems(self, items: Sequence[Item]) -> None: if test_cases_verbosity < -1: counts = Counter(item.nodeid.split("::", 1)[0] for item in items) for name, count in sorted(counts.items()): - self._tw.line("%s: %d" % (name, count)) + self._tw.line(f"{name}: {count}") else: for item in items: self._tw.line(item.nodeid) @@ -889,7 +956,7 @@ def _printcollecteditems(self, items: Sequence[Item]) -> None: @hookimpl(wrapper=True) def pytest_sessionfinish( self, session: Session, exitstatus: int | ExitCode - ) -> Generator[None, None, None]: + ) -> Generator[None]: result = yield self._tw.line("") summary_exit_codes = ( @@ -914,7 +981,7 @@ def pytest_sessionfinish( return result @hookimpl(wrapper=True) - def pytest_terminal_summary(self) -> Generator[None, None, None]: + def pytest_terminal_summary(self) -> Generator[None]: self.summary_errors() self.summary_failures() self.summary_xfailures() @@ -1122,6 +1189,7 @@ def summary_failures_combined( if style == "line": for rep in reports: line = self._getcrashline(rep) + self._outrep_summary(rep) self.write_line(line) else: for rep in reports: @@ -1162,7 +1230,7 @@ def summary_stats(self) -> None: if self.verbosity < -1: return - session_duration = timing.time() - self._sessionstarttime + session_duration = self._session_start.elapsed() (parts, main_color) = self.build_summary_stats_line() line_parts = [] @@ -1177,7 +1245,7 @@ def summary_stats(self) -> None: msg = ", ".join(line_parts) main_markup = {main_color: True} - duration = f" in {format_session_duration(session_duration)}" + duration = f" in {format_session_duration(session_duration.seconds)}" duration_with_markup = self._tw.markup(duration, **main_markup) if display_sep: fullwidth += len(duration_with_markup) - len(duration) @@ -1254,11 +1322,9 @@ def show_skipped_folded(lines: list[str]) -> None: if reason.startswith(prefix): reason = reason[len(prefix) :] if lineno is not None: - lines.append( - "%s [%d] %s:%d: %s" % (markup_word, num, fspath, lineno, reason) - ) + lines.append(f"{markup_word} [{num}] {fspath}:{lineno}: {reason}") else: - lines.append("%s [%d] %s: %s" % (markup_word, num, fspath, reason)) + lines.append(f"{markup_word} [{num}] {fspath}: {reason}") def show_skipped_unfolded(lines: list[str]) -> None: skipped: list[CollectReport] = self.stats.get("skipped", []) @@ -1340,7 +1406,7 @@ def build_summary_stats_line(self) -> tuple[list[tuple[str, dict[str, bool]]], s The summary stats line is the line shown at the end, "=== 12 passed, 2 errors in Xs===". This function builds a list of the "parts" that make up for the text in that line, in - the example above it would be: + the example above it would be:: [ ("12 passed", {"green": True}), @@ -1375,7 +1441,7 @@ def _build_normal_summary_stats_line( count = len(reports) color = _color_for_type.get(key, _color_for_type_default) markup = {color: True, "bold": color == main_color} - parts.append(("%d %s" % pluralize(count, key), markup)) + parts.append(("%d %s" % pluralize(count, key), markup)) # noqa: UP031 if not parts: parts = [("no tests ran", {_color_for_type_default: True})] @@ -1394,7 +1460,7 @@ def _build_collect_only_summary_stats_line( elif deselected == 0: main_color = "green" - collected_output = "%d %s collected" % pluralize(self._numcollected, "test") + collected_output = "%d %s collected" % pluralize(self._numcollected, "test") # noqa: UP031 parts = [(collected_output, {main_color: True})] else: all_tests_were_deselected = self._numcollected == deselected @@ -1410,7 +1476,7 @@ def _build_collect_only_summary_stats_line( if errors: main_color = _color_for_type["error"] - parts += [("%d %s" % pluralize(errors, "error"), {main_color: True})] + parts += [("%d %s" % pluralize(errors, "error"), {main_color: True})] # noqa: UP031 return parts, main_color @@ -1463,13 +1529,19 @@ def _get_line_with_reprcrash_message( line = f"{word} {node}" line_width = wcswidth(line) + msg: str | None try: - # Type ignored intentionally -- possible AttributeError expected. - msg = rep.longrepr.reprcrash.message # type: ignore[union-attr] + if isinstance(rep.longrepr, str): + msg = rep.longrepr + else: + # Type ignored intentionally -- possible AttributeError expected. + msg = rep.longrepr.reprcrash.message # type: ignore[union-attr] except AttributeError: pass else: - if running_on_ci() or config.option.verbose >= 2: + if ( + running_on_ci() or config.option.verbose >= 2 + ) and not config.option.force_short_summary: msg = f" - {msg}" else: available_width = tw.fullwidth - line_width @@ -1516,6 +1588,8 @@ def _folded_skips( "error": "red", "warnings": "yellow", "passed": "green", + "subtests passed": "green", + "subtests failed": "red", } _color_for_type_default = "yellow" @@ -1556,6 +1630,29 @@ def format_session_duration(seconds: float) -> str: return f"{seconds:.2f}s ({dt})" +def format_node_duration(seconds: float) -> str: + """Format the given seconds in a human readable manner to show in the test progress.""" + # The formatting is designed to be compact and readable, with at most 7 characters + # for durations below 100 hours. + if seconds < 0.00001: + return f" {seconds * 1000000:.3f}us" + if seconds < 0.0001: + return f" {seconds * 1000000:.2f}us" + if seconds < 0.001: + return f" {seconds * 1000000:.1f}us" + if seconds < 0.01: + return f" {seconds * 1000:.3f}ms" + if seconds < 0.1: + return f" {seconds * 1000:.2f}ms" + if seconds < 1: + return f" {seconds * 1000:.1f}ms" + if seconds < 60: + return f" {seconds:.3f}s" + if seconds < 3600: + return f" {seconds // 60:.0f}m {seconds % 60:.0f}s" + return f" {seconds // 3600:.0f}h {(seconds % 3600) // 60:.0f}m" + + def _get_raw_skip_reason(report: TestReport) -> str: """Get the reason string of a skip/xfail/xpass test report. @@ -1575,3 +1672,92 @@ def _get_raw_skip_reason(report: TestReport) -> str: elif reason == "Skipped": reason = "" return reason + + +class TerminalProgressPlugin: + """Terminal progress reporting plugin using OSC 9;4 ANSI sequences. + + Emits OSC 9;4 sequences to indicate test progress to terminal + tabs/windows/etc. + + Not all terminal emulators support this feature. + + Ref: https://conemu.github.io/en/AnsiEscapeCodes.html#ConEmu_specific_OSC + """ + + def __init__(self, tr: TerminalReporter) -> None: + self._tr = tr + self._session: Session | None = None + self._has_failures = False + + def _emit_progress( + self, + state: Literal["remove", "normal", "error", "indeterminate", "paused"], + progress: int | None = None, + ) -> None: + """Emit OSC 9;4 sequence for indicating progress to the terminal. + + :param state: + Progress state to set. + :param progress: + Progress value 0-100. Required for "normal", optional for "error" + and "paused", otherwise ignored. + """ + assert progress is None or 0 <= progress <= 100 + + # OSC 9;4 sequence: ESC ] 9 ; 4 ; state ; progress ST + # ST can be ESC \ or BEL. ESC \ seems better supported. + match state: + case "remove": + sequence = "\x1b]9;4;0;\x1b\\" + case "normal": + assert progress is not None + sequence = f"\x1b]9;4;1;{progress}\x1b\\" + case "error": + if progress is not None: + sequence = f"\x1b]9;4;2;{progress}\x1b\\" + else: + sequence = "\x1b]9;4;2;\x1b\\" + case "indeterminate": + sequence = "\x1b]9;4;3;\x1b\\" + case "paused": + if progress is not None: + sequence = f"\x1b]9;4;4;{progress}\x1b\\" + else: + sequence = "\x1b]9;4;4;\x1b\\" + + self._tr.write_raw(sequence, flush=True) + + @hookimpl + def pytest_sessionstart(self, session: Session) -> None: + self._session = session + # Show indeterminate progress during collection. + self._emit_progress("indeterminate") + + @hookimpl + def pytest_collection_finish(self) -> None: + assert self._session is not None + if self._session.testscollected > 0: + # Switch from indeterminate to 0% progress. + self._emit_progress("normal", 0) + + @hookimpl + def pytest_runtest_logreport(self, report: TestReport) -> None: + if report.failed: + self._has_failures = True + + # Let's consider the "call" phase for progress. + if report.when != "call": + return + + # Calculate and emit progress. + assert self._session is not None + collected = self._session.testscollected + if collected > 0: + reported = self._tr.reported_progress + progress = min(reported * 100 // collected, 100) + self._emit_progress("error" if self._has_failures else "normal", progress) + + @hookimpl + def pytest_sessionfinish(self) -> None: + self._emit_progress("remove") diff --git a/src/_pytest/terminalprogress.py b/src/_pytest/terminalprogress.py new file mode 100644 index 00000000000..287f0d569ff --- /dev/null +++ b/src/_pytest/terminalprogress.py @@ -0,0 +1,30 @@ +# A plugin to register the TerminalProgressPlugin plugin. +# +# This plugin is not loaded by default due to compatibility issues (#13896), +# but can be enabled in one of these ways: +# - The terminal plugin enables it in a few cases where it's safe, and not +# blocked by the user (using e.g. `-p no:terminalprogress`). +# - The user explicitly requests it, e.g. using `-p terminalprogress`. +# +# In a few years, if it's safe, we can consider enabling it by default. Then, +# this file will become unnecessary and can be inlined into terminal.py. + +from __future__ import annotations + +import os + +from _pytest.config import Config +from _pytest.config import hookimpl +from _pytest.terminal import TerminalProgressPlugin +from _pytest.terminal import TerminalReporter + + +@hookimpl(trylast=True) +def pytest_configure(config: Config) -> None: + reporter: TerminalReporter | None = config.pluginmanager.get_plugin( + "terminalreporter" + ) + + if reporter is not None and reporter.isatty() and os.environ.get("TERM") != "dumb": + plugin = TerminalProgressPlugin(reporter) + config.pluginmanager.register(plugin, name="terminalprogress-plugin") diff --git a/src/_pytest/threadexception.py b/src/_pytest/threadexception.py index d78c32c852f..eb57783be26 100644 --- a/src/_pytest/threadexception.py +++ b/src/_pytest/threadexception.py @@ -1,97 +1,152 @@ from __future__ import annotations +import collections +from collections.abc import Callable +import functools +import sys import threading import traceback -from types import TracebackType -from typing import Any -from typing import Callable -from typing import Generator +from typing import NamedTuple from typing import TYPE_CHECKING import warnings +from _pytest.config import Config +from _pytest.nodes import Item +from _pytest.stash import StashKey +from _pytest.tracemalloc import tracemalloc_message import pytest if TYPE_CHECKING: - from typing_extensions import Self - - -# Copied from cpython/Lib/test/support/threading_helper.py, with modifications. -class catch_threading_exception: - """Context manager catching threading.Thread exception using - threading.excepthook. - - Storing exc_value using a custom hook can create a reference cycle. The - reference cycle is broken explicitly when the context manager exits. - - Storing thread using a custom hook can resurrect it if it is set to an - object which is being finalized. Exiting the context manager clears the - stored object. - - Usage: - with threading_helper.catch_threading_exception() as cm: - # code spawning a thread which raises an exception - ... - # check the thread exception: use cm.args - ... - # cm.args attribute no longer exists at this point - # (to break a reference cycle) - """ - - def __init__(self) -> None: - self.args: threading.ExceptHookArgs | None = None - self._old_hook: Callable[[threading.ExceptHookArgs], Any] | None = None - - def _hook(self, args: threading.ExceptHookArgs) -> None: - self.args = args - - def __enter__(self) -> Self: - self._old_hook = threading.excepthook - threading.excepthook = self._hook - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - assert self._old_hook is not None - threading.excepthook = self._old_hook - self._old_hook = None - del self.args - - -def thread_exception_runtest_hook() -> Generator[None, None, None]: - with catch_threading_exception() as cm: - try: - yield - finally: - if cm.args: - thread_name = ( - "" if cm.args.thread is None else cm.args.thread.name - ) - msg = f"Exception in thread {thread_name}\n\n" - msg += "".join( - traceback.format_exception( - cm.args.exc_type, - cm.args.exc_value, - cm.args.exc_traceback, - ) - ) - warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg)) + pass + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + +class ThreadExceptionMeta(NamedTuple): + msg: str + cause_msg: str + exc_value: BaseException | None -@pytest.hookimpl(wrapper=True, trylast=True) -def pytest_runtest_setup() -> Generator[None, None, None]: - yield from thread_exception_runtest_hook() +thread_exceptions: StashKey[collections.deque[ThreadExceptionMeta | BaseException]] = ( + StashKey() +) -@pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_runtest_call() -> Generator[None, None, None]: - yield from thread_exception_runtest_hook() +def collect_thread_exception(config: Config) -> None: + pop_thread_exception = config.stash[thread_exceptions].pop + errors: list[pytest.PytestUnhandledThreadExceptionWarning | RuntimeError] = [] + meta = None + hook_error = None + try: + while True: + try: + meta = pop_thread_exception() + except IndexError: + break -@pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_runtest_teardown() -> Generator[None, None, None]: - yield from thread_exception_runtest_hook() + if isinstance(meta, BaseException): + hook_error = RuntimeError("Failed to process thread exception") + hook_error.__cause__ = meta + errors.append(hook_error) + continue + + msg = meta.msg + try: + warnings.warn(pytest.PytestUnhandledThreadExceptionWarning(msg)) + except pytest.PytestUnhandledThreadExceptionWarning as e: + # This except happens when the warning is treated as an error (e.g. `-Werror`). + if meta.exc_value is not None: + # Exceptions have a better way to show the traceback, but + # warnings do not, so hide the traceback from the msg and + # set the cause so the traceback shows up in the right place. + e.args = (meta.cause_msg,) + e.__cause__ = meta.exc_value + errors.append(e) + + if len(errors) == 1: + raise errors[0] + if errors: + raise ExceptionGroup("multiple thread exception warnings", errors) + finally: + del errors, meta, hook_error + + +def cleanup( + *, config: Config, prev_hook: Callable[[threading.ExceptHookArgs], object] +) -> None: + try: + try: + # We don't join threads here, so exceptions raised from any + # threads still running by the time _threading_atexits joins them + # do not get captured (see #13027). + collect_thread_exception(config) + finally: + threading.excepthook = prev_hook + finally: + del config.stash[thread_exceptions] + + +def thread_exception_hook( + args: threading.ExceptHookArgs, + /, + *, + append: Callable[[ThreadExceptionMeta | BaseException], object], +) -> None: + try: + # we need to compute these strings here as they might change after + # the excepthook finishes and before the metadata object is + # collected by a pytest hook + thread_name = "" if args.thread is None else args.thread.name + summary = f"Exception in thread {thread_name}" + traceback_message = "\n\n" + "".join( + traceback.format_exception( + args.exc_type, + args.exc_value, + args.exc_traceback, + ) + ) + tracemalloc_tb = "\n" + tracemalloc_message(args.thread) + msg = summary + traceback_message + tracemalloc_tb + cause_msg = summary + tracemalloc_tb + + append( + ThreadExceptionMeta( + # Compute these strings here as they might change later + msg=msg, + cause_msg=cause_msg, + exc_value=args.exc_value, + ) + ) + except BaseException as e: + append(e) + # Raising this will cause the exception to be logged twice, once in our + # collect_thread_exception and once by sys.excepthook + # which is fine - this should never happen anyway and if it does + # it should probably be reported as a pytest bug. + raise + + +def pytest_configure(config: Config) -> None: + prev_hook = threading.excepthook + deque: collections.deque[ThreadExceptionMeta | BaseException] = collections.deque() + config.stash[thread_exceptions] = deque + config.add_cleanup(functools.partial(cleanup, config=config, prev_hook=prev_hook)) + threading.excepthook = functools.partial(thread_exception_hook, append=deque.append) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup(item: Item) -> None: + collect_thread_exception(item.config) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_call(item: Item) -> None: + collect_thread_exception(item.config) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item: Item) -> None: + collect_thread_exception(item.config) diff --git a/src/_pytest/timing.py b/src/_pytest/timing.py index b23c7f69e2d..51c3db23f6f 100644 --- a/src/_pytest/timing.py +++ b/src/_pytest/timing.py @@ -8,9 +8,88 @@ from __future__ import annotations +import dataclasses +from datetime import datetime +from datetime import timezone from time import perf_counter from time import sleep from time import time +from typing import TYPE_CHECKING + + +if TYPE_CHECKING: + from pytest import MonkeyPatch + + +@dataclasses.dataclass(frozen=True) +class Instant: + """ + Represents an instant in time, used to both get the timestamp value and to measure + the duration of a time span. + + Inspired by Rust's `std::time::Instant`. + """ + + # Creation time of this instant, using time.time(), to measure actual time. + # Note: using a `lambda` to correctly get the mocked time via `MockTiming`. + time: float = dataclasses.field(default_factory=lambda: time(), init=False) + + # Performance counter tick of the instant, used to measure precise elapsed time. + # Note: using a `lambda` to correctly get the mocked time via `MockTiming`. + perf_count: float = dataclasses.field( + default_factory=lambda: perf_counter(), init=False + ) + + def elapsed(self) -> Duration: + """Measure the duration since `Instant` was created.""" + return Duration(start=self, stop=Instant()) + + def as_utc(self) -> datetime: + """Instant as UTC datetime.""" + return datetime.fromtimestamp(self.time, timezone.utc) + + +@dataclasses.dataclass(frozen=True) +class Duration: + """A span of time as measured by `Instant.elapsed()`.""" + + start: Instant + stop: Instant + + @property + def seconds(self) -> float: + """Elapsed time of the duration in seconds, measured using a performance counter for precise timing.""" + return self.stop.perf_count - self.start.perf_count + + +@dataclasses.dataclass +class MockTiming: + """Mocks _pytest.timing with a known object that can be used to control timing in tests + deterministically. + + pytest itself should always use functions from `_pytest.timing` instead of `time` directly. + + This then allows us more control over time during testing, if testing code also + uses `_pytest.timing` functions. + + Time is static, and only advances through `sleep` calls, thus tests might sleep over large + numbers and obtain accurate time() calls at the end, making tests reliable and instant.""" + + _current_time: float = datetime(2020, 5, 22, 14, 20, 50).timestamp() + + def sleep(self, seconds: float) -> None: + self._current_time += seconds + + def time(self) -> float: + return self._current_time + + def patch(self, monkeypatch: MonkeyPatch) -> None: + # pylint: disable-next=import-self + from _pytest import timing # noqa: PLW0406 + + monkeypatch.setattr(timing, "sleep", self.sleep) + monkeypatch.setattr(timing, "time", self.time) + monkeypatch.setattr(timing, "perf_counter", self.time) __all__ = ["perf_counter", "sleep", "time"] diff --git a/src/_pytest/tmpdir.py b/src/_pytest/tmpdir.py index 91109ea69ef..66ca9f190e3 100644 --- a/src/_pytest/tmpdir.py +++ b/src/_pytest/tmpdir.py @@ -3,16 +3,16 @@ from __future__ import annotations +from collections.abc import Generator import dataclasses import os from pathlib import Path import re from shutil import rmtree +import stat import tempfile from typing import Any -from typing import Dict from typing import final -from typing import Generator from typing import Literal from .pathlib import cleanup_dead_symlinks @@ -34,16 +34,15 @@ from _pytest.stash import StashKey -tmppath_result_key = StashKey[Dict[str, bool]]() +tmppath_result_key = StashKey[dict[str, bool]]() RetentionType = Literal["all", "failed", "none"] @final @dataclasses.dataclass class TempPathFactory: - """Factory for temporary directories under the common base temp directory. - - The base directory can be configured using the ``--basetemp`` option. + """Factory for temporary directories under the common base temp directory, + as discussed at :ref:`temporary directory location and retention`. """ _given_basetemp: Path | None @@ -172,16 +171,37 @@ def getbasetemp(self) -> Path: # Also, to keep things private, fixup any world-readable temp # rootdir's permissions. Historically 0o755 was used, so we can't # just error out on this, at least for a while. + # Don't follow symlinks, otherwise we're open to symlink-swapping + # TOCTOU vulnerability. + # This check makes us vulnerable to a DoS - a user can `mkdir + # /tmp/pytest-of-otheruser` and then `otheruser` will fail this + # check. For now we don't consider it a real problem. otheruser can + # change their TMPDIR or --basetemp, and maybe give the prankster a + # good scolding. uid = get_user_id() if uid is not None: - rootdir_stat = rootdir.stat() + stat_follow_symlinks = ( + False if os.stat in os.supports_follow_symlinks else True + ) + rootdir_stat = rootdir.stat(follow_symlinks=stat_follow_symlinks) + if stat.S_ISLNK(rootdir_stat.st_mode): + raise OSError( + f"The temporary directory {rootdir} is a symbolic link. " + "Fix this and try again." + ) if rootdir_stat.st_uid != uid: raise OSError( f"The temporary directory {rootdir} is not owned by the current user. " "Fix this and try again." ) if (rootdir_stat.st_mode & 0o077) != 0: - os.chmod(rootdir, rootdir_stat.st_mode & ~0o077) + chmod_follow_symlinks = ( + False if os.chmod in os.supports_follow_symlinks else True + ) + rootdir.chmod( + rootdir_stat.st_mode & ~0o077, + follow_symlinks=chmod_follow_symlinks, + ) keep = self._retention_count if self._retention_policy == "none": keep = 0 @@ -227,13 +247,16 @@ def pytest_addoption(parser: Parser) -> None: parser.addini( "tmp_path_retention_count", help="How many sessions should we keep the `tmp_path` directories, according to `tmp_path_retention_policy`.", - default=3, + default="3", + # NOTE: Would have been better as an `int` but can't change it now. + type="string", ) parser.addini( "tmp_path_retention_policy", help="Controls which directories created by the `tmp_path` fixture are kept around, based on test outcome. " "(all/failed/none)", + type="string", default="all", ) @@ -256,25 +279,17 @@ def _mk_tmp(request: FixtureRequest, factory: TempPathFactory) -> Path: @fixture def tmp_path( request: FixtureRequest, tmp_path_factory: TempPathFactory -) -> Generator[Path, None, None]: - """Return a temporary directory path object which is unique to each test - function invocation, created as a sub directory of the base temporary - directory. - - By default, a new base temporary directory is created each test session, - and old bases are removed after 3 sessions, to aid in debugging. - This behavior can be configured with :confval:`tmp_path_retention_count` and - :confval:`tmp_path_retention_policy`. - If ``--basetemp`` is used then it is cleared each session. See - :ref:`temporary directory location and retention`. - - The returned object is a :class:`pathlib.Path` object. +) -> Generator[Path]: + """Return a temporary directory (as :class:`pathlib.Path` object) + which is unique to each test function invocation. + The temporary directory is created as a subdirectory + of the base temporary directory, with configurable retention, + as discussed in :ref:`temporary directory location and retention`. """ path = _mk_tmp(request, tmp_path_factory) yield path # Remove the tmpdir if the policy is "failed" and the test passed. - tmp_path_factory: TempPathFactory = request.session.config._tmp_path_factory # type: ignore policy = tmp_path_factory._retention_policy result_dict = request.node.stash[tmppath_result_key] diff --git a/src/_pytest/tracemalloc.py b/src/_pytest/tracemalloc.py new file mode 100644 index 00000000000..5d0b19855c7 --- /dev/null +++ b/src/_pytest/tracemalloc.py @@ -0,0 +1,24 @@ +from __future__ import annotations + + +def tracemalloc_message(source: object) -> str: + if source is None: + return "" + + try: + import tracemalloc + except ImportError: + return "" + + tb = tracemalloc.get_object_traceback(source) + if tb is not None: + formatted_tb = "\n".join(tb.format()) + # Use a leading new line to better separate the (large) output + # from the traceback to the previous warning text. + return f"\nObject allocated at:\n{formatted_tb}" + # No need for a leading new line. + url = "https://docs.pytest.org/en/stable/how-to/capture-warnings.html#resource-warnings" + return ( + "Enable tracemalloc to get traceback where the object was allocated.\n" + f"See {url} for more info." + ) diff --git a/src/_pytest/unittest.py b/src/_pytest/unittest.py index aefea1333d9..31be8847821 100644 --- a/src/_pytest/unittest.py +++ b/src/_pytest/unittest.py @@ -3,23 +3,27 @@ from __future__ import annotations +from collections.abc import Callable +from collections.abc import Generator +from collections.abc import Iterable +from collections.abc import Iterator +from enum import auto +from enum import Enum import inspect import sys import traceback import types from typing import Any -from typing import Callable -from typing import Generator -from typing import Iterable -from typing import Tuple -from typing import Type from typing import TYPE_CHECKING -from typing import Union +from unittest import TestCase import _pytest._code +from _pytest._code import ExceptionInfo +from _pytest.compat import assert_never from _pytest.compat import is_async_function from _pytest.config import hookimpl from _pytest.fixtures import FixtureRequest +from _pytest.monkeypatch import MonkeyPatch from _pytest.nodes import Collector from _pytest.nodes import Item from _pytest.outcomes import exit @@ -30,22 +34,25 @@ from _pytest.python import Function from _pytest.python import Module from _pytest.runner import CallInfo -import pytest +from _pytest.runner import check_interactive_exception +from _pytest.subtests import SubtestContext +from _pytest.subtests import SubtestReport if sys.version_info[:2] < (3, 11): from exceptiongroup import ExceptionGroup if TYPE_CHECKING: + from types import TracebackType import unittest import twisted.trial.unittest -_SysExcInfoType = Union[ - Tuple[Type[BaseException], BaseException, types.TracebackType], - Tuple[None, None, None], -] +_SysExcInfoType = ( + tuple[type[BaseException], BaseException, types.TracebackType] + | tuple[None, None, None] +) def pytest_pycollect_makeitem( @@ -137,11 +144,11 @@ def process_teardown_exceptions() -> None: def unittest_setup_class_fixture( request: FixtureRequest, - ) -> Generator[None, None, None]: + ) -> Generator[None]: cls = request.cls if _is_skipped(cls): reason = cls.__unittest_skip_why__ - raise pytest.skip.Exception(reason, _use_item_location=True) + raise skip.Exception(reason, _use_item_location=True) if setup is not None: try: setup() @@ -178,11 +185,11 @@ def _register_unittest_setup_method_fixture(self, cls: type) -> None: def unittest_setup_method_fixture( request: FixtureRequest, - ) -> Generator[None, None, None]: + ) -> Generator[None]: self = request.instance if _is_skipped(self): reason = self.__unittest_skip_why__ - raise pytest.skip.Exception(reason, _use_item_location=True) + raise skip.Exception(reason, _use_item_location=True) if setup is not None: setup(self, request.function) yield @@ -201,6 +208,7 @@ def unittest_setup_method_fixture( class TestCaseFunction(Function): nofuncargs = True + failfast = False _excinfo: list[_pytest._code.ExceptionInfo[BaseException]] | None = None def _getinstance(self): @@ -217,6 +225,10 @@ def setup(self) -> None: # A bound method to be called during teardown() if set (see 'runtest()'). self._explicit_tearDown: Callable[[], None] | None = None super().setup() + if sys.version_info < (3, 11): + # A cache of the subTest errors and non-subtest skips in self._outcome. + # Compute and cache these lists once, instead of computing them again and again for each subtest (#13965). + self._cached_errors_and_skips: tuple[list[Any], list[Any]] | None = None def teardown(self) -> None: if self._explicit_tearDown is not None: @@ -230,8 +242,7 @@ def startTest(self, testcase: unittest.TestCase) -> None: pass def _addexcinfo(self, rawexcinfo: _SysExcInfoType) -> None: - # Unwrap potential exception info (see twisted trial support below). - rawexcinfo = getattr(rawexcinfo, "_rawexcinfo", rawexcinfo) + rawexcinfo = _handle_twisted_exc_info(rawexcinfo) try: excinfo = _pytest._code.ExceptionInfo[BaseException].from_exc_info( rawexcinfo # type: ignore[arg-type] @@ -279,11 +290,38 @@ def addFailure( ) -> None: self._addexcinfo(rawexcinfo) - def addSkip(self, testcase: unittest.TestCase, reason: str) -> None: - try: - raise pytest.skip.Exception(reason, _use_item_location=True) - except skip.Exception: - self._addexcinfo(sys.exc_info()) + def addSkip( + self, testcase: unittest.TestCase, reason: str, *, handle_subtests: bool = True + ) -> None: + from unittest.case import _SubTest # type: ignore[attr-defined] + + def add_skip() -> None: + try: + raise skip.Exception(reason, _use_item_location=True) + except skip.Exception: + self._addexcinfo(sys.exc_info()) + + if not handle_subtests: + add_skip() + return + + if isinstance(testcase, _SubTest): + add_skip() + if self._excinfo is not None: + exc_info = self._excinfo[-1] + self.addSubTest(testcase.test_case, testcase, exc_info) + else: + # For python < 3.11: the non-subtest skips have to be added by `add_skip` only after all subtest + # failures are processed by `_addSubTest`: `self.instance._outcome` has no attribute + # `skipped/errors` anymore. + # We also need to check if `self.instance._outcome` is `None` (this happens if the test + # class/method is decorated with `unittest.skip`, see pytest-dev/pytest-subtests#173). + if sys.version_info < (3, 11) and self.instance._outcome is not None: + subtest_errors, _ = self._obtain_errors_and_skips() + if len(subtest_errors) == 0: + add_skip() + else: + add_skip() def addExpectedFailure( self, @@ -363,6 +401,88 @@ def _traceback_filter( ntraceback = traceback return ntraceback + def addSubTest( + self, + test_case: Any, + test: TestCase, + exc_info: ExceptionInfo[BaseException] + | tuple[type[BaseException], BaseException, TracebackType] + | None, + ) -> None: + # Importing this private symbol locally in case this symbol is renamed/removed in the future; importing + # it globally would break pytest entirely, importing it locally only will break unittests using `addSubTest`. + from unittest.case import _subtest_msg_sentinel # type: ignore[attr-defined] + + exception_info: ExceptionInfo[BaseException] | None + match exc_info: + case tuple(): + exception_info = ExceptionInfo(exc_info, _ispytest=True) + case ExceptionInfo() | None: + exception_info = exc_info + case unreachable: + assert_never(unreachable) + + call_info = CallInfo[None]( + None, + exception_info, + start=0, + stop=0, + duration=0, + when="call", + _ispytest=True, + ) + msg = None if test._message is _subtest_msg_sentinel else str(test._message) # type: ignore[attr-defined] + report = self.ihook.pytest_runtest_makereport(item=self, call=call_info) + sub_report = SubtestReport._new( + report, + SubtestContext(msg=msg, kwargs=dict(test.params)), # type: ignore[attr-defined] + captured_output=None, + captured_logs=None, + ) + self.ihook.pytest_runtest_logreport(report=sub_report) + if check_interactive_exception(call_info, sub_report): + self.ihook.pytest_exception_interact( + node=self, call=call_info, report=sub_report + ) + + # For python < 3.11: add non-subtest skips once all subtest failures are processed by # `_addSubTest`. + if sys.version_info < (3, 11): + subtest_errors, non_subtest_skip = self._obtain_errors_and_skips() + + # Check if we have non-subtest skips: if there are also sub failures, non-subtest skips are not treated in + # `_addSubTest` and have to be added using `add_skip` after all subtest failures are processed. + if len(non_subtest_skip) > 0 and len(subtest_errors) > 0: + # Make sure we have processed the last subtest failure + last_subset_error = subtest_errors[-1] + if exc_info is last_subset_error[-1]: + # Add non-subtest skips (as they could not be treated in `_addSkip`) + for testcase, reason in non_subtest_skip: + self.addSkip(testcase, reason, handle_subtests=False) + + def _obtain_errors_and_skips(self) -> tuple[list[Any], list[Any]]: + """Compute or obtain the cached values for subtest errors and non-subtest skips.""" + from unittest.case import _SubTest # type: ignore[attr-defined] + + assert sys.version_info < (3, 11), ( + "This workaround only should be used in Python 3.10" + ) + if self._cached_errors_and_skips is not None: + return self._cached_errors_and_skips + + subtest_errors = [ + (x, y) + for x, y in self.instance._outcome.errors + if isinstance(x, _SubTest) and y is not None + ] + + non_subtest_skips = [ + (x, y) + for x, y in self.instance._outcome.skipped + if not isinstance(x, _SubTest) + ] + self._cached_errors_and_skips = (subtest_errors, non_subtest_skips) + return subtest_errors, non_subtest_skips + @hookimpl(tryfirst=True) def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None: @@ -375,61 +495,138 @@ def pytest_runtest_makereport(item: Item, call: CallInfo[None]) -> None: pass # Convert unittest.SkipTest to pytest.skip. - # This is actually only needed for nose, which reuses unittest.SkipTest for - # its own nose.SkipTest. For unittest TestCases, SkipTest is already - # handled internally, and doesn't reach here. + # This covers explicit `raise unittest.SkipTest`. unittest = sys.modules.get("unittest") if unittest and call.excinfo and isinstance(call.excinfo.value, unittest.SkipTest): excinfo = call.excinfo - call2 = CallInfo[None].from_call( - lambda: pytest.skip(str(excinfo.value)), call.when - ) + call2 = CallInfo[None].from_call(lambda: skip(str(excinfo.value)), call.when) call.excinfo = call2.excinfo -# Twisted trial support. -classImplements_has_run = False +def _is_skipped(obj) -> bool: + """Return True if the given object has been marked with @unittest.skip.""" + return bool(getattr(obj, "__unittest_skip__", False)) + + +def pytest_configure() -> None: + """Register the TestCaseFunction class as an IReporter if twisted.trial is available.""" + if _get_twisted_version() is not TwistedVersion.NotInstalled: + from twisted.trial.itrial import IReporter + from zope.interface import classImplements + + classImplements(TestCaseFunction, IReporter) + + +class TwistedVersion(Enum): + """ + The Twisted version installed in the environment. + + We have different workarounds in place for different versions of Twisted. + """ + + # Twisted version 24 or prior. + Version24 = auto() + # Twisted version 25 or later. + Version25 = auto() + # Twisted version is not available. + NotInstalled = auto() + + +def _get_twisted_version() -> TwistedVersion: + # We need to check if "twisted.trial.unittest" is specifically present in sys.modules. + # This is because we intend to integrate with Trial only when it's actively running + # the test suite, but not needed when only other Twisted components are in use. + if "twisted.trial.unittest" not in sys.modules: + return TwistedVersion.NotInstalled + + import importlib.metadata + + import packaging.version + + version_str = importlib.metadata.version("twisted") + version = packaging.version.parse(version_str) + if version.major <= 24: + return TwistedVersion.Version24 + else: + return TwistedVersion.Version25 + + +# Name of the attribute in `twisted.python.Failure` instances that stores +# the `sys.exc_info()` tuple. +# See twisted.trial support in `pytest_runtest_protocol`. +TWISTED_RAW_EXCINFO_ATTR = "_twisted_raw_excinfo" @hookimpl(wrapper=True) -def pytest_runtest_protocol(item: Item) -> Generator[None, object, object]: - if isinstance(item, TestCaseFunction) and "twisted.trial.unittest" in sys.modules: - ut: Any = sys.modules["twisted.python.failure"] - global classImplements_has_run - Failure__init__ = ut.Failure.__init__ - if not classImplements_has_run: - from twisted.trial.itrial import IReporter - from zope.interface import classImplements - - classImplements(TestCaseFunction, IReporter) - classImplements_has_run = True - - def excstore( +def pytest_runtest_protocol(item: Item) -> Iterator[None]: + if _get_twisted_version() is TwistedVersion.Version24: + import twisted.python.failure as ut + + # Monkeypatch `Failure.__init__` to store the raw exception info. + original__init__ = ut.Failure.__init__ + + def store_raw_exception_info( self, exc_value=None, exc_type=None, exc_tb=None, captureVars=None - ): + ): # pragma: no cover if exc_value is None: - self._rawexcinfo = sys.exc_info() + raw_exc_info = sys.exc_info() else: if exc_type is None: exc_type = type(exc_value) - self._rawexcinfo = (exc_type, exc_value, exc_tb) + if exc_tb is None: + exc_tb = sys.exc_info()[2] + raw_exc_info = (exc_type, exc_value, exc_tb) + setattr(self, TWISTED_RAW_EXCINFO_ATTR, tuple(raw_exc_info)) try: - Failure__init__( + original__init__( self, exc_value, exc_type, exc_tb, captureVars=captureVars ) - except TypeError: - Failure__init__(self, exc_value, exc_type, exc_tb) + except TypeError: # pragma: no cover + original__init__(self, exc_value, exc_type, exc_tb) - ut.Failure.__init__ = excstore - try: - res = yield - finally: - ut.Failure.__init__ = Failure__init__ + with MonkeyPatch.context() as patcher: + patcher.setattr(ut.Failure, "__init__", store_raw_exception_info) + return (yield) else: - res = yield - return res - - -def _is_skipped(obj) -> bool: - """Return True if the given object has been marked with @unittest.skip.""" - return bool(getattr(obj, "__unittest_skip__", False)) + return (yield) + + +def _handle_twisted_exc_info( + rawexcinfo: _SysExcInfoType | BaseException, +) -> _SysExcInfoType: + """ + Twisted passes a custom Failure instance to `addError()` instead of using `sys.exc_info()`. + Therefore, if `rawexcinfo` is a `Failure` instance, convert it into the equivalent `sys.exc_info()` tuple + as expected by pytest. + """ + twisted_version = _get_twisted_version() + if twisted_version is TwistedVersion.NotInstalled: + # Unfortunately, because we cannot import `twisted.python.failure` at the top of the file + # and use it in the signature, we need to use `type:ignore` here because we cannot narrow + # the type properly in the `if` statement above. + return rawexcinfo # type:ignore[return-value] + elif twisted_version is TwistedVersion.Version24: + # Twisted calls addError() passing its own classes (like `twisted.python.Failure`), which violates + # the `addError()` signature, so we extract the original `sys.exc_info()` tuple which is stored + # in the object. + if hasattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR): + saved_exc_info = getattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR) + # Delete the attribute from the original object to avoid leaks. + delattr(rawexcinfo, TWISTED_RAW_EXCINFO_ATTR) + return saved_exc_info # type:ignore[no-any-return] + return rawexcinfo # type:ignore[return-value] + elif twisted_version is TwistedVersion.Version25: + if isinstance(rawexcinfo, BaseException): + import twisted.python.failure + + if isinstance(rawexcinfo, twisted.python.failure.Failure): + tb = rawexcinfo.__traceback__ + if tb is None: + tb = sys.exc_info()[2] + return type(rawexcinfo.value), rawexcinfo.value, tb + + return rawexcinfo # type:ignore[return-value] + else: + # Ideally we would use assert_never() here, but it is not available in all Python versions + # we support, plus we do not require `type_extensions` currently. + assert False, f"Unexpected Twisted version: {twisted_version}" diff --git a/src/_pytest/unraisableexception.py b/src/_pytest/unraisableexception.py index c191703a3de..0faca36aa00 100644 --- a/src/_pytest/unraisableexception.py +++ b/src/_pytest/unraisableexception.py @@ -1,100 +1,163 @@ from __future__ import annotations +import collections +from collections.abc import Callable +import functools +import gc import sys import traceback -from types import TracebackType -from typing import Any -from typing import Callable -from typing import Generator +from typing import NamedTuple from typing import TYPE_CHECKING import warnings +from _pytest.config import Config +from _pytest.nodes import Item +from _pytest.stash import StashKey +from _pytest.tracemalloc import tracemalloc_message import pytest if TYPE_CHECKING: - from typing_extensions import Self - - -# Copied from cpython/Lib/test/support/__init__.py, with modifications. -class catch_unraisable_exception: - """Context manager catching unraisable exception using sys.unraisablehook. - - Storing the exception value (cm.unraisable.exc_value) creates a reference - cycle. The reference cycle is broken explicitly when the context manager - exits. - - Storing the object (cm.unraisable.object) can resurrect it if it is set to - an object which is being finalized. Exiting the context manager clears the - stored object. - - Usage: - with catch_unraisable_exception() as cm: - # code creating an "unraisable exception" - ... - # check the unraisable exception: use cm.unraisable - ... - # cm.unraisable attribute no longer exists at this point - # (to break a reference cycle) - """ - - def __init__(self) -> None: - self.unraisable: sys.UnraisableHookArgs | None = None - self._old_hook: Callable[[sys.UnraisableHookArgs], Any] | None = None - - def _hook(self, unraisable: sys.UnraisableHookArgs) -> None: - # Storing unraisable.object can resurrect an object which is being - # finalized. Storing unraisable.exc_value creates a reference cycle. - self.unraisable = unraisable - - def __enter__(self) -> Self: - self._old_hook = sys.unraisablehook - sys.unraisablehook = self._hook - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - assert self._old_hook is not None - sys.unraisablehook = self._old_hook - self._old_hook = None - del self.unraisable - - -def unraisable_exception_runtest_hook() -> Generator[None, None, None]: - with catch_unraisable_exception() as cm: - try: - yield - finally: - if cm.unraisable: - if cm.unraisable.err_msg is not None: - err_msg = cm.unraisable.err_msg - else: - err_msg = "Exception ignored in" - msg = f"{err_msg}: {cm.unraisable.object!r}\n\n" - msg += "".join( - traceback.format_exception( - cm.unraisable.exc_type, - cm.unraisable.exc_value, - cm.unraisable.exc_traceback, - ) - ) - warnings.warn(pytest.PytestUnraisableExceptionWarning(msg)) + pass + +if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + + +# This is a stash item and not a simple constant to allow pytester to override it. +gc_collect_iterations_key = StashKey[int]() + +def gc_collect_harder(iterations: int) -> None: + for _ in range(iterations): + gc.collect() -@pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_runtest_setup() -> Generator[None, None, None]: - yield from unraisable_exception_runtest_hook() +class UnraisableMeta(NamedTuple): + msg: str + cause_msg: str + exc_value: BaseException | None -@pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_runtest_call() -> Generator[None, None, None]: - yield from unraisable_exception_runtest_hook() +unraisable_exceptions: StashKey[collections.deque[UnraisableMeta | BaseException]] = ( + StashKey() +) -@pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_runtest_teardown() -> Generator[None, None, None]: - yield from unraisable_exception_runtest_hook() + +def collect_unraisable(config: Config) -> None: + pop_unraisable = config.stash[unraisable_exceptions].pop + errors: list[pytest.PytestUnraisableExceptionWarning | RuntimeError] = [] + meta = None + hook_error = None + try: + while True: + try: + meta = pop_unraisable() + except IndexError: + break + + if isinstance(meta, BaseException): + hook_error = RuntimeError("Failed to process unraisable exception") + hook_error.__cause__ = meta + errors.append(hook_error) + continue + + msg = meta.msg + try: + warnings.warn(pytest.PytestUnraisableExceptionWarning(msg)) + except pytest.PytestUnraisableExceptionWarning as e: + # This except happens when the warning is treated as an error (e.g. `-Werror`). + if meta.exc_value is not None: + # Exceptions have a better way to show the traceback, but + # warnings do not, so hide the traceback from the msg and + # set the cause so the traceback shows up in the right place. + e.args = (meta.cause_msg,) + e.__cause__ = meta.exc_value + errors.append(e) + + if len(errors) == 1: + raise errors[0] + if errors: + raise ExceptionGroup("multiple unraisable exception warnings", errors) + finally: + del errors, meta, hook_error + + +def cleanup( + *, config: Config, prev_hook: Callable[[sys.UnraisableHookArgs], object] +) -> None: + # A single collection doesn't necessarily collect everything. + # Constant determined experimentally by the Trio project. + gc_collect_iterations = config.stash.get(gc_collect_iterations_key, 5) + try: + try: + gc_collect_harder(gc_collect_iterations) + collect_unraisable(config) + finally: + sys.unraisablehook = prev_hook + finally: + del config.stash[unraisable_exceptions] + + +def unraisable_hook( + unraisable: sys.UnraisableHookArgs, + /, + *, + append: Callable[[UnraisableMeta | BaseException], object], +) -> None: + try: + # we need to compute these strings here as they might change after + # the unraisablehook finishes and before the metadata object is + # collected by a pytest hook + err_msg = ( + "Exception ignored in" if unraisable.err_msg is None else unraisable.err_msg + ) + summary = f"{err_msg}: {unraisable.object!r}" + traceback_message = "\n\n" + "".join( + traceback.format_exception( + unraisable.exc_type, + unraisable.exc_value, + unraisable.exc_traceback, + ) + ) + tracemalloc_tb = "\n" + tracemalloc_message(unraisable.object) + msg = summary + traceback_message + tracemalloc_tb + cause_msg = summary + tracemalloc_tb + + append( + UnraisableMeta( + msg=msg, + cause_msg=cause_msg, + exc_value=unraisable.exc_value, + ) + ) + except BaseException as e: + append(e) + # Raising this will cause the exception to be logged twice, once in our + # collect_unraisable and once by the unraisablehook calling machinery + # which is fine - this should never happen anyway and if it does + # it should probably be reported as a pytest bug. + raise + + +def pytest_configure(config: Config) -> None: + prev_hook = sys.unraisablehook + deque: collections.deque[UnraisableMeta | BaseException] = collections.deque() + config.stash[unraisable_exceptions] = deque + config.add_cleanup(functools.partial(cleanup, config=config, prev_hook=prev_hook)) + sys.unraisablehook = functools.partial(unraisable_hook, append=deque.append) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_setup(item: Item) -> None: + collect_unraisable(item.config) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_call(item: Item) -> None: + collect_unraisable(item.config) + + +@pytest.hookimpl(trylast=True) +def pytest_runtest_teardown(item: Item) -> None: + collect_unraisable(item.config) diff --git a/src/_pytest/warning_types.py b/src/_pytest/warning_types.py index 4ab14e48c92..93071b4a1b2 100644 --- a/src/_pytest/warning_types.py +++ b/src/_pytest/warning_types.py @@ -56,8 +56,8 @@ class PytestRemovedIn9Warning(PytestDeprecationWarning): __module__ = "pytest" -class PytestReturnNotNoneWarning(PytestWarning): - """Warning emitted when a test function is returning value other than None.""" +class PytestRemovedIn10Warning(PytestDeprecationWarning): + """Warning class for features that will be removed in pytest 10.""" __module__ = "pytest" @@ -78,12 +78,11 @@ def simple(cls, apiname: str) -> PytestExperimentalApiWarning: @final -class PytestUnhandledCoroutineWarning(PytestReturnNotNoneWarning): - """Warning emitted for an unhandled coroutine. +class PytestReturnNotNoneWarning(PytestWarning): + """ + Warning emitted when a test function returns a value other than ``None``. - A coroutine was encountered when collecting test functions, but was not - handled by any async-aware plugin. - Coroutine test functions are not natively supported. + See :ref:`return-not-none` for details. """ __module__ = "pytest" @@ -141,6 +140,13 @@ def format(self, **kwargs: Any) -> _W: return self.category(self.template.format(**kwargs)) +@final +class PytestFDWarning(PytestWarning): + """When the lsof plugin finds leaked fds.""" + + __module__ = "pytest" + + def warn_explicit_for(method: FunctionType, message: PytestWarning) -> None: """ Issue the warning :param:`message` for the definition of the given :param:`method` diff --git a/src/_pytest/warnings.py b/src/_pytest/warnings.py index 5c59e55c5db..1dbf0025a31 100644 --- a/src/_pytest/warnings.py +++ b/src/_pytest/warnings.py @@ -1,9 +1,10 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator from contextlib import contextmanager +from contextlib import ExitStack import sys -from typing import Generator from typing import Literal import warnings @@ -13,24 +14,19 @@ from _pytest.main import Session from _pytest.nodes import Item from _pytest.terminal import TerminalReporter +from _pytest.tracemalloc import tracemalloc_message import pytest -def pytest_configure(config: Config) -> None: - config.addinivalue_line( - "markers", - "filterwarnings(warning): add a warning filter to the given test. " - "see https://docs.pytest.org/en/stable/how-to/capture-warnings.html#pytest-mark-filterwarnings ", - ) - - @contextmanager def catch_warnings_for_item( config: Config, ihook, when: Literal["config", "collect", "runtest"], item: Item | None, -) -> Generator[None, None, None]: + *, + record: bool = True, +) -> Generator[None]: """Context manager that catches warnings generated in the contained execution block. ``item`` can be None if we are not in the context of an item execution. @@ -39,17 +35,13 @@ def catch_warnings_for_item( """ config_filters = config.getini("filterwarnings") cmdline_filters = config.known_args_namespace.pythonwarnings or [] - with warnings.catch_warnings(record=True) as log: - # mypy can't infer that record=True means log is not None; help it. - assert log is not None - + with warnings.catch_warnings(record=record) as log: if not sys.warnoptions: # If user is not explicitly configuring warning filters, show deprecation warnings by default (#2908). warnings.filterwarnings("always", category=DeprecationWarning) warnings.filterwarnings("always", category=PendingDeprecationWarning) - # To be enabled in pytest 9.0.0. - # warnings.filterwarnings("error", category=pytest.PytestRemovedIn9Warning) + warnings.filterwarnings("error", category=pytest.PytestRemovedIn9Warning) apply_warning_filters(config_filters, cmdline_filters) @@ -63,45 +55,30 @@ def catch_warnings_for_item( try: yield finally: - for warning_message in log: - ihook.pytest_warning_recorded.call_historic( - kwargs=dict( - warning_message=warning_message, - nodeid=nodeid, - when=when, - location=None, + if record: + # mypy can't infer that record=True means log is not None; help it. + assert log is not None + + for warning_message in log: + ihook.pytest_warning_recorded.call_historic( + kwargs=dict( + warning_message=warning_message, + nodeid=nodeid, + when=when, + location=None, + ) ) - ) def warning_record_to_str(warning_message: warnings.WarningMessage) -> str: """Convert a warnings.WarningMessage to a string.""" - warn_msg = warning_message.message - msg = warnings.formatwarning( - str(warn_msg), + return warnings.formatwarning( + str(warning_message.message), warning_message.category, warning_message.filename, warning_message.lineno, warning_message.line, - ) - if warning_message.source is not None: - try: - import tracemalloc - except ImportError: - pass - else: - tb = tracemalloc.get_object_traceback(warning_message.source) - if tb is not None: - formatted_tb = "\n".join(tb.format()) - # Use a leading new line to better separate the (large) output - # from the traceback to the previous warning text. - msg += f"\nObject allocated at:\n{formatted_tb}" - else: - # No need for a leading new line. - url = "https://docs.pytest.org/en/stable/how-to/capture-warnings.html#resource-warnings" - msg += "Enable tracemalloc to get traceback where the object was allocated.\n" - msg += f"See {url} for more info." - return msg + ) + tracemalloc_message(warning_message.source) @pytest.hookimpl(wrapper=True, tryfirst=True) @@ -124,7 +101,7 @@ def pytest_collection(session: Session) -> Generator[None, object, object]: @pytest.hookimpl(wrapper=True) def pytest_terminal_summary( terminalreporter: TerminalReporter, -) -> Generator[None, None, None]: +) -> Generator[None]: config = terminalreporter.config with catch_warnings_for_item( config=config, ihook=config.hook, when="config", item=None @@ -133,7 +110,7 @@ def pytest_terminal_summary( @pytest.hookimpl(wrapper=True) -def pytest_sessionfinish(session: Session) -> Generator[None, None, None]: +def pytest_sessionfinish(session: Session) -> Generator[None]: config = session.config with catch_warnings_for_item( config=config, ihook=config.hook, when="config", item=None @@ -144,8 +121,31 @@ def pytest_sessionfinish(session: Session) -> Generator[None, None, None]: @pytest.hookimpl(wrapper=True) def pytest_load_initial_conftests( early_config: Config, -) -> Generator[None, None, None]: +) -> Generator[None]: with catch_warnings_for_item( config=early_config, ihook=early_config.hook, when="config", item=None ): return (yield) + + +def pytest_configure(config: Config) -> None: + with ExitStack() as stack: + stack.enter_context( + catch_warnings_for_item( + config=config, + ihook=config.hook, + when="config", + item=None, + # this disables recording because the terminalreporter has + # finished by the time it comes to reporting logged warnings + # from the end of config cleanup. So for now, this is only + # useful for setting a warning filter with an 'error' action. + record=False, + ) + ) + config.addinivalue_line( + "markers", + "filterwarnings(warning): add a warning filter to the given test. " + "see https://docs.pytest.org/en/stable/how-to/capture-warnings.html#pytest-mark-filterwarnings ", + ) + config.add_cleanup(stack.pop_all().close) diff --git a/src/pytest/__init__.py b/src/pytest/__init__.py index 90abcdab036..3e6281ac388 100644 --- a/src/pytest/__init__.py +++ b/src/pytest/__init__.py @@ -33,6 +33,7 @@ from _pytest.logging import LogCaptureFixture from _pytest.main import Dir from _pytest.main import Session +from _pytest.mark import HIDDEN_PARAM from _pytest.mark import Mark from _pytest.mark import MARK_GEN as mark from _pytest.mark import MarkDecorator @@ -59,7 +60,9 @@ from _pytest.python import Module from _pytest.python import Package from _pytest.python_api import approx -from _pytest.python_api import raises +from _pytest.raises import raises +from _pytest.raises import RaisesExc +from _pytest.raises import RaisesGroup from _pytest.recwarn import deprecated_call from _pytest.recwarn import WarningsRecorder from _pytest.recwarn import warns @@ -68,6 +71,9 @@ from _pytest.runner import CallInfo from _pytest.stash import Stash from _pytest.stash import StashKey +from _pytest.subtests import SubtestReport +from _pytest.subtests import Subtests +from _pytest.terminal import TerminalReporter from _pytest.terminal import TestShortLogReport from _pytest.tmpdir import TempPathFactory from _pytest.warning_types import PytestAssertRewriteWarning @@ -76,9 +82,10 @@ from _pytest.warning_types import PytestConfigWarning from _pytest.warning_types import PytestDeprecationWarning from _pytest.warning_types import PytestExperimentalApiWarning +from _pytest.warning_types import PytestFDWarning from _pytest.warning_types import PytestRemovedIn9Warning +from _pytest.warning_types import PytestRemovedIn10Warning from _pytest.warning_types import PytestReturnNotNoneWarning -from _pytest.warning_types import PytestUnhandledCoroutineWarning from _pytest.warning_types import PytestUnhandledThreadExceptionWarning from _pytest.warning_types import PytestUnknownMarkWarning from _pytest.warning_types import PytestUnraisableExceptionWarning @@ -89,41 +96,28 @@ __all__ = [ - "__version__", - "approx", + "HIDDEN_PARAM", "Cache", "CallInfo", "CaptureFixture", "Class", - "cmdline", - "Collector", "CollectReport", + "Collector", "Config", - "console_main", - "deprecated_call", "Dir", "Directory", "DoctestItem", - "exit", "ExceptionInfo", "ExitCode", - "fail", "File", - "fixture", "FixtureDef", "FixtureLookupError", "FixtureRequest", - "freeze_includes", "Function", - "hookimpl", "HookRecorder", - "hookspec", - "importorskip", "Item", "LineMatcher", "LogCaptureFixture", - "main", - "mark", "Mark", "MarkDecorator", "MarkGenerator", @@ -132,7 +126,6 @@ "MonkeyPatch", "OptionGroup", "Package", - "param", "Parser", "PytestAssertRewriteWarning", "PytestCacheWarning", @@ -140,32 +133,53 @@ "PytestConfigWarning", "PytestDeprecationWarning", "PytestExperimentalApiWarning", + "PytestFDWarning", + "PytestPluginManager", "PytestRemovedIn9Warning", + "PytestRemovedIn10Warning", "PytestReturnNotNoneWarning", - "Pytester", - "PytestPluginManager", - "PytestUnhandledCoroutineWarning", "PytestUnhandledThreadExceptionWarning", "PytestUnknownMarkWarning", "PytestUnraisableExceptionWarning", "PytestWarning", - "raises", + "Pytester", + "RaisesExc", + "RaisesGroup", "RecordedHookCall", - "register_assert_rewrite", "RunResult", "Session", - "set_trace", - "skip", "Stash", "StashKey", - "version_tuple", - "TempdirFactory", + "SubtestReport", + "Subtests", "TempPathFactory", - "Testdir", + "TempdirFactory", + "TerminalReporter", "TestReport", "TestShortLogReport", + "Testdir", "UsageError", "WarningsRecorder", + "__version__", + "approx", + "cmdline", + "console_main", + "deprecated_call", + "exit", + "fail", + "fixture", + "freeze_includes", + "hookimpl", + "hookspec", + "importorskip", + "main", + "mark", + "param", + "raises", + "register_assert_rewrite", + "set_trace", + "skip", + "version_tuple", "warns", "xfail", "yield_fixture", diff --git a/testing/_py/test_local.py b/testing/_py/test_local.py index 4a95e2d0cd9..592058a54a5 100644 --- a/testing/_py/test_local.py +++ b/testing/_py/test_local.py @@ -9,17 +9,17 @@ from unittest import mock import warnings -from py import error from py.path import local +from py import error + import pytest @contextlib.contextmanager def ignore_encoding_warning(): with warnings.catch_warnings(): - if sys.version_info > (3, 10): - warnings.simplefilter("ignore", EncodingWarning) + warnings.simplefilter("ignore", EncodingWarning) yield @@ -207,15 +207,11 @@ def test_visit_norecurse(self, path1): assert "sampledir" in lst assert path1.sep.join(["sampledir", "otherfile"]) not in lst - @pytest.mark.parametrize( - "fil", - ["*dir", "*dir", pytest.mark.skip("sys.version_info < (3,6)")(b"*dir")], - ) - def test_visit_filterfunc_is_string(self, path1, fil): + def test_visit_filterfunc_is_string(self, path1): lst = [] - for i in path1.visit(fil): + for i in path1.visit("*dir"): lst.append(i.relto(path1)) - assert len(lst), 2 + assert len(lst), 2 # noqa: PLC1802,RUF040 assert "sampledir" in lst assert "otherdir" in lst @@ -463,12 +459,11 @@ def test_fspath_func_match_strpath(self, path1): assert fspath(path1) == path1.strpath - @pytest.mark.skip("sys.version_info < (3,6)") def test_fspath_open(self, path1): - f = path1.join("opentestfile") - open(f) + f = path1.join("samplefile") + stream = open(f, encoding="utf-8") + stream.close() - @pytest.mark.skip("sys.version_info < (3,6)") def test_fspath_fsencode(self, path1): from os import fsencode @@ -555,9 +550,9 @@ def batch_make_numbered_dirs(rootdir, repeats): file_ = dir_.join("foo") file_.write_text(f"{i}", encoding="utf-8") actual = int(file_.read_text(encoding="utf-8")) - assert ( - actual == i - ), f"int(file_.read_text(encoding='utf-8')) is {actual} instead of {i}" + assert actual == i, ( + f"int(file_.read_text(encoding='utf-8')) is {actual} instead of {i}" + ) dir_.join(".lock").remove(ignore_errors=True) return True @@ -738,7 +733,6 @@ def test_dump(self, tmpdir, bin): def test_setmtime(self): import tempfile - import time try: fd, name = tempfile.mkstemp() @@ -747,6 +741,7 @@ def test_setmtime(self): name = tempfile.mktemp() open(name, "w").close() try: + # Do not use _pytest.timing here, as we do not want time mocking to affect this test. mtime = int(time.time()) - 100 path = local(name) assert path.mtime() != mtime @@ -855,7 +850,7 @@ def test_fnmatch_file_abspath(self, tmpdir): assert b.fnmatch(pattern) def test_sysfind(self): - name = sys.platform == "win32" and "cmd" or "test" + name = (sys.platform == "win32" and "cmd") or "test" x = local.sysfind(name) assert x.check(file=1) assert local.sysfind("jaksdkasldqwe") is None @@ -948,7 +943,7 @@ def test_make_numbered_dir(self, tmpdir): prefix="base.", rootdir=tmpdir, keep=2, lock_timeout=0 ) assert numdir.check() - assert numdir.basename == "base.%d" % i + assert numdir.basename == f"base.{i}" if i >= 1: assert numdir.new(ext=str(i - 1)).check() if i >= 2: @@ -993,7 +988,7 @@ def test_locked_make_numbered_dir(self, tmpdir): for i in range(10): numdir = local.make_numbered_dir(prefix="base2.", rootdir=tmpdir, keep=2) assert numdir.check() - assert numdir.basename == "base2.%d" % i + assert numdir.basename == f"base2.{i}" for j in range(i): assert numdir.new(ext=str(j)).check() @@ -1250,7 +1245,7 @@ def test_owner_group_not_implemented(self, path1): def test_chmod_simple_int(self, path1): mode = path1.stat().mode # Ensure that we actually change the mode to something different. - path1.chmod(mode == 0 and 1 or 0) + path1.chmod((mode == 0 and 1) or 0) try: print(path1.stat().mode) print(mode) @@ -1405,6 +1400,7 @@ def test_atime(self, tmpdir): import time path = tmpdir.ensure("samplefile") + # Do not use _pytest.timing here, as we do not want time mocking to affect this test. now = time.time() atime1 = path.atime() # we could wait here but timer resolution is very diff --git a/testing/acceptance_test.py b/testing/acceptance_test.py index 01d911e8ca4..b9384008483 100644 --- a/testing/acceptance_test.py +++ b/testing/acceptance_test.py @@ -1,6 +1,7 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Sequence import dataclasses import importlib.metadata import os @@ -9,6 +10,8 @@ import sys import types +import setuptools + from _pytest.config import ExitCode from _pytest.pathlib import symlink_or_skip from _pytest.pytester import Pytester @@ -641,10 +644,9 @@ def test_invoke_with_invalid_type(self) -> None: ): pytest.main("-h") # type: ignore[arg-type] - def test_invoke_with_path(self, pytester: Pytester, capsys) -> None: + def test_invoke_with_path(self, pytester: Pytester) -> None: retcode = pytest.main([str(pytester.path)]) assert retcode == ExitCode.NO_TESTS_COLLECTED - out, err = capsys.readouterr() def test_invoke_plugin_api(self, capsys) -> None: class MyPlugin: @@ -652,7 +654,7 @@ def pytest_addoption(self, parser): parser.addoption("--myopt") pytest.main(["-h"], plugins=[MyPlugin()]) - out, err = capsys.readouterr() + out, _err = capsys.readouterr() assert "--myopt" in out def test_pyargs_importerror(self, pytester: Pytester, monkeypatch) -> None: @@ -721,10 +723,14 @@ def test_cmdline_python_package(self, pytester: Pytester, monkeypatch) -> None: assert result.ret != 0 result.stderr.fnmatch_lines(["*not*found*test_missing*"]) - def test_cmdline_python_namespace_package( + @pytest.mark.skipif( + int(setuptools.__version__.split(".")[0]) >= 80, + reason="modern setuptools removing pkg_resources", + ) + def test_cmdline_python_legacy_namespace_package( self, pytester: Pytester, monkeypatch ) -> None: - """Test --pyargs option with namespace packages (#1567). + """Test --pyargs option with legacy namespace packages (#1567). Ref: https://packaging.python.org/guides/packaging-namespace-packages/ """ @@ -970,28 +976,43 @@ def test_calls_showall(self, pytester: Pytester, mock_timing) -> None: pytester.makepyfile(self.source) result = pytester.runpytest_inprocess("--durations=0") assert result.ret == 0 - - tested = "3" - for x in tested: - for y in ("call",): # 'setup', 'call', 'teardown': - for line in result.stdout.lines: - if (f"test_{x}") in line and y in line: - break - else: - raise AssertionError(f"not found {x} {y}") + TestDurations.check_tests_in_output(result.stdout.lines, 2, 3) def test_calls_showall_verbose(self, pytester: Pytester, mock_timing) -> None: pytester.makepyfile(self.source) result = pytester.runpytest_inprocess("--durations=0", "-vv") assert result.ret == 0 + TestDurations.check_tests_in_output(result.stdout.lines, 1, 2, 3) + + def test_calls_showall_durationsmin(self, pytester: Pytester, mock_timing) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess("--durations=0", "--durations-min=0.015") + assert result.ret == 0 + TestDurations.check_tests_in_output(result.stdout.lines, 3) + + def test_calls_showall_durationsmin_verbose( + self, pytester: Pytester, mock_timing + ) -> None: + pytester.makepyfile(self.source) + result = pytester.runpytest_inprocess( + "--durations=0", "--durations-min=0.015", "-vv" + ) + assert result.ret == 0 + TestDurations.check_tests_in_output(result.stdout.lines, 3) - for x in "123": - for y in ("call",): # 'setup', 'call', 'teardown': - for line in result.stdout.lines: - if (f"test_{x}") in line and y in line: - break - else: - raise AssertionError(f"not found {x} {y}") + @staticmethod + def check_tests_in_output( + lines: Sequence[str], *expected_test_numbers: int, number_of_tests: int = 3 + ) -> None: + found_test_numbers = { + test_number + for test_number in range(1, number_of_tests + 1) + if any( + line.endswith(f"test_{test_number}") and " call " in line + for line in lines + ) + } + assert found_test_numbers == set(expected_test_numbers) def test_with_deselected(self, pytester: Pytester, mock_timing) -> None: pytester.makepyfile(self.source) @@ -1235,7 +1256,7 @@ def test_usage_error_code(pytester: Pytester) -> None: assert result.ret == ExitCode.USAGE_ERROR -def test_warn_on_async_function(pytester: Pytester) -> None: +def test_error_on_async_function(pytester: Pytester) -> None: # In the below we .close() the coroutine only to avoid # "RuntimeWarning: coroutine 'test_2' was never awaited" # which messes with other tests. @@ -1251,23 +1272,19 @@ def test_3(): return coro """ ) - result = pytester.runpytest("-Wdefault") + result = pytester.runpytest() result.stdout.fnmatch_lines( [ - "test_async.py::test_1", - "test_async.py::test_2", - "test_async.py::test_3", "*async def functions are not natively supported*", - "*3 skipped, 3 warnings in*", + "*test_async.py::test_1*", + "*test_async.py::test_2*", + "*test_async.py::test_3*", ] ) - # ensure our warning message appears only once - assert ( - result.stdout.str().count("async def functions are not natively supported") == 1 - ) + result.assert_outcomes(failed=3) -def test_warn_on_async_gen_function(pytester: Pytester) -> None: +def test_error_on_async_gen_function(pytester: Pytester) -> None: pytester.makepyfile( test_async=""" async def test_1(): @@ -1278,20 +1295,114 @@ def test_3(): return test_2() """ ) - result = pytester.runpytest("-Wdefault") + result = pytester.runpytest() result.stdout.fnmatch_lines( [ - "test_async.py::test_1", - "test_async.py::test_2", - "test_async.py::test_3", "*async def functions are not natively supported*", - "*3 skipped, 3 warnings in*", + "*test_async.py::test_1*", + "*test_async.py::test_2*", + "*test_async.py::test_3*", ] ) - # ensure our warning message appears only once - assert ( - result.stdout.str().count("async def functions are not natively supported") == 1 + result.assert_outcomes(failed=3) + + +def test_warning_on_sync_test_async_fixture(pytester: Pytester) -> None: + pytester.makepyfile( + test_sync=""" + import pytest + + @pytest.fixture + async def async_fixture(): + ... + + def test_foo(async_fixture): + # suppress unawaited coroutine warning + try: + async_fixture.send(None) + except StopIteration: + pass + """ + ) + result = pytester.runpytest("-Wdefault::pytest.PytestRemovedIn9Warning") + result.stdout.fnmatch_lines( + [ + "*== warnings summary ==*", + ( + "*PytestRemovedIn9Warning: 'test_foo' requested an async " + "fixture 'async_fixture', with no plugin or hook that handled it. " + "This is usually an error, as pytest does not natively support it. " + "This will turn into an error in pytest 9." + ), + " See: https://docs.pytest.org/en/stable/deprecations.html#sync-test-depending-on-async-fixture", + ] ) + result.assert_outcomes(passed=1, warnings=1) + + +def test_warning_on_sync_test_async_fixture_gen(pytester: Pytester) -> None: + pytester.makepyfile( + test_sync=""" + import pytest + + @pytest.fixture + async def async_fixture(): + yield + + def test_foo(async_fixture): + # async gens don't emit unawaited-coroutine + ... + """ + ) + result = pytester.runpytest("-Wdefault::pytest.PytestRemovedIn9Warning") + result.stdout.fnmatch_lines( + [ + "*== warnings summary ==*", + ( + "*PytestRemovedIn9Warning: 'test_foo' requested an async " + "fixture 'async_fixture', with no plugin or hook that handled it. " + "This is usually an error, as pytest does not natively support it. " + "This will turn into an error in pytest 9." + ), + " See: https://docs.pytest.org/en/stable/deprecations.html#sync-test-depending-on-async-fixture", + ] + ) + result.assert_outcomes(passed=1, warnings=1) + + +def test_warning_on_sync_test_async_autouse_fixture(pytester: Pytester) -> None: + pytester.makepyfile( + test_sync=""" + import pytest + + @pytest.fixture(autouse=True) + async def async_fixture(): + ... + + # We explicitly request the fixture to be able to + # suppress the RuntimeWarning for unawaited coroutine. + def test_foo(async_fixture): + try: + async_fixture.send(None) + except StopIteration: + pass + """ + ) + result = pytester.runpytest("-Wdefault::pytest.PytestRemovedIn9Warning") + result.stdout.fnmatch_lines( + [ + "*== warnings summary ==*", + ( + "*PytestRemovedIn9Warning: 'test_foo' requested an async " + "fixture 'async_fixture' with autouse=True, with no plugin or hook " + "that handled it. " + "This is usually an error, as pytest does not natively support it. " + "This will turn into an error in pytest 9." + ), + " See: https://docs.pytest.org/en/stable/deprecations.html#sync-test-depending-on-async-fixture", + ] + ) + result.assert_outcomes(passed=1, warnings=1) def test_pdb_can_be_rewritten(pytester: Pytester) -> None: @@ -1377,6 +1488,7 @@ def test_no_brokenpipeerror_message(pytester: Pytester) -> None: popen.stderr.close() +@pytest.mark.filterwarnings("default") def test_function_return_non_none_warning(pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1486,3 +1598,56 @@ def my_fixture(self, request): raise AssertionError( f"pytest command failed:\n{exc.stdout=!s}\n{exc.stderr=!s}" ) from exc + + +def test_no_terminal_plugin(pytester: Pytester) -> None: + """Smoke test to ensure pytest can execute without the terminal plugin (#9422).""" + pytester.makepyfile("def test(): assert 1 == 2") + result = pytester.runpytest("-pno:terminal", "-s") + assert result.ret == ExitCode.TESTS_FAILED + + +def test_stop_iteration_from_collect(pytester: Pytester) -> None: + pytester.makepyfile(test_it="raise StopIteration('hello')") + result = pytester.runpytest() + assert result.ret == ExitCode.INTERRUPTED + result.assert_outcomes(failed=0, passed=0, errors=1) + result.stdout.fnmatch_lines( + [ + "=* short test summary info =*", + "ERROR test_it.py - StopIteration: hello", + "!* Interrupted: 1 error during collection !*", + "=* 1 error in * =*", + ] + ) + + +def test_stop_iteration_runtest_protocol(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import pytest + @pytest.fixture + def fail_setup(): + raise StopIteration(1) + def test_fail_setup(fail_setup): + pass + def test_fail_teardown(request): + def stop_iteration(): + raise StopIteration(2) + request.addfinalizer(stop_iteration) + def test_fail_call(): + raise StopIteration(3) + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.TESTS_FAILED + result.assert_outcomes(failed=1, passed=1, errors=2) + result.stdout.fnmatch_lines( + [ + "=* short test summary info =*", + "FAILED test_it.py::test_fail_call - StopIteration: 3", + "ERROR test_it.py::test_fail_setup - StopIteration: 1", + "ERROR test_it.py::test_fail_teardown - StopIteration: 2", + "=* 1 failed, 1 passed, 2 errors in * =*", + ] + ) diff --git a/testing/code/test_code.py b/testing/code/test_code.py index 7ae5ad46100..ae5e0e949cf 100644 --- a/testing/code/test_code.py +++ b/testing/code/test_code.py @@ -86,9 +86,9 @@ def test_unicode_handling() -> None: value = "ąć".encode() def f() -> None: - raise Exception(value) + raise ValueError(value) - excinfo = pytest.raises(Exception, f) + excinfo = pytest.raises(ValueError, f) str(excinfo) diff --git a/testing/code/test_excinfo.py b/testing/code/test_excinfo.py index fc60ae9ac99..70499fec893 100644 --- a/testing/code/test_excinfo.py +++ b/testing/code/test_excinfo.py @@ -31,6 +31,7 @@ from _pytest._code.code import TracebackStyle if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup from exceptiongroup import ExceptionGroup @@ -262,7 +263,7 @@ def do_stuff() -> None: def reraise_me() -> None: import sys - exc, val, tb = sys.exc_info() + _exc, val, tb = sys.exc_info() assert val is not None raise val.with_traceback(tb) @@ -441,9 +442,9 @@ def test_division_zero(): assert result.ret != 0 match = [ - r"E .* AssertionError: Regex pattern did not match.", - r"E .* Regex: '\[123\]\+'", - r"E .* Input: 'division by zero'", + r"E\s+AssertionError: Regex pattern did not match.", + r"E\s+Expected regex: '\[123\]\+'", + r"E\s+Actual message: 'division by zero'", ] result.stdout.re_match_lines(match) result.stdout.no_fnmatch_line("*__tracebackhide__ = True*") @@ -453,6 +454,39 @@ def test_division_zero(): result.stdout.re_match_lines([r".*__tracebackhide__ = True.*", *match]) +def test_raises_accepts_generic_group() -> None: + with pytest.raises(ExceptionGroup[Exception]) as exc_info: + raise ExceptionGroup("", [RuntimeError()]) + assert exc_info.group_contains(RuntimeError) + + +def test_raises_accepts_generic_base_group() -> None: + with pytest.raises(BaseExceptionGroup[BaseException]) as exc_info: + raise ExceptionGroup("", [RuntimeError()]) + assert exc_info.group_contains(RuntimeError) + + +def test_raises_rejects_specific_generic_group() -> None: + with pytest.raises(ValueError): + pytest.raises(ExceptionGroup[RuntimeError]) + + +def test_raises_accepts_generic_group_in_tuple() -> None: + with pytest.raises((ValueError, ExceptionGroup[Exception])) as exc_info: + raise ExceptionGroup("", [RuntimeError()]) + assert exc_info.group_contains(RuntimeError) + + +def test_raises_exception_escapes_generic_group() -> None: + try: + with pytest.raises(ExceptionGroup[Exception]): + raise ValueError("my value error") + except ValueError as e: + assert str(e) == "my value error" + else: + pytest.fail("Expected ValueError to be raised") + + class TestGroupContains: def test_contains_exception_type(self) -> None: exc_group = ExceptionGroup("", [RuntimeError()]) @@ -849,6 +883,37 @@ def entry(): assert basename in str(reprtb.reprfileloc.path) assert reprtb.reprfileloc.lineno == 3 + @pytest.mark.skipif( + "sys.version_info < (3,11)", + reason="Column level traceback info added in python 3.11", + ) + def test_repr_traceback_entry_short_carets(self, importasmod) -> None: + mod = importasmod( + """ + def div_by_zero(): + return 1 / 0 + def func1(): + return 42 + div_by_zero() + def entry(): + func1() + """ + ) + excinfo = pytest.raises(ZeroDivisionError, mod.entry) + p = FormattedExcinfo(style="short") + reprtb = p.repr_traceback_entry(excinfo.traceback[-3]) + assert len(reprtb.lines) == 1 + assert reprtb.lines[0] == " func1()" + + reprtb = p.repr_traceback_entry(excinfo.traceback[-2]) + assert len(reprtb.lines) == 2 + assert reprtb.lines[0] == " return 42 + div_by_zero()" + assert reprtb.lines[1] == " ^^^^^^^^^^^^^" + + reprtb = p.repr_traceback_entry(excinfo.traceback[-1]) + assert len(reprtb.lines) == 2 + assert reprtb.lines[0] == " return 1 / 0" + assert reprtb.lines[1] == " ^^^^^" + def test_repr_tracebackentry_no(self, importasmod): mod = importasmod( """ @@ -964,8 +1029,8 @@ def raiseos(): upframe = sys._getframe().f_back assert upframe is not None if upframe.f_code.co_name == "_makepath": - # Only raise with expected calls, but not via e.g. inspect for - # py38-windows. + # Only raise with expected calls, and not accidentally via 'inspect' + # See 79ae86cc3f76d69460e1c7beca4ce95e68ab80a6 raised += 1 raise OSError(2, "custom_oserror") return orig_path_cwd() @@ -1194,6 +1259,23 @@ def f(): line = tw_mock.lines[-1] assert line == ":3: ValueError" + def test_toterminal_value(self, importasmod, tw_mock): + mod = importasmod( + """ + def g(x): + raise ValueError(x) + def f(): + g('some_value') + """ + ) + excinfo = pytest.raises(ValueError, mod.f) + excinfo.traceback = excinfo.traceback.filter(excinfo) + repr = excinfo.getrepr(style="value") + repr.toterminal(tw_mock) + + assert tw_mock.get_write_msg(0) == "some_value" + assert tw_mock.get_write_msg(1) == "\n" + @pytest.mark.parametrize( "reproptions", [ @@ -1292,7 +1374,7 @@ def g(): raise ValueError() def h(): - raise AttributeError() + if True: raise AttributeError() """ ) excinfo = pytest.raises(AttributeError, mod.f) @@ -1353,12 +1435,22 @@ def h(): assert tw_mock.lines[40] == ("_ ", None) assert tw_mock.lines[41] == "" assert tw_mock.lines[42] == " def h():" - assert tw_mock.lines[43] == "> raise AttributeError()" - assert tw_mock.lines[44] == "E AttributeError" - assert tw_mock.lines[45] == "" - line = tw_mock.get_write_msg(46) - assert line.endswith("mod.py") - assert tw_mock.lines[47] == ":15: AttributeError" + # On python 3.11 and greater, check for carets in the traceback. + if sys.version_info >= (3, 11): + assert tw_mock.lines[43] == "> if True: raise AttributeError()" + assert tw_mock.lines[44] == " ^^^^^^^^^^^^^^^^^^^^^^" + assert tw_mock.lines[45] == "E AttributeError" + assert tw_mock.lines[46] == "" + line = tw_mock.get_write_msg(47) + assert line.endswith("mod.py") + assert tw_mock.lines[48] == ":15: AttributeError" + else: + assert tw_mock.lines[43] == "> if True: raise AttributeError()" + assert tw_mock.lines[44] == "E AttributeError" + assert tw_mock.lines[45] == "" + line = tw_mock.get_write_msg(46) + assert line.endswith("mod.py") + assert tw_mock.lines[47] == ":15: AttributeError" @pytest.mark.parametrize("mode", ["from_none", "explicit_suppress"]) def test_exc_repr_chain_suppression(self, importasmod, mode, tw_mock): @@ -1477,23 +1569,44 @@ def unreraise(): r = excinfo.getrepr(style="short") r.toterminal(tw_mock) out = "\n".join(line for line in tw_mock.lines if isinstance(line, str)) - expected_out = textwrap.dedent( - """\ - :13: in unreraise - reraise() - :10: in reraise - raise Err() from e - E test_exc_chain_repr_cycle0.mod.Err - - During handling of the above exception, another exception occurred: - :15: in unreraise - raise e.__cause__ - :8: in reraise - fail() - :5: in fail - return 0 / 0 - E ZeroDivisionError: division by zero""" - ) + # Assert highlighting carets in python3.11+ + if sys.version_info >= (3, 11): + expected_out = textwrap.dedent( + """\ + :13: in unreraise + reraise() + :10: in reraise + raise Err() from e + E test_exc_chain_repr_cycle0.mod.Err + + During handling of the above exception, another exception occurred: + :15: in unreraise + raise e.__cause__ + :8: in reraise + fail() + :5: in fail + return 0 / 0 + ^^^^^ + E ZeroDivisionError: division by zero""" + ) + else: + expected_out = textwrap.dedent( + """\ + :13: in unreraise + reraise() + :10: in reraise + raise Err() from e + E test_exc_chain_repr_cycle0.mod.Err + + During handling of the above exception, another exception occurred: + :15: in unreraise + raise e.__cause__ + :8: in reraise + fail() + :5: in fail + return 0 / 0 + E ZeroDivisionError: division by zero""" + ) assert out == expected_out def test_exec_type_error_filter(self, importasmod): @@ -1684,6 +1797,9 @@ def test(): rf"FAILED test_excgroup.py::test - {pre_catch}BaseExceptionGroup: Oops \(2.*" ) result.stdout.re_match_lines(match_lines) + # Check for traceback filtering of pytest internals. + result.stdout.no_fnmatch_line("*, line *, in pytest_pyfunc_call") + result.stdout.no_fnmatch_line("*, line *, in pytest_runtest_call") @pytest.mark.skipif( @@ -1703,20 +1819,101 @@ def test_exceptiongroup(pytester: Pytester, outer_chain, inner_chain) -> None: _exceptiongroup_common(pytester, outer_chain, inner_chain, native=False) +def test_exceptiongroup_short_summary_info(pytester: Pytester): + pytester.makepyfile( + """ + import sys + + if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup, ExceptionGroup + + def test_base() -> None: + raise BaseExceptionGroup("NOT IN SUMMARY", [SystemExit("a" * 10)]) + + def test_nonbase() -> None: + raise ExceptionGroup("NOT IN SUMMARY", [ValueError("a" * 10)]) + + def test_nested() -> None: + raise ExceptionGroup( + "NOT DISPLAYED", [ + ExceptionGroup("NOT IN SUMMARY", [ValueError("a" * 10)]) + ] + ) + + def test_multiple() -> None: + raise ExceptionGroup( + "b" * 10, + [ + ValueError("NOT IN SUMMARY"), + TypeError("NOT IN SUMMARY"), + ] + ) + + def test_nested_multiple() -> None: + raise ExceptionGroup( + "b" * 10, + [ + ExceptionGroup( + "c" * 10, + [ + ValueError("NOT IN SUMMARY"), + TypeError("NOT IN SUMMARY"), + ] + ) + ] + ) + """ + ) + # run with -vv to not truncate summary info, default width in tests is very low + result = pytester.runpytest("-vv") + assert result.ret == 1 + backport_str = "exceptiongroup." if sys.version_info < (3, 11) else "" + result.stdout.fnmatch_lines( + [ + "*= short test summary info =*", + ( + "FAILED test_exceptiongroup_short_summary_info.py::test_base - " + "SystemExit('aaaaaaaaaa') [single exception in BaseExceptionGroup]" + ), + ( + "FAILED test_exceptiongroup_short_summary_info.py::test_nonbase - " + "ValueError('aaaaaaaaaa') [single exception in ExceptionGroup]" + ), + ( + "FAILED test_exceptiongroup_short_summary_info.py::test_nested - " + "ValueError('aaaaaaaaaa') [single exception in ExceptionGroup]" + ), + ( + "FAILED test_exceptiongroup_short_summary_info.py::test_multiple - " + f"{backport_str}ExceptionGroup: bbbbbbbbbb (2 sub-exceptions)" + ), + ( + "FAILED test_exceptiongroup_short_summary_info.py::test_nested_multiple - " + f"{backport_str}ExceptionGroup: bbbbbbbbbb (1 sub-exception)" + ), + "*= 5 failed in *", + ] + ) + + @pytest.mark.parametrize("tbstyle", ("long", "short", "auto", "line", "native")) -def test_all_entries_hidden(pytester: Pytester, tbstyle: str) -> None: +@pytest.mark.parametrize("group", (True, False), ids=("group", "bare")) +def test_all_entries_hidden(pytester: Pytester, tbstyle: str, group: bool) -> None: """Regression test for #10903.""" pytester.makepyfile( - """ + f""" + import sys + if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup def test(): __tracebackhide__ = True - 1 / 0 + raise {'ExceptionGroup("", [ValueError("bar")])' if group else 'ValueError("bar")'} """ ) result = pytester.runpytest("--tb", tbstyle) assert result.ret == 1 if tbstyle != "line": - result.stdout.fnmatch_lines(["*ZeroDivisionError: division by zero"]) + result.stdout.fnmatch_lines(["*ValueError: bar"]) if tbstyle not in ("line", "native"): result.stdout.fnmatch_lines(["All traceback entries are hidden.*"]) diff --git a/testing/code/test_source.py b/testing/code/test_source.py index a00259976c4..e413af3766e 100644 --- a/testing/code/test_source.py +++ b/testing/code/test_source.py @@ -1,21 +1,19 @@ # mypy: allow-untyped-defs -# flake8: noqa -# disable flake check on this file because some constructs are strange -# or redundant on purpose and can't be disable on a line-by-line basis +from __future__ import annotations + import inspect import linecache +from pathlib import Path import sys import textwrap -from pathlib import Path from typing import Any -from typing import Dict -import pytest from _pytest._code import Code from _pytest._code import Frame from _pytest._code import getfslineno from _pytest._code import Source from _pytest.pathlib import import_path +import pytest def test_source_str_function() -> None: @@ -336,7 +334,7 @@ def test_findsource(monkeypatch) -> None: assert src is not None assert "if 1:" in str(src) - d: Dict[str, Any] = {} + d: dict[str, Any] = {} eval(co, d) src, lineno = findsource(d["x"]) assert src is not None @@ -401,7 +399,7 @@ def getstatement(lineno: int, source) -> Source: from _pytest._code.source import getstatementrange_ast src = Source(source) - ast, start, end = getstatementrange_ast(lineno, src) + _ast, start, end = getstatementrange_ast(lineno, src) return src[start:end] @@ -420,7 +418,7 @@ def test_comment_and_no_newline_at_end() -> None: "# vim: filetype=pyopencl:fdm=marker", ] ) - ast, start, end = getstatementrange_ast(1, source) + _ast, _start, end = getstatementrange_ast(1, source) assert end == 2 @@ -464,7 +462,6 @@ def test_comment_in_statement() -> None: def test_source_with_decorator() -> None: """Test behavior with Source / Code().source with regard to decorators.""" - from _pytest.compat import get_real_func @pytest.mark.foo def deco_mark(): @@ -478,14 +475,14 @@ def deco_mark(): def deco_fixture(): assert False - src = inspect.getsource(deco_fixture) + src = inspect.getsource(deco_fixture._get_wrapped_function()) assert src == " @pytest.fixture\n def deco_fixture():\n assert False\n" - # currently Source does not unwrap decorators, testing the - # existing behavior here for explicitness, but perhaps we should revisit/change this - # in the future - assert str(Source(deco_fixture)).startswith("@functools.wraps(function)") + # Make sure the decorator is not a wrapped function + assert not str(Source(deco_fixture)).startswith("@functools.wraps(function)") assert ( - textwrap.indent(str(Source(get_real_func(deco_fixture))), " ") + "\n" == src + textwrap.indent(str(Source(deco_fixture._get_wrapped_function())), " ") + + "\n" + == src ) diff --git a/testing/conftest.py b/testing/conftest.py index 24e5d183094..663c9d80b3e 100644 --- a/testing/conftest.py +++ b/testing/conftest.py @@ -1,10 +1,12 @@ # mypy: allow-untyped-defs from __future__ import annotations -import dataclasses +from collections.abc import Generator +import importlib.metadata import re import sys -from typing import Generator + +from packaging.version import Version from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester @@ -46,7 +48,7 @@ def reset_colors(monkeypatch: pytest.MonkeyPatch) -> None: @pytest.hookimpl(wrapper=True, tryfirst=True) -def pytest_collection_modifyitems(items) -> Generator[None, None, None]: +def pytest_collection_modifyitems(items) -> Generator[None]: """Prefer faster tests. Use a hook wrapper to do this in the beginning, so e.g. --ff still works @@ -109,7 +111,7 @@ def write(self, msg, **kw): def _write_source(self, lines, indents=()): if not indents: indents = [""] * len(lines) - for indent, line in zip(indents, lines): + for indent, line in zip(indents, lines, strict=True): self.line(indent + line) def line(self, line, **kw): @@ -119,8 +121,8 @@ def markup(self, text, **kw): return text def get_write_msg(self, idx): - flag, msg = self.lines[idx] - assert flag == TWMock.WRITE + assert self.lines[idx][0] == TWMock.WRITE + msg = self.lines[idx][1] return msg fullwidth = 80 @@ -168,6 +170,9 @@ def color_mapping(): Used by tests which check the actual colors output by pytest. """ + # https://github.com/pygments/pygments/commit/d24e272894a56a98b1b718d9ac5fabc20124882a + pygments_version = Version(importlib.metadata.version("pygments")) + pygments_has_kwspace_hl = pygments_version >= Version("2.19") class ColorMapping: COLORS = { @@ -180,6 +185,7 @@ class ColorMapping: "bold": "\x1b[1m", "reset": "\x1b[0m", "kw": "\x1b[94m", + "kwspace": "\x1b[90m \x1b[39;49;00m" if pygments_has_kwspace_hl else " ", "hl-reset": "\x1b[39;49;00m", "function": "\x1b[92m", "number": "\x1b[94m", @@ -226,24 +232,20 @@ def mock_timing(monkeypatch: MonkeyPatch): Time is static, and only advances through `sleep` calls, thus tests might sleep over large numbers and obtain accurate time() calls at the end, making tests reliable and instant. """ + from _pytest.timing import MockTiming - @dataclasses.dataclass - class MockTiming: - _current_time: float = 1590150050.0 - - def sleep(self, seconds: float) -> None: - self._current_time += seconds - - def time(self) -> float: - return self._current_time + result = MockTiming() + result.patch(monkeypatch) + return result - def patch(self) -> None: - from _pytest import timing - monkeypatch.setattr(timing, "sleep", self.sleep) - monkeypatch.setattr(timing, "time", self.time) - monkeypatch.setattr(timing, "perf_counter", self.time) +@pytest.fixture(autouse=True) +def remove_ci_env_var(monkeypatch: MonkeyPatch, request: pytest.FixtureRequest) -> None: + """Make the test insensitive if it is running in CI or not. - result = MockTiming() - result.patch() - return result + Use `@pytest.mark.keep_ci_var` in a test to avoid applying this fixture, letting the test + see the real `CI` variable (if present). + """ + has_keep_ci_mark = request.node.get_closest_marker("keep_ci_var") is not None + if not has_keep_ci_mark: + monkeypatch.delenv("CI", raising=False) diff --git a/testing/example_scripts/dataclasses/test_compare_dataclasses_with_custom_eq.py b/testing/example_scripts/dataclasses/test_compare_dataclasses_with_custom_eq.py index b787cb39ee2..5ae9a02f99b 100644 --- a/testing/example_scripts/dataclasses/test_compare_dataclasses_with_custom_eq.py +++ b/testing/example_scripts/dataclasses/test_compare_dataclasses_with_custom_eq.py @@ -10,8 +10,8 @@ class SimpleDataObject: field_a: int = field() field_b: str = field() - def __eq__(self, __o: object) -> bool: - return super().__eq__(__o) + def __eq__(self, o: object, /) -> bool: + return super().__eq__(o) left = SimpleDataObject(1, "b") right = SimpleDataObject(1, "c") diff --git a/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py b/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py index 112d1e05f27..0185628c3a0 100644 --- a/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py +++ b/testing/example_scripts/fixtures/fill_fixtures/test_conftest_funcargs_only_available_in_subdir/sub2/conftest.py @@ -1,9 +1,11 @@ # mypy: allow-untyped-defs from __future__ import annotations +from _pytest.fixtures import FixtureLookupError import pytest @pytest.fixture def arg2(request): - pytest.raises(Exception, request.getfixturevalue, "arg1") + with pytest.raises(FixtureLookupError): + request.getfixturevalue("arg1") diff --git a/testing/example_scripts/issue_519.py b/testing/example_scripts/issue_519.py index 138c07e95be..da5f5ad6aa9 100644 --- a/testing/example_scripts/issue_519.py +++ b/testing/example_scripts/issue_519.py @@ -23,13 +23,13 @@ def checked_order(): assert order == [ ("issue_519.py", "fix1", "arg1v1"), ("test_one[arg1v1-arg2v1]", "fix2", "arg2v1"), - ("test_one[arg1v1-arg2v2]", "fix2", "arg2v2"), ("test_two[arg1v1-arg2v1]", "fix2", "arg2v1"), + ("test_one[arg1v1-arg2v2]", "fix2", "arg2v2"), ("test_two[arg1v1-arg2v2]", "fix2", "arg2v2"), ("issue_519.py", "fix1", "arg1v2"), ("test_one[arg1v2-arg2v1]", "fix2", "arg2v1"), - ("test_one[arg1v2-arg2v2]", "fix2", "arg2v2"), ("test_two[arg1v2-arg2v1]", "fix2", "arg2v1"), + ("test_one[arg1v2-arg2v2]", "fix2", "arg2v2"), ("test_two[arg1v2-arg2v2]", "fix2", "arg2v2"), ] diff --git a/testing/io/test_terminalwriter.py b/testing/io/test_terminalwriter.py index 043c2d1d904..9aa89da0e41 100644 --- a/testing/io/test_terminalwriter.py +++ b/testing/io/test_terminalwriter.py @@ -1,13 +1,14 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator import io +from io import StringIO import os from pathlib import Path import re import shutil import sys -from typing import Generator from unittest import mock from _pytest._io import terminalwriter @@ -67,9 +68,8 @@ def test_terminalwriter_not_unicode() -> None: class TestTerminalWriter: @pytest.fixture(params=["path", "stringio"]) - def tw( - self, request, tmp_path: Path - ) -> Generator[terminalwriter.TerminalWriter, None, None]: + def tw(self, request, tmp_path: Path) -> Generator[terminalwriter.TerminalWriter]: + f: io.TextIOWrapper | StringIO if request.param == "path": p = tmp_path.joinpath("tmpfile") f = open(str(p), "w+", encoding="utf8") @@ -224,6 +224,7 @@ def test_NO_COLOR_and_FORCE_COLOR( def test_empty_NO_COLOR_and_FORCE_COLOR_ignored(monkeypatch: MonkeyPatch) -> None: + monkeypatch.setenv("TERM", "xterm-256color") monkeypatch.setitem(os.environ, "NO_COLOR", "") monkeypatch.setitem(os.environ, "FORCE_COLOR", "") assert_color(True, True) diff --git a/testing/logging/test_fixture.py b/testing/logging/test_fixture.py index 0603eaba218..5f94cb8508a 100644 --- a/testing/logging/test_fixture.py +++ b/testing/logging/test_fixture.py @@ -2,8 +2,8 @@ # mypy: disallow-untyped-defs from __future__ import annotations +from collections.abc import Iterator import logging -from typing import Iterator from _pytest.logging import caplog_records_key from _pytest.pytester import Pytester diff --git a/testing/logging/test_reporting.py b/testing/logging/test_reporting.py index cf54788e246..4974532e888 100644 --- a/testing/logging/test_reporting.py +++ b/testing/logging/test_reporting.py @@ -589,7 +589,8 @@ def test_log_cli(request): ) def test_log_cli_auto_enable(pytester: Pytester, cli_args: str) -> None: """Check that live logs are enabled if --log-level or --log-cli-level is passed on the CLI. - It should not be auto enabled if the same configs are set on the INI file. + + It should not be auto enabled if the same configs are set on the configuration file. """ pytester.makepyfile( """ diff --git a/testing/plugins_integration/pytest.ini b/testing/plugins_integration/pytest.ini index 3bacdef62ab..b0eb9c3806f 100644 --- a/testing/plugins_integration/pytest.ini +++ b/testing/plugins_integration/pytest.ini @@ -1,6 +1,7 @@ [pytest] -addopts = --strict-markers +strict_markers = True asyncio_mode = strict filterwarnings = error::pytest.PytestWarning + ignore:usefixtures.* without arguments has no effect:pytest.PytestWarning ignore:.*.fspath is deprecated and will be replaced by .*.path.*:pytest.PytestDeprecationWarning diff --git a/testing/plugins_integration/requirements.txt b/testing/plugins_integration/requirements.txt index 4c1efcf32ed..f33ac01f848 100644 --- a/testing/plugins_integration/requirements.txt +++ b/testing/plugins_integration/requirements.txt @@ -1,15 +1,15 @@ -anyio[curio,trio]==4.4.0 -django==5.0.7 -pytest-asyncio==0.23.7 -pytest-bdd==7.2.0 -pytest-cov==5.0.0 -pytest-django==4.8.0 +anyio[trio]==4.11.0 +django==5.2.8 +pytest-asyncio==1.3.0 +pytest-bdd==8.1.0 +pytest-cov==7.0.0 +pytest-django==4.11.1 pytest-flakes==4.0.5 pytest-html==4.1.1 -pytest-mock==3.14.0 -pytest-rerunfailures==14.0 -pytest-sugar==1.0.0 +pytest-mock==3.15.1 +pytest-rerunfailures==16.1 +pytest-sugar==1.1.1 pytest-trio==0.8.0 -pytest-twisted==1.14.2 -twisted==24.3.0 -pytest-xvfb==3.0.0 +pytest-twisted==1.14.3 +twisted==25.5.0 +pytest-xvfb==3.1.1 diff --git a/testing/python/approx.py b/testing/python/approx.py index 69743cdbe17..481df80565c 100644 --- a/testing/python/approx.py +++ b/testing/python/approx.py @@ -2,12 +2,16 @@ from __future__ import annotations from contextlib import contextmanager +import decimal from decimal import Decimal from fractions import Fraction +from math import inf +from math import nan from math import sqrt import operator from operator import eq from operator import ne +import re from _pytest.pytester import Pytester from _pytest.python_api import _recursive_sequence_map @@ -15,9 +19,6 @@ from pytest import approx -inf, nan = float("inf"), float("nan") - - @pytest.fixture def mocked_doctest_runner(monkeypatch): import doctest @@ -76,7 +77,7 @@ def do_assert(lhs, rhs, expected_message, verbosity_level=0): ) for i, (obtained_line, expected_line) in enumerate( - zip(obtained_message, expected_message) + zip(obtained_message, expected_message, strict=True) ): regex = re.compile(expected_line) assert regex.match(obtained_line) is not None, ( @@ -90,12 +91,26 @@ def do_assert(lhs, rhs, expected_message, verbosity_level=0): return do_assert -SOME_FLOAT = r"[+-]?([0-9]*[.])?[0-9]+\s*" +SOME_FLOAT = r"[+-]?((?:([0-9]*[.])?[0-9]+(e-?[0-9]+)?)|inf|nan)\s*" SOME_INT = r"[0-9]+\s*" +SOME_TOLERANCE = rf"({SOME_FLOAT}|[+-]?[0-9]+(\.[0-9]+)?[eE][+-]?[0-9]+\s*)" class TestApprox: def test_error_messages_native_dtypes(self, assert_approx_raises_regex): + # Treat bool exactly. + assert_approx_raises_regex( + {"a": 1.0, "b": True}, + {"a": 1.0, "b": False}, + [ + "", + " comparison failed. Mismatched elements: 1 / 2:", + f" Max absolute difference: {SOME_FLOAT}", + f" Max relative difference: {SOME_FLOAT}", + r" Index\s+\| Obtained\s+\| Expected", + r".*(True|False)\s+", + ], + ) assert_approx_raises_regex( 2.0, 1.0, @@ -103,7 +118,7 @@ def test_error_messages_native_dtypes(self, assert_approx_raises_regex): "", " comparison failed", f" Obtained: {SOME_FLOAT}", - f" Expected: {SOME_FLOAT} ± {SOME_FLOAT}", + f" Expected: {SOME_FLOAT} ± {SOME_TOLERANCE}", ], ) @@ -119,9 +134,9 @@ def test_error_messages_native_dtypes(self, assert_approx_raises_regex): r" comparison failed. Mismatched elements: 2 / 3:", rf" Max absolute difference: {SOME_FLOAT}", rf" Max relative difference: {SOME_FLOAT}", - r" Index \| Obtained\s+\| Expected ", - rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", - rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected\s+", + rf" a \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_TOLERANCE}", + rf" c \| {SOME_FLOAT} \| {SOME_FLOAT} ± {SOME_TOLERANCE}", ], ) @@ -334,6 +349,11 @@ def test_repr_string(self): "approx({'b': 2.0 ± 2.0e-06, 'a': 1.0 ± 1.0e-06})", ) + assert repr(approx(42, abs=1)) == "42 ± 1" + assert repr(approx(5, rel=0.01)) == "5 ± 0.05" + assert repr(approx(24000, abs=500)) == "24000 ± 500" + assert repr(approx(1500, abs=555)) == "1500 ± 555" + def test_repr_complex_numbers(self): assert repr(approx(inf + 1j)) == "(inf+1j)" assert repr(approx(1.0j, rel=inf)) == "1j ± inf" @@ -347,7 +367,7 @@ def test_repr_complex_numbers(self): assert repr(approx(3 + 4 * 1j)) == "(3+4j) ± 5.0e-06 ∠ ±180°" # absolute tolerance is not scaled - assert repr(approx(3.3 + 4.4 * 1j, abs=0.02)) == "(3.3+4.4j) ± 2.0e-02 ∠ ±180°" + assert repr(approx(3.3 + 4.4 * 1j, abs=0.02)) == "(3.3+4.4j) ± 0.02 ∠ ±180°" @pytest.mark.parametrize( "value, expected_repr_string", @@ -371,6 +391,37 @@ def test_bool(self): assert err.match(r"approx\(\) is not supported in a boolean context") + def test_mixed_sequence(self, assert_approx_raises_regex) -> None: + """Approx should work on sequences that also contain non-numbers (#13010).""" + assert_approx_raises_regex( + [1.1, 2, "word"], + [1.0, 2, "different"], + [ + "", + r" comparison failed. Mismatched elements: 2 / 3:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected\s+", + r"\s*0\s*\|\s*1\.1\s*\|\s*1\.0\s*±\s*1\.0e\-06\s*", + r"\s*2\s*\|\s*word\s*\|\s*different\s*", + ], + verbosity_level=2, + ) + assert_approx_raises_regex( + [1.1, 2, "word"], + [1.0, 2, "word"], + [ + "", + r" comparison failed. Mismatched elements: 1 / 3:", + rf" Max absolute difference: {SOME_FLOAT}", + rf" Max relative difference: {SOME_FLOAT}", + r" Index \| Obtained\s+\| Expected\s+", + r"\s*0\s*\|\s*1\.1\s*\|\s*1\.0\s*±\s*1\.0e\-06\s*", + ], + verbosity_level=2, + ) + assert [1.1, 2, "word"] == pytest.approx([1.1, 2, "word"]) + def test_operator_overloading(self): assert 1 == approx(1, rel=1e-6, abs=1e-12) assert not (1 != approx(1, rel=1e-6, abs=1e-12)) @@ -590,6 +641,22 @@ def test_complex(self): assert approx(x, rel=5e-6, abs=0) == a assert approx(x, rel=5e-7, abs=0) != a + def test_expecting_bool(self) -> None: + assert True == approx(True) # noqa: E712 + assert False == approx(False) # noqa: E712 + assert True != approx(False) # noqa: E712 + assert True != approx(False, abs=2) # noqa: E712 + assert 1 != approx(True) + + def test_expecting_bool_numpy(self) -> None: + """Check approx comparing with numpy.bool (#13047).""" + np = pytest.importorskip("numpy") + assert np.False_ != approx(True) + assert np.True_ != approx(False) + assert np.True_ == approx(True) + assert np.False_ == approx(False) + assert np.True_ != approx(False, abs=2) + def test_list(self): actual = [1 + 1e-7, 2 + 1e-8] expected = [1, 2] @@ -655,6 +722,7 @@ def test_dict_wrong_len(self): def test_dict_nonnumeric(self): assert {"a": 1.0, "b": None} == pytest.approx({"a": 1.0, "b": None}) assert {"a": 1.0, "b": 1} != pytest.approx({"a": 1.0, "b": None}) + assert {"a": 1.0, "b": True} != pytest.approx({"a": 1.0, "b": False}, abs=2) def test_dict_vs_other(self): assert 1 != approx({"a": 0}) @@ -673,6 +741,17 @@ def test_dict_for_div_by_zero(self, assert_approx_raises_regex): ], ) + def test_dict_differing_lengths(self, assert_approx_raises_regex): + assert_approx_raises_regex( + {"a": 0}, + {"a": 0, "b": 1}, + [ + " ", + r" Impossible to compare mappings with different sizes\.", + r" Lengths: 2 and 1", + ], + ) + def test_numpy_array(self): np = pytest.importorskip("numpy") @@ -872,7 +951,7 @@ def test_nonnumeric_okay_if_equal(self, x): ], ) def test_nonnumeric_false_if_unequal(self, x): - """For nonnumeric types, x != pytest.approx(y) reduces to x != y""" + """For non-numeric types, x != pytest.approx(y) reduces to x != y""" assert "ab" != approx("abc") assert ["ab"] != approx(["abc"]) # in particular, both of these should return False @@ -948,6 +1027,11 @@ def __len__(self): expected_repr = "approx([1 ± 1.0e-06, 2 ± 2.0e-06, 3 ± 3.0e-06, 4 ± 4.0e-06])" assert repr(approx(expected)) == expected_repr + def test_decimal_approx_repr(self, monkeypatch) -> None: + monkeypatch.setitem(decimal.getcontext().traps, decimal.FloatOperation, True) + approx_obj = pytest.approx(decimal.Decimal("2.60")) + assert decimal.Decimal("2.600001") == approx_obj + def test_allow_ordered_sequences_only(self) -> None: """pytest.approx() should raise an error on unordered sequences (#9692).""" with pytest.raises(TypeError, match="only supports ordered sequences"): @@ -964,6 +1048,60 @@ def test_strange_sequence(self): assert b == pytest.approx(a, abs=2) assert b != pytest.approx(a, abs=0.5) + def test_approx_dicts_with_mismatch_on_keys(self) -> None: + """https://github.com/pytest-dev/pytest/issues/13816""" + expected = {"a": 1, "b": 3} + actual = {"a": 1, "c": 3} + + with pytest.raises( + AssertionError, + match=re.escape( + "comparison failed.\n Mappings has different keys: " + "expected dict_keys(['a', 'b']) but got dict_keys(['a', 'c'])" + ), + ): + assert actual == approx(expected) + + def test_approx_on_unordered_mapping_with_mismatch( + self, pytester: Pytester + ) -> None: + """https://github.com/pytest-dev/pytest/issues/12444""" + pytester.makepyfile( + """ + import pytest + + def test_approx_on_unordered_mapping_with_mismatch(): + expected = {"a": 1, "b": 2, "c": 3, "d": 4} + actual = {"d": 4, "c": 5, "a": 8, "b": 2} + assert actual == pytest.approx(expected) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "*comparison failed.**Mismatched elements: 2 / 4:*", + "*Max absolute difference: 7*", + "*Index | Obtained | Expected *", + "* a * | 8 * | 1 *", + "* c * | 5 * | 3 *", + ] + ) + + def test_approx_on_unordered_mapping_matching(self, pytester: Pytester) -> None: + """https://github.com/pytest-dev/pytest/issues/12444""" + pytester.makepyfile( + """ + import pytest + def test_approx_on_unordered_mapping_matching(): + expected = {"a": 1, "b": 2, "c": 3, "d": 4} + actual = {"d": 4, "c": 3, "a": 1, "b": 2} + assert actual == pytest.approx(expected) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + class MyVec3: # incomplete """sequence like""" @@ -1012,10 +1150,10 @@ def test_map_over_nested_lists(self): ] def test_map_over_mixed_sequence(self): - assert _recursive_sequence_map(sqrt, [4, (25, 64), [(49)]]) == [ + assert _recursive_sequence_map(sqrt, [4, (25, 64), [49]]) == [ 2, (5, 8), - [(7)], + [7], ] def test_map_over_sequence_like(self): diff --git a/testing/python/collect.py b/testing/python/collect.py index 06386611279..b26931007d9 100644 --- a/testing/python/collect.py +++ b/testing/python/collect.py @@ -263,6 +263,43 @@ def prop(self): result = pytester.runpytest() assert result.ret == ExitCode.NO_TESTS_COLLECTED + def test_does_not_discover_properties(self, pytester: Pytester) -> None: + """Regression test for #12446.""" + pytester.makepyfile( + """\ + class TestCase: + @property + def oops(self): + raise SystemExit('do not call me!') + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + def test_does_not_discover_instance_descriptors(self, pytester: Pytester) -> None: + """Regression test for #12446.""" + pytester.makepyfile( + """\ + # not `@property`, but it acts like one + # this should cover the case of things like `@cached_property` / etc. + class MyProperty: + def __init__(self, func): + self._func = func + def __get__(self, inst, owner): + if inst is None: + return self + else: + return self._func.__get__(inst, owner)() + + class TestCase: + @MyProperty + def oops(self): + raise SystemExit('do not call me!') + """ + ) + result = pytester.runpytest() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + def test_abstract_class_is_not_collected(self, pytester: Pytester) -> None: """Regression test for #12275 (non-unittest version).""" pytester.makepyfile( @@ -1038,7 +1075,8 @@ class TestTracebackCutting: def test_skip_simple(self): with pytest.raises(pytest.skip.Exception) as excinfo: pytest.skip("xxx") - assert excinfo.traceback[-1].frame.code.name == "skip" + if sys.version_info >= (3, 11): + assert excinfo.traceback[-1].frame.code.raw.co_qualname == "_Skip.__call__" assert excinfo.traceback[-1].ishidden(excinfo) assert excinfo.traceback[-2].frame.code.name == "test_skip_simple" assert not excinfo.traceback[-2].ishidden(excinfo) @@ -1235,10 +1273,10 @@ def test_bar(self): ) classcol = pytester.collect_by_name(modcol, "TestClass") assert isinstance(classcol, Class) - path, lineno, msg = classcol.reportinfo() + _path, _lineno, _msg = classcol.reportinfo() func = next(iter(classcol.collect())) assert isinstance(func, Function) - path, lineno, msg = func.reportinfo() + _path, _lineno, _msg = func.reportinfo() def test_customized_python_discovery(pytester: Pytester) -> None: @@ -1451,7 +1489,7 @@ def test_package_collection_init_given_as_argument(pytester: Pytester) -> None: Module, not the entire package. """ p = pytester.copy_example("collect/package_init_given_as_arg") - items, hookrecorder = pytester.inline_genitems(p / "pkg" / "__init__.py") + items, _hookrecorder = pytester.inline_genitems(p / "pkg" / "__init__.py") assert len(items) == 1 assert items[0].name == "test_init" diff --git a/testing/python/fixtures.py b/testing/python/fixtures.py index 8d2646309a8..6a65dce3c4d 100644 --- a/testing/python/fixtures.py +++ b/testing/python/fixtures.py @@ -1,6 +1,7 @@ # mypy: allow-untyped-defs from __future__ import annotations +from itertools import zip_longest import os from pathlib import Path import sys @@ -48,7 +49,23 @@ class A: def f(self, arg1, arg2="hello"): raise NotImplementedError() + def g(self, /, arg1, arg2="hello"): + raise NotImplementedError() + + def h(self, *, arg1, arg2="hello"): + raise NotImplementedError() + + def j(self, arg1, *, arg2, arg3="hello"): + raise NotImplementedError() + + def k(self, /, arg1, *, arg2, arg3="hello"): + raise NotImplementedError() + assert getfuncargnames(A().f) == ("arg1",) + assert getfuncargnames(A().g) == ("arg1",) + assert getfuncargnames(A().h) == ("arg1",) + assert getfuncargnames(A().j) == ("arg1", "arg2") + assert getfuncargnames(A().k) == ("arg1", "arg2") def test_getfuncargnames_staticmethod(): @@ -733,7 +750,7 @@ def test_request_garbage(self, pytester: Pytester) -> None: """ import sys import pytest - from _pytest.fixtures import PseudoFixtureDef + from _pytest.fixtures import RequestFixtureDef import gc @pytest.fixture(autouse=True) @@ -746,7 +763,7 @@ def something(request): try: gc.collect() - leaked = [x for _ in gc.garbage if isinstance(_, PseudoFixtureDef)] + leaked = [x for _ in gc.garbage if isinstance(_, RequestFixtureDef)] assert leaked == [] finally: gc.set_debug(original) @@ -1133,7 +1150,7 @@ def test_session_scoped_unavailable_attributes(self, session_request, name): class TestRequestMarking: def test_applymarker(self, pytester: Pytester) -> None: - item1, item2 = pytester.getitems( + item1, _item2 = pytester.getitems( """ import pytest @@ -1435,6 +1452,23 @@ def test_two(self): reprec = pytester.inline_run() reprec.assertoutcome(passed=2) + def test_empty_usefixtures_marker(self, pytester: Pytester) -> None: + """Empty usefixtures() marker issues a warning (#12439).""" + pytester.makepyfile( + """ + import pytest + + @pytest.mark.usefixtures() + def test_one(): + assert 1 == 1 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + "*PytestWarning: usefixtures() in test_empty_usefixtures_marker.py::test_one" + " without arguments has no effect" + ) + def test_usefixtures_ini(self, pytester: Pytester) -> None: pytester.makeini( """ @@ -1589,6 +1623,63 @@ def teardown_module(): result = pytester.runpytest() result.stdout.no_fnmatch_line("* ERROR at teardown *") + def test_unwrapping_pytest_fixture(self, pytester: Pytester) -> None: + """Ensure the unwrap method on `FixtureFunctionDefinition` correctly wraps and unwraps methods and functions""" + pytester.makepyfile( + """ + import pytest + import inspect + + class FixtureFunctionDefTestClass: + def __init__(self) -> None: + self.i = 10 + + @pytest.fixture + def fixture_function_def_test_method(self): + return self.i + + + @pytest.fixture + def fixture_function_def_test_func(): + return 9 + + + def test_get_wrapped_func_returns_method(): + obj = FixtureFunctionDefTestClass() + wrapped_function_result = ( + obj.fixture_function_def_test_method._get_wrapped_function() + ) + assert inspect.ismethod(wrapped_function_result) + assert wrapped_function_result() == 10 + + + def test_get_wrapped_func_returns_function(): + assert fixture_function_def_test_func._get_wrapped_function()() == 9 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=2) + + def test_fixture_wrapped_looks_liked_wrapped_function( + self, pytester: Pytester + ) -> None: + """Ensure that `FixtureFunctionDefinition` behaves like the function it wrapped.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def fixture_function_def_test_func(): + return 9 + fixture_function_def_test_func.__doc__ = "documentation" + + def test_fixture_has_same_doc(): + assert fixture_function_def_test_func.__doc__ == "documentation" + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + class TestFixtureManagerParseFactories: @pytest.fixture @@ -2241,14 +2332,14 @@ def test_ordering_dependencies_torndown_first( ) -> None: """#226""" pytester.makepyfile( - """ + f""" import pytest values = [] - @pytest.fixture(%(param1)s) + @pytest.fixture({param1}) def arg1(request): request.addfinalizer(lambda: values.append("fin1")) values.append("new1") - @pytest.fixture(%(param2)s) + @pytest.fixture({param2}) def arg2(request, arg1): request.addfinalizer(lambda: values.append("fin2")) values.append("new2") @@ -2257,8 +2348,7 @@ def test_arg(arg2): pass def test_check(): assert values == ["new1", "new2", "fin2", "fin1"] - """ # noqa: UP031 (python syntax issues) - % locals() + """ ) reprec = pytester.inline_run("-s") reprec.assertoutcome(passed=2) @@ -2954,7 +3044,7 @@ def test_4(modarg, arg): ] import pprint - pprint.pprint(list(zip(values, expected))) + pprint.pprint(list(zip_longest(values, expected))) assert values == expected def test_parametrized_fixture_teardown_order(self, pytester: Pytester) -> None: @@ -2996,7 +3086,7 @@ def test_finish(): *3 passed* """ ) - result.stdout.no_fnmatch_line("*error*") + assert result.ret == 0 def test_fixture_finalizer(self, pytester: Pytester) -> None: pytester.makeconftest( @@ -3138,21 +3228,21 @@ def test_finalizer_order_on_parametrization( ) -> None: """#246""" pytester.makepyfile( - """ + f""" import pytest values = [] - @pytest.fixture(scope=%(scope)r, params=["1"]) + @pytest.fixture(scope={scope!r}, params=["1"]) def fix1(request): return request.param - @pytest.fixture(scope=%(scope)r) + @pytest.fixture(scope={scope!r}) def fix2(request, base): def cleanup_fix2(): assert not values, "base should not have been finalized" request.addfinalizer(cleanup_fix2) - @pytest.fixture(scope=%(scope)r) + @pytest.fixture(scope={scope!r}) def base(request, fix1): def cleanup_base(): values.append("fin_base") @@ -3165,8 +3255,7 @@ def test_baz(base, fix2): pass def test_other(): pass - """ # noqa: UP031 (python syntax issues) - % {"scope": scope} + """ ) reprec = pytester.inline_run("-lvs") reprec.assertoutcome(passed=3) @@ -3352,42 +3441,40 @@ class TestRequestScopeAccess: def test_setup(self, pytester: Pytester, scope, ok, error) -> None: pytester.makepyfile( - """ + f""" import pytest - @pytest.fixture(scope=%r, autouse=True) + @pytest.fixture(scope={scope!r}, autouse=True) def myscoped(request): - for x in %r: + for x in {ok.split()}: assert hasattr(request, x) - for x in %r: + for x in {error.split()}: pytest.raises(AttributeError, lambda: getattr(request, x)) assert request.session assert request.config def test_func(): pass - """ # noqa: UP031 (python syntax issues) - % (scope, ok.split(), error.split()) + """ ) reprec = pytester.inline_run("-l") reprec.assertoutcome(passed=1) def test_funcarg(self, pytester: Pytester, scope, ok, error) -> None: pytester.makepyfile( - """ + f""" import pytest - @pytest.fixture(scope=%r) + @pytest.fixture(scope={scope!r}) def arg(request): - for x in %r: + for x in {ok.split()!r}: assert hasattr(request, x) - for x in %r: + for x in {error.split()!r}: pytest.raises(AttributeError, lambda: getattr(request, x)) assert request.session assert request.config def test_func(arg): pass - """ # noqa: UP031 (python syntax issues) - % (scope, ok.split(), error.split()) + """ ) reprec = pytester.inline_run() reprec.assertoutcome(passed=1) @@ -4338,7 +4425,7 @@ def test_func(self, f2, f1, m2): assert request.fixturenames == "s1 p1 m1 m2 c1 f2 f1".split() def test_parametrized_package_scope_reordering(self, pytester: Pytester) -> None: - """A paramaterized package-scoped fixture correctly reorders items to + """A parameterized package-scoped fixture correctly reorders items to minimize setups & teardowns. Regression test for #12328. @@ -4509,6 +4596,21 @@ def fixt(): ) +def test_fixture_class(pytester: Pytester) -> None: + """Check if an error is raised when using @pytest.fixture on a class.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + class A: + pass + """ + ) + result = pytester.runpytest() + result.assert_outcomes(errors=1) + + def test_fixture_param_shadowing(pytester: Pytester) -> None: """Parametrized arguments would be shadowed if a fixture with the same name also exists (#5036)""" pytester.makepyfile( @@ -4920,3 +5022,346 @@ def test_result(): ) result = pytester.runpytest() assert result.ret == 0 + + +def test_parametrized_fixture_scope_allowed(pytester: Pytester) -> None: + """ + Make sure scope from parametrize does not affect fixture's ability to be + depended upon. + + Regression test for #13248 + """ + pytester.makepyfile( + """ + import pytest + + @pytest.fixture(scope="session") + def my_fixture(request): + return getattr(request, "param", None) + + @pytest.fixture(scope="session") + def another_fixture(my_fixture): + return my_fixture + + @pytest.mark.parametrize("my_fixture", ["a value"], indirect=True, scope="function") + def test_foo(another_fixture): + assert another_fixture == "a value" + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_collect_positional_only(pytester: Pytester) -> None: + """Support the collection of tests with positional-only arguments (#13376).""" + pytester.makepyfile( + """ + import pytest + + class Test: + @pytest.fixture + def fix(self): + return 1 + + def test_method(self, /, fix): + assert fix == 1 + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=1) + + +def test_parametrization_dependency_pruning(pytester: Pytester) -> None: + """Test that when a fixture is dynamically shadowed by parameterization, it + is properly pruned and not executed.""" + pytester.makepyfile( + """ + import pytest + + + # This fixture should never run because shadowed_fixture is parametrized. + @pytest.fixture + def boom(): + raise RuntimeError("BOOM!") + + + # This fixture is shadowed by metafunc.parametrize in pytest_generate_tests. + @pytest.fixture + def shadowed_fixture(boom): + return "fixture_value" + + + # Dynamically parametrize shadowed_fixture, replacing the fixture with direct values. + def pytest_generate_tests(metafunc): + if "shadowed_fixture" in metafunc.fixturenames: + metafunc.parametrize("shadowed_fixture", ["param1", "param2"]) + + + # This test should receive shadowed_fixture as a parametrized value, and + # boom should not explode. + def test_shadowed(shadowed_fixture): + assert shadowed_fixture in ["param1", "param2"] + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=2) + + +def test_fixture_closure_with_overrides(pytester: Pytester) -> None: + """Test that an item's static fixture closure properly includes transitive + dependencies through overridden fixtures (#13773).""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def db(): pass + + @pytest.fixture + def app(db): pass + """ + ) + pytester.makepyfile( + """ + import pytest + + # Overrides conftest-level `app` and requests it. + @pytest.fixture + def app(app): pass + + class TestClass: + # Overrides module-level `app` and requests it. + @pytest.fixture + def app(self, app): pass + + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert 'db' in request.fixturenames + assert 'db' in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + +def test_fixture_closure_with_overrides_and_intermediary(pytester: Pytester) -> None: + """Test that an item's static fixture closure properly includes transitive + dependencies through overridden fixtures (#13773). + + A more complicated case than test_fixture_closure_with_overrides, adds an + intermediary so the override chain is not direct. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def db(): pass + + @pytest.fixture + def app(db): pass + + @pytest.fixture + def intermediate(app): pass + """ + ) + pytester.makepyfile( + """ + import pytest + + # Overrides conftest-level `app` and requests it. + @pytest.fixture + def app(intermediate): pass + + class TestClass: + # Overrides module-level `app` and requests it. + @pytest.fixture + def app(self, app): pass + + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert 'db' in request.fixturenames + assert 'db' in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + +def test_fixture_closure_with_broken_override_chain(pytester: Pytester) -> None: + """Test that an item's static fixture closure properly includes transitive + dependencies through overridden fixtures (#13773). + + A more complicated case than test_fixture_closure_with_overrides, one of the + fixtures in the chain doesn't call its super, so it shouldn't be included. + """ + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def db(): pass + + @pytest.fixture + def app(db): pass + """ + ) + pytester.makepyfile( + """ + import pytest + + # Overrides conftest-level `app` and *doesn't* request it. + @pytest.fixture + def app(): pass + + class TestClass: + # Overrides module-level `app` and requests it. + @pytest.fixture + def app(self, app): pass + + def test_something(self, request, app): + # Both dynamic and static fixture closures should include 'db'. + assert 'db' not in request.fixturenames + assert 'db' not in request.node.fixturenames + # No dynamic dependencies, should be equal. + assert set(request.fixturenames) == set(request.node.fixturenames) + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + +def test_fixture_closure_handles_circular_dependencies(pytester: Pytester) -> None: + """Test that getfixtureclosure properly handles circular dependencies. + + The test will error in the runtest phase due to the fixture loop, + but the closure computation still completes. + """ + pytester.makepyfile( + """ + import pytest + + # Direct circular dependency. + @pytest.fixture + def fix_a(fix_b): pass + + @pytest.fixture + def fix_b(fix_a): pass + + # Indirect circular dependency through multiple fixtures. + @pytest.fixture + def fix_x(fix_y): pass + + @pytest.fixture + def fix_y(fix_z): pass + + @pytest.fixture + def fix_z(fix_x): pass + + def test_circular_deps(fix_a, fix_x): + pass + """ + ) + items, _hookrec = pytester.inline_genitems() + assert isinstance(items[0], Function) + assert items[0].fixturenames == ["fix_a", "fix_x", "fix_b", "fix_y", "fix_z"] + + +def test_fixture_closure_handles_diamond_dependencies(pytester: Pytester) -> None: + """Test that getfixtureclosure properly handles diamond dependencies.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def db(): pass + + @pytest.fixture + def user(db): pass + + @pytest.fixture + def session(db): pass + + @pytest.fixture + def app(user, session): pass + + def test_diamond_deps(request, app): + assert request.node.fixturenames == ["request", "app", "user", "db", "session"] + assert request.fixturenames == ["request", "app", "user", "db", "session"] + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + +def test_fixture_closure_with_complex_override_and_shared_deps( + pytester: Pytester, +) -> None: + """Test that shared dependencies in override chains are processed only once.""" + pytester.makeconftest( + """ + import pytest + + @pytest.fixture + def db(): pass + + @pytest.fixture + def cache(): pass + + @pytest.fixture + def settings(): pass + + @pytest.fixture + def app(db, cache, settings): pass + """ + ) + pytester.makepyfile( + """ + import pytest + + # Override app, but also directly use cache and settings. + # This creates multiple paths to the same fixtures. + @pytest.fixture + def app(app, cache, settings): pass + + class TestClass: + # Another override that uses both app and cache. + @pytest.fixture + def app(self, app, cache): pass + + def test_shared_deps(self, request, app): + assert request.node.fixturenames == ["request", "app", "db", "cache", "settings"] + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) + + +def test_fixture_closure_with_parametrize_ignore(pytester: Pytester) -> None: + """Test that getfixtureclosure properly handles parametrization argnames + which override a fixture.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def fix1(fix2): pass + + @pytest.fixture + def fix2(fix3): pass + + @pytest.fixture + def fix3(): pass + + @pytest.mark.parametrize('fix2', ['2']) + def test_it(request, fix1): + assert request.node.fixturenames == ["request", "fix1", "fix2"] + assert request.fixturenames == ["request", "fix1", "fix2"] + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1) diff --git a/testing/python/integration.py b/testing/python/integration.py index c52a683a322..d8f8d0ffae9 100644 --- a/testing/python/integration.py +++ b/testing/python/integration.py @@ -21,8 +21,8 @@ def wrap(f): def wrapped_func(x, y, z): pass - fs, lineno = getfslineno(wrapped_func) - fs2, lineno2 = getfslineno(wrap) + _fs, lineno = getfslineno(wrapped_func) + _fs2, lineno2 = getfslineno(wrap) assert lineno > lineno2, "getfslineno does not unwrap correctly" @@ -407,6 +407,23 @@ def test_params(a, b): res = pytester.runpytest("--collect-only") res.stdout.fnmatch_lines(["*spam-2*", "*ham-2*"]) + def test_param_rejects_usefixtures(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", [ + pytest.param(1, marks=[pytest.mark.usefixtures("foo")]), + ]) + def test_foo(x): + pass + """ + ) + res = pytester.runpytest("--collect-only") + res.stdout.fnmatch_lines( + ["*test_param_rejects_usefixtures.py:4*", "*pytest.param(*"] + ) + def test_function_instance(pytester: Pytester) -> None: items = pytester.getitems( diff --git a/testing/python/metafunc.py b/testing/python/metafunc.py index 2dd85607e71..20ccacf4b73 100644 --- a/testing/python/metafunc.py +++ b/testing/python/metafunc.py @@ -1,6 +1,8 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Iterator +from collections.abc import Sequence import dataclasses import itertools import re @@ -8,9 +10,6 @@ import textwrap from typing import Any from typing import cast -from typing import Dict -from typing import Iterator -from typing import Sequence import hypothesis from hypothesis import strategies @@ -20,6 +19,7 @@ from _pytest.compat import getfuncargnames from _pytest.compat import NOTSET from _pytest.outcomes import fail +from _pytest.outcomes import Failed from _pytest.pytester import Pytester from _pytest.python import Function from _pytest.python import IdMaker @@ -82,13 +82,17 @@ def func(x, y): metafunc = self.Metafunc(func) metafunc.parametrize("x", [1, 2]) - pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5, 6])) - pytest.raises(ValueError, lambda: metafunc.parametrize("x", [5, 6])) + with pytest.raises(pytest.Collector.CollectError): + metafunc.parametrize("x", [5, 6]) + with pytest.raises(pytest.Collector.CollectError): + metafunc.parametrize("x", [5, 6]) metafunc.parametrize("y", [1, 2]) - pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6])) - pytest.raises(ValueError, lambda: metafunc.parametrize("y", [5, 6])) + with pytest.raises(pytest.Collector.CollectError): + metafunc.parametrize("y", [5, 6]) + with pytest.raises(pytest.Collector.CollectError): + metafunc.parametrize("y", [5, 6]) - with pytest.raises(TypeError, match="^ids must be a callable or an iterable$"): + with pytest.raises(TypeError, match=r"^ids must be a callable or an iterable$"): metafunc.parametrize("y", [5, 6], ids=42) # type: ignore[arg-type] def test_parametrize_error_iterator(self) -> None: @@ -154,7 +158,7 @@ class DummyFixtureDef: _scope: Scope fixtures_defs = cast( - Dict[str, Sequence[fixtures.FixtureDef[object]]], + dict[str, Sequence[fixtures.FixtureDef[object]]], dict( session_fix=[DummyFixtureDef(Scope.Session)], package_fix=[DummyFixtureDef(Scope.Package)], @@ -425,7 +429,7 @@ def test_idmaker_autoname(self) -> None: def test_idmaker_with_bytes_regex(self) -> None: result = IdMaker( - ("a"), [pytest.param(re.compile(b"foo"), 1.0)], None, None, None, None, None + ("a"), [pytest.param(re.compile(b"foo"))], None, None, None, None, None ).make_unique_parameterset_ids() assert result == ["foo"] @@ -625,6 +629,37 @@ def getini(self, name): ).make_unique_parameterset_ids() assert result == [expected] + def test_idmaker_with_param_id_and_config(self) -> None: + """Unit test for expected behavior to create ids with pytest.param(id=...) and + disable_test_id_escaping_and_forfeit_all_rights_to_community_support + option (#9037). + """ + + class MockConfig: + def __init__(self, config): + self.config = config + + def getini(self, name): + return self.config[name] + + option = "disable_test_id_escaping_and_forfeit_all_rights_to_community_support" + + values: list[tuple[Any, str]] = [ + (MockConfig({option: True}), "ação"), + (MockConfig({option: False}), "a\\xe7\\xe3o"), + ] + for config, expected in values: + result = IdMaker( + ("a",), + [pytest.param("string", id="ação")], + None, + None, + config, + None, + None, + ).make_unique_parameterset_ids() + assert result == [expected] + def test_idmaker_duplicated_empty_str(self) -> None: """Regression test for empty strings parametrized more than once (#11563).""" result = IdMaker( @@ -1005,14 +1040,14 @@ def test3(arg1): result.stdout.re_match_lines( [ r" ", - r" ", r" ", + r" ", + r" ", r" ", + r" ", r" ", - r" ", r" ", r" ", - r" ", r" ", ] ) @@ -1408,13 +1443,13 @@ def test_parametrize_scope_overrides( self, pytester: Pytester, scope: str, length: int ) -> None: pytester.makepyfile( - """ + f""" import pytest values = [] def pytest_generate_tests(metafunc): if "arg" in metafunc.fixturenames: metafunc.parametrize("arg", [1,2], indirect=True, - scope=%r) + scope={scope!r}) @pytest.fixture def arg(request): values.append(request.param) @@ -1424,9 +1459,8 @@ def test_hello(arg): def test_world(arg): assert arg in (1,2) def test_checklength(): - assert len(values) == %d + assert len(values) == {length} """ - % (scope, length) ) reprec = pytester.inline_run() reprec.assertoutcome(passed=5) @@ -2114,3 +2148,127 @@ def test_converted_to_str(a, b): "*= 6 passed in *", ] ) + + +class TestHiddenParam: + """Test that pytest.HIDDEN_PARAM works""" + + def test_parametrize_ids(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + + @pytest.mark.parametrize( + ("foo", "bar"), + [ + ("a", "x"), + ("b", "y"), + ("c", "z"), + ], + ids=["paramset1", pytest.HIDDEN_PARAM, "paramset3"], + ) + def test_func(foo, bar): + pass + """ + ) + names = [item.name for item in items] + assert names == [ + "test_func[paramset1]", + "test_func", + "test_func[paramset3]", + ] + + def test_param_id(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + + @pytest.mark.parametrize( + ("foo", "bar"), + [ + pytest.param("a", "x", id="paramset1"), + pytest.param("b", "y", id=pytest.HIDDEN_PARAM), + ("c", "z"), + ], + ) + def test_func(foo, bar): + pass + """ + ) + names = [item.name for item in items] + assert names == [ + "test_func[paramset1]", + "test_func", + "test_func[c-z]", + ] + + def test_multiple_hidden_param_is_forbidden(self, pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize( + ("foo", "bar"), + [ + ("a", "x"), + ("b", "y"), + ], + ids=[pytest.HIDDEN_PARAM, pytest.HIDDEN_PARAM], + ) + def test_func(foo, bar): + pass + """ + ) + result = pytester.runpytest("--collect-only") + result.stdout.fnmatch_lines( + [ + "collected 0 items / 1 error", + "", + "*= ERRORS =*", + "*_ ERROR collecting test_multiple_hidden_param_is_forbidden.py _*", + "E Failed: In test_func: multiple instances of HIDDEN_PARAM cannot be used " + "in the same parametrize call, because the tests names need to be unique.", + "*! Interrupted: 1 error during collection !*", + "*= no tests collected, 1 error in *", + ] + ) + + def test_multiple_hidden_param_is_forbidden_idmaker(self) -> None: + id_maker = IdMaker( + ("foo", "bar"), + [pytest.param("a", "x"), pytest.param("b", "y")], + None, + [pytest.HIDDEN_PARAM, pytest.HIDDEN_PARAM], + None, + "some_node_id", + None, + ) + expected = "In some_node_id: multiple instances of HIDDEN_PARAM" + with pytest.raises(Failed, match=expected): + id_maker.make_unique_parameterset_ids() + + def test_multiple_parametrize(self, pytester: Pytester) -> None: + items = pytester.getitems( + """ + import pytest + + @pytest.mark.parametrize( + "bar", + ["x", "y"], + ) + @pytest.mark.parametrize( + "foo", + ["a", "b"], + ids=["a", pytest.HIDDEN_PARAM], + ) + def test_func(foo, bar): + pass + """ + ) + names = [item.name for item in items] + assert names == [ + "test_func[a-x]", + "test_func[a-y]", + "test_func[x]", + "test_func[y]", + ] diff --git a/testing/python/raises.py b/testing/python/raises.py index 2011c81615e..c9d57918a83 100644 --- a/testing/python/raises.py +++ b/testing/python/raises.py @@ -1,14 +1,20 @@ # mypy: allow-untyped-defs from __future__ import annotations +import io import re import sys from _pytest.outcomes import Failed from _pytest.pytester import Pytester +from _pytest.warning_types import PytestWarning import pytest +def wrap_escape(s: str) -> str: + return "^" + re.escape(s) + "$" + + class TestRaises: def test_check_callable(self) -> None: with pytest.raises(TypeError, match=r".* must be callable"): @@ -23,13 +29,32 @@ def test_raises_function(self): assert "invalid literal" in str(excinfo.value) def test_raises_does_not_allow_none(self): - with pytest.raises(ValueError, match="Expected an exception type or"): + with pytest.raises( + ValueError, + match=wrap_escape("You must specify at least one parameter to match on."), + ): # We're testing that this invalid usage gives a helpful error, # so we can ignore Mypy telling us that None is invalid. pytest.raises(expected_exception=None) # type: ignore + # it's unclear if this message is helpful, and if it is, should it trigger more + # liberally? Usually you'd get a TypeError here + def test_raises_false_and_arg(self): + with pytest.raises( + ValueError, + match=wrap_escape( + "Expected an exception type or a tuple of exception types, but got `False`. " + "Raising exceptions is already understood as failing the test, so you don't need " + "any special code to say 'this should never raise an exception'." + ), + ): + pytest.raises(False, int) # type: ignore[call-overload] + def test_raises_does_not_allow_empty_tuple(self): - with pytest.raises(ValueError, match="Expected an exception type or"): + with pytest.raises( + ValueError, + match=wrap_escape("You must specify at least one parameter to match on."), + ): pytest.raises(expected_exception=()) def test_raises_callable_no_exception(self) -> None: @@ -181,7 +206,9 @@ def test_no_raise_message(self) -> None: else: assert False, "Expected pytest.raises.Exception" - @pytest.mark.parametrize("method", ["function", "function_match", "with"]) + @pytest.mark.parametrize( + "method", ["function", "function_match", "with", "with_raisesexc", "with_group"] + ) def test_raises_cyclic_reference(self, method): """Ensure pytest.raises does not leave a reference cycle (#1965).""" import gc @@ -197,9 +224,17 @@ def __call__(self): pytest.raises(ValueError, t) elif method == "function_match": pytest.raises(ValueError, t).match("^$") - else: + elif method == "with": with pytest.raises(ValueError): t() + elif method == "with_raisesexc": + with pytest.RaisesExc(ValueError): + t() + elif method == "with_group": + with pytest.RaisesGroup(ValueError, allow_unwrapped=True): + t() + else: # pragma: no cover + raise AssertionError("bad parametrization") # ensure both forms of pytest.raises don't leave exceptions in sys.exc_info() assert sys.exc_info() == (None, None, None) @@ -218,10 +253,10 @@ def test_raises_match(self) -> None: msg = "with base 16" expr = ( "Regex pattern did not match.\n" - f" Regex: {msg!r}\n" - " Input: \"invalid literal for int() with base 10: 'asdf'\"" + f" Expected regex: {msg!r}\n" + f" Actual message: \"invalid literal for int() with base 10: 'asdf'\"" ) - with pytest.raises(AssertionError, match="(?m)" + re.escape(expr)): + with pytest.raises(AssertionError, match="^" + re.escape(expr) + "$"): with pytest.raises(ValueError, match=msg): int("asdf", base=10) @@ -239,12 +274,25 @@ def tfunc(match): pytest.raises(ValueError, tfunc, match="asdf").match("match=asdf") pytest.raises(ValueError, tfunc, match="").match("match=") + # empty string matches everything, which is probably not what the user wants + with pytest.warns( + PytestWarning, + match=wrap_escape( + "matching against an empty string will *always* pass. If you want to check for an empty message you " + "need to pass '^$'. If you don't want to match you should pass `None` or leave out the parameter." + ), + ): + pytest.raises(match="") + def test_match_failure_string_quoting(self): with pytest.raises(AssertionError) as excinfo: with pytest.raises(AssertionError, match="'foo"): raise AssertionError("'bar") (msg,) = excinfo.value.args - assert msg == '''Regex pattern did not match.\n Regex: "'foo"\n Input: "'bar"''' + assert ( + msg + == '''Regex pattern did not match.\n Expected regex: "'foo"\n Actual message: "'bar"''' + ) def test_match_failure_exact_string_message(self): message = "Oh here is a message with (42) numbers in parameters" @@ -254,8 +302,8 @@ def test_match_failure_exact_string_message(self): (msg,) = excinfo.value.args assert msg == ( "Regex pattern did not match.\n" - " Regex: 'Oh here is a message with (42) numbers in parameters'\n" - " Input: 'Oh here is a message with (42) numbers in parameters'\n" + " Expected regex: 'Oh here is a message with (42) numbers in parameters'\n" + " Actual message: 'Oh here is a message with (42) numbers in parameters'\n" " Did you mean to `re.escape()` the regex?" ) @@ -265,7 +313,10 @@ def test_raises_match_wrong_type(self): pytest should throw the unexpected exception - the pattern match is not really relevant if we got a different exception. """ - with pytest.raises(ValueError): + with pytest.raises( + ValueError, + match=wrap_escape("invalid literal for int() with base 10: 'asdf'"), + ): with pytest.raises(IndexError, match="nomatch"): int("asdf") @@ -301,39 +352,58 @@ def __class__(self): assert "via __class__" in excinfo.value.args[0] def test_raises_context_manager_with_kwargs(self): - with pytest.raises(TypeError) as excinfo: + with pytest.raises(expected_exception=ValueError): + raise ValueError + with pytest.raises( + TypeError, + match=wrap_escape( + "Unexpected keyword arguments passed to pytest.raises: foo\n" + "Use context-manager form instead?" + ), + ): with pytest.raises(OSError, foo="bar"): # type: ignore[call-overload] pass - assert "Unexpected keyword arguments" in str(excinfo.value) def test_expected_exception_is_not_a_baseexception(self) -> None: - with pytest.raises(TypeError) as excinfo: + with pytest.raises( + TypeError, + match=wrap_escape("Expected a BaseException type, but got 'str'"), + ): with pytest.raises("hello"): # type: ignore[call-overload] pass # pragma: no cover - assert "must be a BaseException type, not str" in str(excinfo.value) class NotAnException: pass - with pytest.raises(TypeError) as excinfo: + with pytest.raises( + ValueError, + match=wrap_escape( + "Expected a BaseException type, but got 'NotAnException'" + ), + ): with pytest.raises(NotAnException): # type: ignore[type-var] pass # pragma: no cover - assert "must be a BaseException type, not NotAnException" in str(excinfo.value) - with pytest.raises(TypeError) as excinfo: + with pytest.raises( + TypeError, + match=wrap_escape("Expected a BaseException type, but got 'str'"), + ): with pytest.raises(("hello", NotAnException)): # type: ignore[arg-type] pass # pragma: no cover - assert "must be a BaseException type, not str" in str(excinfo.value) def test_issue_11872(self) -> None: """Regression test for #11872. - urllib.error.HTTPError on Python<=3.9 raises KeyError instead of - AttributeError on invalid attribute access. + urllib.error.HTTPError on some Python 3.10/11 minor releases raises + KeyError instead of AttributeError on invalid attribute access. https://github.com/python/cpython/issues/98778 """ + from email.message import Message from urllib.error import HTTPError - with pytest.raises(HTTPError, match="Not Found"): - raise HTTPError(code=404, msg="Not Found", fp=None, hdrs=None, url="") # type: ignore [arg-type] + with pytest.raises(HTTPError, match="Not Found") as exc_info: + raise HTTPError( + code=404, msg="Not Found", fp=io.BytesIO(), hdrs=Message(), url="" + ) + exc_info.value.close() # avoid a resource warning diff --git a/testing/python/raises_group.py b/testing/python/raises_group.py new file mode 100644 index 00000000000..e5e3b5cd2dc --- /dev/null +++ b/testing/python/raises_group.py @@ -0,0 +1,1357 @@ +from __future__ import annotations + +# several expected multi-line strings contain long lines. We don't wanna break them up +# as that makes it confusing to see where the line breaks are. +# ruff: noqa: E501 +from contextlib import AbstractContextManager +import re +import sys + +from _pytest._code import ExceptionInfo +from _pytest.outcomes import Failed +from _pytest.pytester import Pytester +from _pytest.raises import RaisesExc +from _pytest.raises import RaisesGroup +from _pytest.raises import repr_callable +import pytest + + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + + +def wrap_escape(s: str) -> str: + return "^" + re.escape(s) + "$" + + +def fails_raises_group(msg: str, add_prefix: bool = True) -> RaisesExc[Failed]: + assert msg[-1] != "\n", ( + "developer error, expected string should not end with newline" + ) + prefix = "Raised exception group did not match: " if add_prefix else "" + return pytest.raises(Failed, match=wrap_escape(prefix + msg)) + + +def test_raises_group() -> None: + with pytest.raises( + TypeError, + match=wrap_escape("Expected a BaseException type, but got 'int'"), + ): + RaisesExc(5) # type: ignore[call-overload] + with pytest.raises( + ValueError, + match=wrap_escape("Expected a BaseException type, but got 'int'"), + ): + RaisesExc(int) # type: ignore[type-var] + with pytest.raises( + TypeError, + match=wrap_escape( + "Expected a BaseException type, RaisesExc, or RaisesGroup, but got an exception instance: ValueError", + ), + ): + RaisesGroup(ValueError()) # type: ignore[call-overload] + with RaisesGroup(ValueError): + raise ExceptionGroup("foo", (ValueError(),)) + + with ( + fails_raises_group("`SyntaxError()` is not an instance of `ValueError`"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("foo", (SyntaxError(),)) + + # multiple exceptions + with RaisesGroup(ValueError, SyntaxError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # order doesn't matter + with RaisesGroup(SyntaxError, ValueError): + raise ExceptionGroup("foo", (ValueError(), SyntaxError())) + + # nested exceptions + with RaisesGroup(RaisesGroup(ValueError)): + raise ExceptionGroup("foo", (ExceptionGroup("bar", (ValueError(),)),)) + + with RaisesGroup( + SyntaxError, + RaisesGroup(ValueError), + RaisesGroup(RuntimeError), + ): + raise ExceptionGroup( + "foo", + ( + SyntaxError(), + ExceptionGroup("bar", (ValueError(),)), + ExceptionGroup("", (RuntimeError(),)), + ), + ) + + +def test_incorrect_number_exceptions() -> None: + # We previously gave an error saying the number of exceptions was wrong, + # but we now instead indicate excess/missing exceptions + with ( + fails_raises_group( + "1 matched exception. Unexpected exception(s): [RuntimeError()]" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (RuntimeError(), ValueError())) + + # will error if there's missing exceptions + with ( + fails_raises_group( + "1 matched exception. Too few exceptions raised, found no match for: [SyntaxError]" + ), + RaisesGroup(ValueError, SyntaxError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Too few exceptions raised!\n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " It matches `ValueError()` which was paired with `ValueError`" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", (ValueError(),)) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "Unexpected exception(s)!\n" + "The following raised exceptions did not find a match\n" + " ValueError('b'):\n" + " It matches `ValueError` which was paired with `ValueError('a')`" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", (ValueError("a"), ValueError("b"))) + + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " It matches `ValueError()` which was paired with `ValueError`\n" + "The following raised exceptions did not find a match\n" + " SyntaxError():\n" + " `SyntaxError()` is not an instance of `ValueError`" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), SyntaxError()]) + + +def test_flatten_subgroups() -> None: + # loose semantics, as with expect* + with RaisesGroup(ValueError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(), TypeError())),)) + with RaisesGroup(ValueError, TypeError, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()]), TypeError()]) + + # mixed loose is possible if you want it to be at least N deep + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): + raise ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)) + with RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)): + raise ExceptionGroup( + "", + (ExceptionGroup("", (ExceptionGroup("", (ValueError(),)),)),), + ) + + # but not the other way around + with pytest.raises( + ValueError, + match=r"^You cannot specify a nested structure inside a RaisesGroup with", + ): + RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore[call-overload] + + # flatten_subgroups is not sufficient to catch fully unwrapped + with ( + fails_raises_group( + "`ValueError()` is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError, flatten_subgroups=True), + ): + raise ValueError + with ( + fails_raises_group( + "RaisesGroup(ValueError, flatten_subgroups=True): `ValueError()` is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True)), + ): + raise ExceptionGroup("", (ValueError(),)) + + # helpful suggestion if flatten_subgroups would make it pass + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " TypeError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Unexpected nested `ExceptionGroup()`, expected `TypeError`\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # but doesn't consider check (otherwise we'd break typing guarantees) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " TypeError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Unexpected nested `ExceptionGroup()`, expected `TypeError`\n" + "Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup( + ValueError, + TypeError, + check=lambda eg: len(eg.exceptions) == 1, + ), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + # correct number of exceptions, and flatten_subgroups would make it pass + # This now doesn't print a repr of the caught exception at all, but that can be found in the traceback + with ( + fails_raises_group( + "Raised exception group did not match: Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Did you mean to use `flatten_subgroups=True`?", + add_prefix=False, + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + # correct number of exceptions, but flatten_subgroups wouldn't help, so we don't suggest it + with ( + fails_raises_group( + "Unexpected nested `ExceptionGroup()`, expected `ValueError`" + ), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [TypeError()])]) + + # flatten_subgroups can be suggested if nested. This will implicitly ask the user to + # do `RaisesGroup(RaisesGroup(ValueError, flatten_subgroups=True))` which is unlikely + # to be what they actually want - but I don't think it's worth trying to special-case + with ( + fails_raises_group( + "RaisesGroup(ValueError): Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + + # Don't mention "unexpected nested" if expecting an ExceptionGroup. + # Although it should perhaps be an error to specify `RaisesGroup(ExceptionGroup)` in + # favor of doing `RaisesGroup(RaisesGroup(...))`. + with ( + fails_raises_group( + "`BaseExceptionGroup()` is not an instance of `ExceptionGroup`" + ), + RaisesGroup(ExceptionGroup), + ): + raise BaseExceptionGroup("", [BaseExceptionGroup("", [KeyboardInterrupt()])]) + + +def test_catch_unwrapped_exceptions() -> None: + # Catches lone exceptions with strict=False + # just as except* would + with RaisesGroup(ValueError, allow_unwrapped=True): + raise ValueError + + # expecting multiple unwrapped exceptions is not possible + with pytest.raises( + ValueError, + match=r"^You cannot specify multiple exceptions with", + ): + RaisesGroup(SyntaxError, ValueError, allow_unwrapped=True) # type: ignore[call-overload] + # if users want one of several exception types they need to use a RaisesExc + # (which the error message suggests) + with RaisesGroup( + RaisesExc(check=lambda e: isinstance(e, SyntaxError | ValueError)), + allow_unwrapped=True, + ): + raise ValueError + + # Unwrapped nested `RaisesGroup` is likely a user error, so we raise an error. + with pytest.raises(ValueError, match="has no effect when expecting"): + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore[call-overload] + + # But it *can* be used to check for nesting level +- 1 if they move it to + # the nested RaisesGroup. Users should probably use `RaisesExc`s instead though. + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + with RaisesGroup(RaisesGroup(ValueError, allow_unwrapped=True)): + raise ExceptionGroup("", [ValueError()]) + + # with allow_unwrapped=False (default) it will not be caught + with ( + fails_raises_group( + "`ValueError()` is not an exception group, but would match with `allow_unwrapped=True`" + ), + RaisesGroup(ValueError), + ): + raise ValueError("value error text") + + # allow_unwrapped on its own won't match against nested groups + with ( + fails_raises_group( + "Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Did you mean to use `flatten_subgroups=True`?", + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise ExceptionGroup("foo", [ExceptionGroup("bar", [ValueError()])]) + + # you need both allow_unwrapped and flatten_subgroups to fully emulate except* + with RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError()])]) + + # code coverage + with ( + fails_raises_group( + "Raised exception (group) did not match: `TypeError()` is not an instance of `ValueError`", + add_prefix=False, + ), + RaisesGroup(ValueError, allow_unwrapped=True), + ): + raise TypeError("this text doesn't show up in the error message") + with ( + fails_raises_group( + "Raised exception (group) did not match: RaisesExc(ValueError): `TypeError()` is not an instance of `ValueError`", + add_prefix=False, + ), + RaisesGroup(RaisesExc(ValueError), allow_unwrapped=True), + ): + raise TypeError + + # check we don't suggest unwrapping with nested RaisesGroup + with ( + fails_raises_group("`ValueError()` is not an exception group"), + RaisesGroup(RaisesGroup(ValueError)), + ): + raise ValueError + + +def test_match() -> None: + # supports match string + with RaisesGroup(ValueError, match="bar"): + raise ExceptionGroup("bar", (ValueError(),)) + + # now also works with ^$ + with RaisesGroup(ValueError, match="^bar$"): + raise ExceptionGroup("bar", (ValueError(),)) + + # it also includes notes + with RaisesGroup(ValueError, match="my note"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + # and technically you can match it all with ^$ + # but you're probably better off using a RaisesExc at that point + with RaisesGroup(ValueError, match="^bar\nmy note$"): + e = ExceptionGroup("bar", (ValueError(),)) + e.add_note("my note") + raise e + + with ( + fails_raises_group( + "Regex pattern did not match the `ExceptionGroup()`.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bar'" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", (ValueError(),)) + + # Suggest a fix for easy pitfall of adding match to the RaisesGroup instead of + # using a RaisesExc. + # This requires a single expected & raised exception, the expected is a type, + # and `isinstance(raised, expected_type)`. + with ( + fails_raises_group( + "Regex pattern did not match the `ExceptionGroup()`.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bar'\n" + " but matched the expected `ValueError`.\n" + " You might want `RaisesGroup(RaisesExc(ValueError, match='foo'))`" + ), + RaisesGroup(ValueError, match="foo"), + ): + raise ExceptionGroup("bar", [ValueError("foo")]) + + +def test_check() -> None: + exc = ExceptionGroup("", (ValueError(),)) + + def is_exc(e: ExceptionGroup[ValueError]) -> bool: + return e is exc + + is_exc_repr = repr_callable(is_exc) + with RaisesGroup(ValueError, check=is_exc): + raise exc + + with ( + fails_raises_group( + f"check {is_exc_repr} did not return True on the ExceptionGroup" + ), + RaisesGroup(ValueError, check=is_exc), + ): + raise ExceptionGroup("", (ValueError(),)) + + def is_value_error(e: BaseException) -> bool: + return isinstance(e, ValueError) + + # helpful suggestion if the user thinks the check is for the sub-exception + with ( + fails_raises_group( + f"check {is_value_error} did not return True on the ExceptionGroup, but did return True for the expected ValueError. You might want RaisesGroup(RaisesExc(ValueError, check=<...>))" + ), + RaisesGroup(ValueError, check=is_value_error), + ): + raise ExceptionGroup("", (ValueError(),)) + + +def test_unwrapped_match_check() -> None: + def my_check(e: object) -> bool: # pragma: no cover + return True + + msg = ( + "`allow_unwrapped=True` bypasses the `match` and `check` parameters" + " if the exception is unwrapped. If you intended to match/check the" + " exception you should use a `RaisesExc` object. If you want to match/check" + " the exceptiongroup when the exception *is* wrapped you need to" + " do e.g. `if isinstance(exc.value, ExceptionGroup):" + " assert RaisesGroup(...).matches(exc.value)` afterwards." + ) + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, match="foo") # type: ignore[call-overload] + with pytest.raises(ValueError, match=re.escape(msg)): + RaisesGroup(ValueError, allow_unwrapped=True, check=my_check) # type: ignore[call-overload] + + # Users should instead use a RaisesExc + rg = RaisesGroup(RaisesExc(ValueError, match="^foo$"), allow_unwrapped=True) + with rg: + raise ValueError("foo") + with rg: + raise ExceptionGroup("", [ValueError("foo")]) + + # or if they wanted to match/check the group, do a conditional `.matches()` + with RaisesGroup(ValueError, allow_unwrapped=True) as exc: + raise ExceptionGroup("bar", [ValueError("foo")]) + if isinstance(exc.value, ExceptionGroup): # pragma: no branch + assert RaisesGroup(ValueError, match="bar").matches(exc.value) + + +def test_matches() -> None: + rg = RaisesGroup(ValueError) + assert not rg.matches(None) + assert not rg.matches(ValueError()) + assert rg.matches(ExceptionGroup("", (ValueError(),))) + + re = RaisesExc(ValueError) + assert not re.matches(None) + assert re.matches(ValueError()) + + +def test_message() -> None: + def check_message( + message: str, + body: RaisesGroup[BaseException], + ) -> None: + with ( + pytest.raises( + Failed, + match=f"^DID NOT RAISE any exception, expected `{re.escape(message)}`$", + ), + body, + ): + ... + + # basic + check_message("ExceptionGroup(ValueError)", RaisesGroup(ValueError)) + # multiple exceptions + check_message( + "ExceptionGroup(ValueError, ValueError)", + RaisesGroup(ValueError, ValueError), + ) + # nested + check_message( + "ExceptionGroup(ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(ValueError)), + ) + + # RaisesExc + check_message( + "ExceptionGroup(RaisesExc(ValueError, match='my_str'))", + RaisesGroup(RaisesExc(ValueError, match="my_str")), + ) + check_message( + "ExceptionGroup(RaisesExc(match='my_str'))", + RaisesGroup(RaisesExc(match="my_str")), + ) + # one-size tuple is printed as not being a tuple + check_message( + "ExceptionGroup(RaisesExc(ValueError))", + RaisesGroup(RaisesExc((ValueError,))), + ) + check_message( + "ExceptionGroup(RaisesExc((ValueError, IndexError)))", + RaisesGroup(RaisesExc((ValueError, IndexError))), + ) + + # BaseExceptionGroup + check_message( + "BaseExceptionGroup(KeyboardInterrupt)", + RaisesGroup(KeyboardInterrupt), + ) + # BaseExceptionGroup with type inside RaisesExc + check_message( + "BaseExceptionGroup(RaisesExc(KeyboardInterrupt))", + RaisesGroup(RaisesExc(KeyboardInterrupt)), + ) + check_message( + "BaseExceptionGroup(RaisesExc((ValueError, KeyboardInterrupt)))", + RaisesGroup(RaisesExc((ValueError, KeyboardInterrupt))), + ) + # Base-ness transfers to parent containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt))", + RaisesGroup(RaisesGroup(KeyboardInterrupt)), + ) + # but not to child containers + check_message( + "BaseExceptionGroup(BaseExceptionGroup(KeyboardInterrupt), ExceptionGroup(ValueError))", + RaisesGroup(RaisesGroup(KeyboardInterrupt), RaisesGroup(ValueError)), + ) + + +def test_assert_message() -> None: + # the message does not need to list all parameters to RaisesGroup, nor all exceptions + # in the exception group, as those are both visible in the traceback. + # first fails to match + with ( + fails_raises_group("`TypeError()` is not an instance of `ValueError`"), + RaisesGroup(ValueError), + ): + raise ExceptionGroup("a", [TypeError()]) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(ValueError, match='a')\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [RuntimeError()]):\n" + " RaisesGroup(ValueError): `RuntimeError()` is not an instance of `ValueError`\n" + " RaisesGroup(ValueError, match='a'): Regex pattern did not match the `ExceptionGroup()`.\n" + " Expected regex: 'a'\n" + " Actual message: ''\n" + " RuntimeError():\n" + " RaisesGroup(ValueError): `RuntimeError()` is not an exception group\n" + " RaisesGroup(ValueError, match='a'): `RuntimeError()` is not an exception group", + add_prefix=False, # to see the full structure + ), + RaisesGroup(RaisesGroup(ValueError), RaisesGroup(ValueError, match="a")), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [RuntimeError()]), RuntimeError()], + ) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "2 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(RuntimeError)\n" + " RaisesGroup(ValueError)\n" + "The following raised exceptions did not find a match\n" + " RuntimeError():\n" + " RaisesGroup(RuntimeError): `RuntimeError()` is not an exception group, but would match with `allow_unwrapped=True`\n" + " RaisesGroup(ValueError): `RuntimeError()` is not an exception group\n" + " ValueError('bar'):\n" + " It matches `ValueError` which was paired with `ValueError('foo')`\n" + " RaisesGroup(RuntimeError): `ValueError()` is not an exception group\n" + " RaisesGroup(ValueError): `ValueError()` is not an exception group, but would match with `allow_unwrapped=True`", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + ValueError, + RaisesExc(TypeError), + RaisesGroup(RuntimeError), + RaisesGroup(ValueError), + ), + ): + raise ExceptionGroup( + "a", + [RuntimeError(), TypeError(), ValueError("foo"), ValueError("bar")], + ) + + with ( + fails_raises_group( + "1 matched exception. `AssertionError()` is not an instance of `TypeError`" + ), + RaisesGroup(ValueError, TypeError), + ): + raise ExceptionGroup("a", [ValueError(), AssertionError()]) + + with ( + fails_raises_group( + "RaisesExc(ValueError): `TypeError()` is not an instance of `ValueError`" + ), + RaisesGroup(RaisesExc(ValueError)), + ): + raise ExceptionGroup("a", [TypeError()]) + + # suggest escaping + with ( + fails_raises_group( + # TODO: did not match Exceptiongroup('h(ell)o', ...) ? + "Raised exception group did not match: Regex pattern did not match the `ExceptionGroup()`.\n" + " Expected regex: 'h(ell)o'\n" + " Actual message: 'h(ell)o'\n" + " Did you mean to `re.escape()` the regex?", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, match="h(ell)o"), + ): + raise ExceptionGroup("h(ell)o", [ValueError()]) + with ( + fails_raises_group( + "RaisesExc(match='h(ell)o'): Regex pattern did not match.\n" + " Expected regex: 'h(ell)o'\n" + " Actual message: 'h(ell)o'\n" + " Did you mean to `re.escape()` the regex?", + ), + RaisesGroup(RaisesExc(match="h(ell)o")), + ): + raise ExceptionGroup("", [ValueError("h(ell)o")]) + + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " ValueError\n" + " ValueError\n" + " ValueError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), TypeError()]):\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`", + add_prefix=False, # to see the full structure + ), + RaisesGroup(ValueError, ValueError, ValueError, ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("", [ValueError(), TypeError()])]) + + +def test_message_indent() -> None: + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " ValueError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [TypeError(), RuntimeError()]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " ValueError\n" + " ValueError\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " `TypeError()` is not an instance of `ValueError`\n" + " `TypeError()` is not an instance of `ValueError`\n" + " RuntimeError():\n" + " `RuntimeError()` is not an instance of `ValueError`\n" + " `RuntimeError()` is not an instance of `ValueError`\n" + # TODO: this line is not great, should maybe follow the same format as the other and say + # ValueError: Unexpected nested `ExceptionGroup()` (?) + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " TypeError():\n" + " RaisesGroup(ValueError, ValueError): `TypeError()` is not an exception group\n" + " `TypeError()` is not an instance of `ValueError`", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ValueError, + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + TypeError(), + ], + ) + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "RaisesGroup(ValueError, ValueError): \n" + " The following expected exceptions did not find a match:\n" + " ValueError\n" + " ValueError\n" + " The following raised exceptions did not find a match\n" + " TypeError():\n" + " `TypeError()` is not an instance of `ValueError`\n" + " `TypeError()` is not an instance of `ValueError`\n" + " RuntimeError():\n" + " `RuntimeError()` is not an instance of `ValueError`\n" + " `RuntimeError()` is not an instance of `ValueError`", + add_prefix=False, + ), + RaisesGroup( + RaisesGroup(ValueError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + ExceptionGroup("", [TypeError(), RuntimeError()]), + ], + ) + + +def test_suggestion_on_nested_and_brief_error() -> None: + # Make sure "Did you mean" suggestion gets indented iff it follows a single-line error + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " ValueError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError): Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`", + ), + RaisesGroup(RaisesGroup(ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ExceptionGroup("", [ValueError()])])], + ) + # if indented here it would look like another raised exception + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError, ValueError)\n" + " ValueError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError(), ExceptionGroup('', [ValueError()])]):\n" + " RaisesGroup(ValueError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " ValueError\n" + " It matches `ValueError()` which was paired with `ValueError`\n" + " The following raised exceptions did not find a match\n" + " ExceptionGroup('', [ValueError()]):\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`\n" + " Did you mean to use `flatten_subgroups=True`?\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`" + ), + RaisesGroup(RaisesGroup(ValueError, ValueError), ValueError), + ): + raise ExceptionGroup( + "", + [ExceptionGroup("", [ValueError(), ExceptionGroup("", [ValueError()])])], + ) + + # re.escape always comes after single-line errors + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(Exception, match='^hello')\n" + " ValueError\n" + "The following raised exceptions did not find a match\n" + " ExceptionGroup('^hello', [Exception()]):\n" + " RaisesGroup(Exception, match='^hello'): Regex pattern did not match the `ExceptionGroup()`.\n" + " Expected regex: '^hello'\n" + " Actual message: '^hello'\n" + " Did you mean to `re.escape()` the regex?\n" + " Unexpected nested `ExceptionGroup()`, expected `ValueError`" + ), + RaisesGroup(RaisesGroup(Exception, match="^hello"), ValueError), + ): + raise ExceptionGroup("", [ExceptionGroup("^hello", [Exception()])]) + + +def test_assert_message_nested() -> None: + # we only get one instance of aaaaaaaaaa... and bbbbbb..., but we do get multiple instances of ccccc... and dddddd.. + # but I think this now only prints the full repr when that is necessary to disambiguate exceptions + with ( + fails_raises_group( + "Raised exception group did not match: \n" + "The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " RaisesGroup(RaisesGroup(ValueError))\n" + " RaisesGroup(RaisesExc(TypeError, match='foo'))\n" + " RaisesGroup(TypeError, ValueError)\n" + "The following raised exceptions did not find a match\n" + " TypeError('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'):\n" + " RaisesGroup(ValueError): `TypeError()` is not an exception group\n" + " RaisesGroup(RaisesGroup(ValueError)): `TypeError()` is not an exception group\n" + " RaisesGroup(RaisesExc(TypeError, match='foo')): `TypeError()` is not an exception group\n" + " RaisesGroup(TypeError, ValueError): `TypeError()` is not an exception group\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb')]):\n" + " RaisesGroup(ValueError): `TypeError()` is not an instance of `ValueError`\n" + " RaisesGroup(RaisesGroup(ValueError)): RaisesGroup(ValueError): `TypeError()` is not an exception group\n" + " RaisesGroup(RaisesExc(TypeError, match='foo')): RaisesExc(TypeError, match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'\n" + " RaisesGroup(TypeError, ValueError): 1 matched exception. Too few exceptions raised, found no match for: [ValueError]\n" + " ExceptionGroup('Exceptions from Trio nursery', [TypeError('cccccccccccccccccccccccccccccc'), TypeError('dddddddddddddddddddddddddddddd')]):\n" + " RaisesGroup(ValueError): \n" + " The following expected exceptions did not find a match:\n" + " ValueError\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " `TypeError()` is not an instance of `ValueError`\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " `TypeError()` is not an instance of `ValueError`\n" + " RaisesGroup(RaisesGroup(ValueError)): \n" + " The following expected exceptions did not find a match:\n" + " RaisesGroup(ValueError)\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " RaisesGroup(ValueError): `TypeError()` is not an exception group\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " RaisesGroup(ValueError): `TypeError()` is not an exception group\n" + " RaisesGroup(RaisesExc(TypeError, match='foo')): \n" + " The following expected exceptions did not find a match:\n" + " RaisesExc(TypeError, match='foo')\n" + " The following raised exceptions did not find a match\n" + " TypeError('cccccccccccccccccccccccccccccc'):\n" + " RaisesExc(TypeError, match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'cccccccccccccccccccccccccccccc'\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " RaisesExc(TypeError, match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'dddddddddddddddddddddddddddddd'\n" + " RaisesGroup(TypeError, ValueError): \n" + " 1 matched exception. \n" + " The following expected exceptions did not find a match:\n" + " ValueError\n" + " The following raised exceptions did not find a match\n" + " TypeError('dddddddddddddddddddddddddddddd'):\n" + " It matches `TypeError` which was paired with `TypeError('cccccccccccccccccccccccccccccc')`\n" + " `TypeError()` is not an instance of `ValueError`", + add_prefix=False, # to see the full structure + ), + RaisesGroup( + RaisesGroup(ValueError), + RaisesGroup(RaisesGroup(ValueError)), + RaisesGroup(RaisesExc(TypeError, match="foo")), + RaisesGroup(TypeError, ValueError), + ), + ): + raise ExceptionGroup( + "", + [ + TypeError("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + ExceptionGroup( + "Exceptions from Trio nursery", + [TypeError("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")], + ), + ExceptionGroup( + "Exceptions from Trio nursery", + [ + TypeError("cccccccccccccccccccccccccccccc"), + TypeError("dddddddddddddddddddddddddddddd"), + ], + ), + ], + ) + + +# CI always runs with hypothesis, but this is not a critical test - it overlaps +# with several others +@pytest.mark.skipif( + "hypothesis" in sys.modules, + reason="hypothesis may have monkeypatched _check_repr", +) +def test_check_no_patched_repr() -> None: # pragma: no cover + # We make `_check_repr` monkeypatchable to avoid this very ugly and verbose + # repr. The other tests that use `check` make use of `_check_repr` so they'll + # continue passing in case it is patched - but we have this one test that + # demonstrates just how nasty it gets otherwise. + match_str = ( + r"^Raised exception group did not match: \n" + r"The following expected exceptions did not find a match:\n" + r" RaisesExc\(check=. at .*>\)\n" + r" TypeError\n" + r"The following raised exceptions did not find a match\n" + r" ValueError\('foo'\):\n" + r" RaisesExc\(check=. at .*>\): check did not return True\n" + r" `ValueError\(\)` is not an instance of `TypeError`\n" + r" ValueError\('bar'\):\n" + r" RaisesExc\(check=. at .*>\): check did not return True\n" + r" `ValueError\(\)` is not an instance of `TypeError`$" + ) + with ( + pytest.raises(Failed, match=match_str), + RaisesGroup(RaisesExc(check=lambda x: False), TypeError), + ): + raise ExceptionGroup("", [ValueError("foo"), ValueError("bar")]) + + +def test_misordering_example() -> None: + with ( + fails_raises_group( + "\n" + "3 matched exceptions. \n" + "The following expected exceptions did not find a match:\n" + " RaisesExc(ValueError, match='foo')\n" + " It matches `ValueError('foo')` which was paired with `ValueError`\n" + " It matches `ValueError('foo')` which was paired with `ValueError`\n" + " It matches `ValueError('foo')` which was paired with `ValueError`\n" + "The following raised exceptions did not find a match\n" + " ValueError('bar'):\n" + " It matches `ValueError` which was paired with `ValueError('foo')`\n" + " It matches `ValueError` which was paired with `ValueError('foo')`\n" + " It matches `ValueError` which was paired with `ValueError('foo')`\n" + " RaisesExc(ValueError, match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bar'\n" + "There exist a possible match when attempting an exhaustive check, but RaisesGroup uses a greedy algorithm. Please make your expected exceptions more stringent with `RaisesExc` etc so the greedy algorithm can function." + ), + RaisesGroup( + ValueError, ValueError, ValueError, RaisesExc(ValueError, match="foo") + ), + ): + raise ExceptionGroup( + "", + [ + ValueError("foo"), + ValueError("foo"), + ValueError("foo"), + ValueError("bar"), + ], + ) + + +def test_brief_error_on_one_fail() -> None: + """If only one raised and one expected fail to match up, we print a full table iff + the raised exception would match one of the expected that previously got matched""" + # no also-matched + with ( + fails_raises_group( + "1 matched exception. `TypeError()` is not an instance of `RuntimeError`" + ), + RaisesGroup(ValueError, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # raised would match an expected + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " RuntimeError\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " It matches `Exception` which was paired with `ValueError()`\n" + " `TypeError()` is not an instance of `RuntimeError`" + ), + RaisesGroup(Exception, RuntimeError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + # expected would match a raised + with ( + fails_raises_group( + "\n" + "1 matched exception. \n" + "The following expected exceptions did not find a match:\n" + " ValueError\n" + " It matches `ValueError()` which was paired with `ValueError`\n" + "The following raised exceptions did not find a match\n" + " TypeError():\n" + " `TypeError()` is not an instance of `ValueError`" + ), + RaisesGroup(ValueError, ValueError), + ): + raise ExceptionGroup("", [ValueError(), TypeError()]) + + +def test_identity_oopsies() -> None: + # it's both possible to have several instances of the same exception in the same group + # and to expect multiple of the same type + # this previously messed up the logic + + with ( + fails_raises_group( + "3 matched exceptions. `RuntimeError()` is not an instance of `TypeError`" + ), + RaisesGroup(ValueError, ValueError, ValueError, TypeError), + ): + raise ExceptionGroup( + "", [ValueError(), ValueError(), ValueError(), RuntimeError()] + ) + + e = ValueError("foo") + m = RaisesExc(match="bar") + with ( + fails_raises_group( + "\n" + "The following expected exceptions did not find a match:\n" + " RaisesExc(match='bar')\n" + " RaisesExc(match='bar')\n" + " RaisesExc(match='bar')\n" + "The following raised exceptions did not find a match\n" + " ValueError('foo'):\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " ValueError('foo'):\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " ValueError('foo'):\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'\n" + " RaisesExc(match='bar'): Regex pattern did not match.\n" + " Expected regex: 'bar'\n" + " Actual message: 'foo'" + ), + RaisesGroup(m, m, m), + ): + raise ExceptionGroup("", [e, e, e]) + + +def test_raisesexc() -> None: + with pytest.raises( + ValueError, + match=r"^You must specify at least one parameter to match on.$", + ): + RaisesExc() # type: ignore[call-overload] + with pytest.raises( + ValueError, + match=wrap_escape("Expected a BaseException type, but got 'object'"), + ): + RaisesExc(object) # type: ignore[type-var] + + with RaisesGroup(RaisesExc(ValueError)): + raise ExceptionGroup("", (ValueError(),)) + with ( + fails_raises_group( + "RaisesExc(TypeError): `ValueError()` is not an instance of `TypeError`" + ), + RaisesGroup(RaisesExc(TypeError)), + ): + raise ExceptionGroup("", (ValueError(),)) + + with RaisesExc(ValueError): + raise ValueError + + # FIXME: leaving this one formatted differently for now to not change + # tests in python/raises.py + with pytest.raises(Failed, match=wrap_escape("DID NOT RAISE ")): + with RaisesExc(ValueError): + ... + + with pytest.raises(Failed, match=wrap_escape("DID NOT RAISE any exception")): + with RaisesExc(match="foo"): + ... + + with pytest.raises( + # FIXME: do we want repr(type) or type.__name__ ? + Failed, + match=wrap_escape( + "DID NOT RAISE any of (, )" + ), + ): + with RaisesExc((ValueError, TypeError)): + ... + + # currently RaisesGroup says "Raised exception did not match" but RaisesExc doesn't... + with pytest.raises( + AssertionError, + match=wrap_escape( + "Regex pattern did not match.\n Expected regex: 'foo'\n Actual message: 'bar'" + ), + ): + with RaisesExc(TypeError, match="foo"): + raise TypeError("bar") + + +def test_raisesexc_match() -> None: + with RaisesGroup(RaisesExc(ValueError, match="foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "RaisesExc(ValueError, match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bar'" + ), + RaisesGroup(RaisesExc(ValueError, match="foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # Can be used without specifying the type + with RaisesGroup(RaisesExc(match="foo")): + raise ExceptionGroup("", (ValueError("foo"),)) + with ( + fails_raises_group( + "RaisesExc(match='foo'): Regex pattern did not match.\n" + " Expected regex: 'foo'\n" + " Actual message: 'bar'" + ), + RaisesGroup(RaisesExc(match="foo")), + ): + raise ExceptionGroup("", (ValueError("bar"),)) + + # check ^$ + with RaisesGroup(RaisesExc(ValueError, match="^bar$")): + raise ExceptionGroup("", [ValueError("bar")]) + with ( + fails_raises_group( + "\nRaisesExc(ValueError, match='^bar$'): \n - barr\n ? -\n + bar" + ), + RaisesGroup(RaisesExc(ValueError, match="^bar$")), + ): + raise ExceptionGroup("", [ValueError("barr")]) + + +def test_RaisesExc_check() -> None: + def check_oserror_and_errno_is_5(e: BaseException) -> bool: + return isinstance(e, OSError) and e.errno == 5 + + with RaisesGroup(RaisesExc(check=check_oserror_and_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # specifying exception_type narrows the parameter type to the callable + def check_errno_is_5(e: OSError) -> bool: + return e.errno == 5 + + with RaisesGroup(RaisesExc(OSError, check=check_errno_is_5)): + raise ExceptionGroup("", (OSError(5, ""),)) + + # avoid printing overly verbose repr multiple times + with ( + fails_raises_group( + f"RaisesExc(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(RaisesExc(OSError, check=check_errno_is_5)), + ): + raise ExceptionGroup("", (OSError(6, ""),)) + + # in nested cases you still get it multiple times though + # to address this you'd need logic in RaisesExc.__repr__ and RaisesGroup.__repr__ + with ( + fails_raises_group( + f"RaisesGroup(RaisesExc(OSError, check={check_errno_is_5!r})): RaisesExc(OSError, check={check_errno_is_5!r}): check did not return True" + ), + RaisesGroup(RaisesGroup(RaisesExc(OSError, check=check_errno_is_5))), + ): + raise ExceptionGroup("", [ExceptionGroup("", [OSError(6, "")])]) + + +def test_raisesexc_tostring() -> None: + assert str(RaisesExc(ValueError)) == "RaisesExc(ValueError)" + assert str(RaisesExc(match="[a-z]")) == "RaisesExc(match='[a-z]')" + pattern_no_flags = re.compile(r"noflag", 0) + assert str(RaisesExc(match=pattern_no_flags)) == "RaisesExc(match='noflag')" + pattern_flags = re.compile(r"noflag", re.IGNORECASE) + assert str(RaisesExc(match=pattern_flags)) == f"RaisesExc(match={pattern_flags!r})" + assert ( + str(RaisesExc(ValueError, match="re", check=bool)) + == f"RaisesExc(ValueError, match='re', check={bool!r})" + ) + + +def test_raisesgroup_tostring() -> None: + def check_str_and_repr(s: str) -> None: + evaled = eval(s) + assert s == str(evaled) == repr(evaled) + + check_str_and_repr("RaisesGroup(ValueError)") + check_str_and_repr("RaisesGroup(RaisesGroup(ValueError))") + check_str_and_repr("RaisesGroup(RaisesExc(ValueError))") + check_str_and_repr("RaisesGroup(ValueError, allow_unwrapped=True)") + check_str_and_repr("RaisesGroup(ValueError, match='aoeu')") + + assert ( + str(RaisesGroup(ValueError, match="[a-z]", check=bool)) + == f"RaisesGroup(ValueError, match='[a-z]', check={bool!r})" + ) + + +def test_assert_matches() -> None: + e = ValueError() + + # it's easy to do this + assert RaisesExc(ValueError).matches(e) + + # but you don't get a helpful error + with pytest.raises(AssertionError, match=r"assert False\n \+ where False = .*"): + assert RaisesExc(TypeError).matches(e) + + with pytest.raises( + AssertionError, + match=wrap_escape( + "`ValueError()` is not an instance of `TypeError`\n" + "assert False\n" + " + where False = matches(ValueError())\n" + " + where matches = RaisesExc(TypeError).matches" + ), + ): + # you'd need to do this arcane incantation + assert (m := RaisesExc(TypeError)).matches(e), m.fail_reason + + # but even if we add assert_matches, will people remember to use it? + # other than writing a linter rule, I don't think we can catch `assert RaisesExc(...).matches` + # ... no wait pytest catches other asserts ... so we probably can?? + + +# https://github.com/pytest-dev/pytest/issues/12504 +def test_xfail_raisesgroup(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import sys + import pytest + if sys.version_info < (3, 11): + from exceptiongroup import ExceptionGroup + @pytest.mark.xfail(raises=pytest.RaisesGroup(ValueError)) + def test_foo() -> None: + raise ExceptionGroup("foo", [ValueError()]) + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xfailed=1) + + +def test_xfail_RaisesExc(pytester: Pytester) -> None: + pytester.makepyfile( + """ + import pytest + @pytest.mark.xfail(raises=pytest.RaisesExc(ValueError)) + def test_foo() -> None: + raise ValueError + """ + ) + result = pytester.runpytest() + result.assert_outcomes(xfailed=1) + + +@pytest.mark.parametrize( + "wrap_in_group,handler", + [ + (False, pytest.raises(ValueError)), + (True, RaisesGroup(ValueError)), + ], +) +def test_parametrizing_conditional_raisesgroup( + wrap_in_group: bool, handler: AbstractContextManager[ExceptionInfo[BaseException]] +) -> None: + with handler: + if wrap_in_group: + raise ExceptionGroup("", [ValueError()]) + raise ValueError() + + +def test_annotated_group() -> None: + # repr depends on if exceptiongroup backport is being used or not + t = repr(ExceptionGroup[ValueError]) + msg = "Only `ExceptionGroup[Exception]` or `BaseExceptionGroup[BaseException]` are accepted as generic types but got `{}`. As `raises` will catch all instances of the specified group regardless of the generic argument specific nested exceptions has to be checked with `RaisesGroup`." + + fail_msg = wrap_escape(msg.format(t)) + with pytest.raises(ValueError, match=fail_msg): + RaisesGroup(ExceptionGroup[ValueError]) + with pytest.raises(ValueError, match=fail_msg): + RaisesExc(ExceptionGroup[ValueError]) + with pytest.raises( + ValueError, + match=wrap_escape(msg.format(repr(BaseExceptionGroup[KeyboardInterrupt]))), + ): + with RaisesExc(BaseExceptionGroup[KeyboardInterrupt]): + raise BaseExceptionGroup("", [KeyboardInterrupt()]) + + with RaisesGroup(ExceptionGroup[Exception]): + raise ExceptionGroup( + "", [ExceptionGroup("", [ValueError(), ValueError(), ValueError()])] + ) + with RaisesExc(BaseExceptionGroup[BaseException]): + raise BaseExceptionGroup("", [KeyboardInterrupt()]) + + # assure AbstractRaises.is_baseexception is set properly + assert ( + RaisesGroup(ExceptionGroup[Exception]).expected_type() + == "ExceptionGroup(ExceptionGroup)" + ) + assert ( + RaisesGroup(BaseExceptionGroup[BaseException]).expected_type() + == "BaseExceptionGroup(BaseExceptionGroup)" + ) + + +def test_tuples() -> None: + # raises has historically supported one of several exceptions being raised + with pytest.raises((ValueError, IndexError)): + raise ValueError + # so now RaisesExc also does + with RaisesExc((ValueError, IndexError)): + raise IndexError + # but RaisesGroup currently doesn't. There's an argument it shouldn't because + # it can be confusing - RaisesGroup((ValueError, TypeError)) looks a lot like + # RaisesGroup(ValueError, TypeError), and the former might be interpreted as the latter. + with pytest.raises( + TypeError, + match=wrap_escape( + "Expected a BaseException type, RaisesExc, or RaisesGroup, but got 'tuple'.\n" + "RaisesGroup does not support tuples of exception types when expecting one of " + "several possible exception types like RaisesExc.\n" + "If you meant to expect a group with multiple exceptions, list them as separate arguments." + ), + ): + RaisesGroup((ValueError, IndexError)) # type: ignore[call-overload] diff --git a/testing/test_assertion.py b/testing/test_assertion.py index 69ca0f73ff2..5179b13b0e9 100644 --- a/testing/test_assertion.py +++ b/testing/test_assertion.py @@ -1,10 +1,10 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import MutableSequence import sys import textwrap from typing import Any -from typing import MutableSequence from typing import NamedTuple import attr @@ -218,10 +218,36 @@ def test_foo(pytestconfig): assert result.ret == 0 @pytest.mark.parametrize("mode", ["plain", "rewrite"]) + @pytest.mark.parametrize("disable_plugin_autoload", ["env_var", "cli", ""]) + @pytest.mark.parametrize("explicit_specify", ["env_var", "cli", ""]) def test_installed_plugin_rewrite( - self, pytester: Pytester, mode, monkeypatch + self, + pytester: Pytester, + mode: str, + monkeypatch: pytest.MonkeyPatch, + disable_plugin_autoload: str, + explicit_specify: str, ) -> None: - monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + args = ["mainwrapper.py", "-s", f"--assert={mode}"] + if disable_plugin_autoload == "env_var": + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + elif disable_plugin_autoload == "cli": + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + args.append("--disable-plugin-autoload") + else: + assert disable_plugin_autoload == "" + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + + name = "spamplugin" + + if explicit_specify == "env_var": + monkeypatch.setenv("PYTEST_PLUGINS", name) + elif explicit_specify == "cli": + args.append("-p") + args.append(name) + else: + assert explicit_specify == "" + # Make sure the hook is installed early enough so that plugins # installed via distribution package are rewritten. pytester.mkdir("hampkg") @@ -250,7 +276,7 @@ def check(values, value): import pytest class DummyEntryPoint(object): - name = 'spam' + name = 'spamplugin' module_name = 'spam.py' group = 'pytest11' @@ -275,20 +301,29 @@ def test(check_first): check_first([10, 30], 30) def test2(check_first2): - check_first([10, 30], 30) + check_first2([10, 30], 30) """, } pytester.makepyfile(**contents) - result = pytester.run( - sys.executable, "mainwrapper.py", "-s", f"--assert={mode}" - ) + result = pytester.run(sys.executable, *args) if mode == "plain": expected = "E AssertionError" elif mode == "rewrite": expected = "*assert 10 == 30*" else: assert 0 - result.stdout.fnmatch_lines([expected]) + + if not disable_plugin_autoload or explicit_specify: + result.assert_outcomes(failed=2) + result.stdout.fnmatch_lines([expected, expected]) + else: + result.assert_outcomes(errors=2) + result.stdout.fnmatch_lines( + [ + "E fixture 'check_first' not found", + "E fixture 'check_first2' not found", + ] + ) def test_rewrite_ast(self, pytester: Pytester) -> None: pytester.mkdir("pkg") @@ -532,6 +567,11 @@ def test_full_diff(): result = pytester.runpytest() result.stdout.fnmatch_lines(["E Full diff:"]) + # Setting CI to empty string is same as having it undefined + monkeypatch.setenv("CI", "") + result = pytester.runpytest() + result.stdout.fnmatch_lines(["E Use -v to get more diff"]) + monkeypatch.delenv("CI", raising=False) result = pytester.runpytest() result.stdout.fnmatch_lines(["E Use -v to get more diff"]) @@ -823,7 +863,7 @@ def __setitem__(self, item, value): def __delitem__(self, item): pass - def insert(self, item, index): + def insert(self, index, value): pass expl = callequal(TestSequence([0, 1]), list([0, 2])) @@ -1406,15 +1446,14 @@ def test_full_output_truncated(self, monkeypatch, pytester: Pytester) -> None: line_len = 100 expected_truncated_lines = 2 pytester.makepyfile( - r""" + rf""" def test_many_lines(): - a = list([str(i)[0] * %d for i in range(%d)]) + a = list([str(i)[0] * {line_len} for i in range({line_count})]) b = a[::2] a = '\n'.join(map(str, a)) b = '\n'.join(map(str, b)) assert a == b """ - % (line_len, line_count) ) monkeypatch.delenv("CI", raising=False) @@ -1424,17 +1463,88 @@ def test_many_lines(): [ "*+ 1*", "*+ 3*", - "*truncated (%d lines hidden)*use*-vv*" % expected_truncated_lines, + f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", ] ) result = pytester.runpytest("-vv") result.stdout.fnmatch_lines(["* 6*"]) + # Setting CI to empty string is same as having it undefined + monkeypatch.setenv("CI", "") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*+ 1*", + "*+ 3*", + f"*truncated ({expected_truncated_lines} lines hidden)*use*-vv*", + ] + ) + monkeypatch.setenv("CI", "1") result = pytester.runpytest() result.stdout.fnmatch_lines(["* 6*"]) + @pytest.mark.parametrize( + ["truncation_lines", "truncation_chars", "expected_lines_hidden"], + ( + (3, None, 3), + (4, None, 0), + (0, None, 0), + (None, 8, 6), + (None, 9, 0), + (None, 0, 0), + (0, 0, 0), + (0, 1000, 0), + (1000, 0, 0), + ), + ) + def test_truncation_with_ini( + self, + monkeypatch, + pytester: Pytester, + truncation_lines: int | None, + truncation_chars: int | None, + expected_lines_hidden: int, + ) -> None: + pytester.makepyfile( + """\ + string_a = "123456789\\n23456789\\n3" + string_b = "123456789\\n23456789\\n4" + + def test(): + assert string_a == string_b + """ + ) + + # This test produces 6 lines of diff output or 79 characters + # So the effect should be when threshold is < 4 lines (considering 2 additional lines for explanation) + # Or < 9 characters (considering 70 additional characters for explanation) + + monkeypatch.delenv("CI", raising=False) + + ini = "[pytest]\n" + if truncation_lines is not None: + ini += f"truncation_limit_lines = {truncation_lines}\n" + if truncation_chars is not None: + ini += f"truncation_limit_chars = {truncation_chars}\n" + pytester.makeini(ini) + + result = pytester.runpytest() + + if expected_lines_hidden != 0: + result.stdout.fnmatch_lines( + [f"*truncated ({expected_lines_hidden} lines hidden)*"] + ) + else: + result.stdout.no_fnmatch_line("*truncated*") + result.stdout.fnmatch_lines( + [ + "*- 4*", + "*+ 3*", + ] + ) + def test_python25_compile_issue257(pytester: Pytester) -> None: pytester.makepyfile( @@ -1960,6 +2070,16 @@ def test(): "{bold}{red}E {light-green}+ 'number-is-5': 5,{hl-reset}{endline}{reset}", ], ), + ( + """ + def test(): + assert "abcd" == "abce" + """, + [ + "{bold}{red}E {reset}{light-red}- abce{hl-reset}{endline}{reset}", + "{bold}{red}E {light-green}+ abcd{hl-reset}{endline}{reset}", + ], + ), ), ) def test_comparisons_handle_colors( diff --git a/testing/test_assertrewrite.py b/testing/test_assertrewrite.py index 5ee40ee6568..92664354470 100644 --- a/testing/test_assertrewrite.py +++ b/testing/test_assertrewrite.py @@ -2,10 +2,14 @@ from __future__ import annotations import ast +from collections.abc import Generator +from collections.abc import Mapping +import dis import errno from functools import partial import glob import importlib +import inspect import marshal import os from pathlib import Path @@ -15,8 +19,6 @@ import sys import textwrap from typing import cast -from typing import Generator -from typing import Mapping from unittest import mock import zipfile @@ -130,11 +132,212 @@ def test_location_is_set(self) -> None: if isinstance(node, ast.Import): continue for n in [node, *ast.iter_child_nodes(node)]: - assert isinstance(n, (ast.stmt, ast.expr)) - assert n.lineno == 3 - assert n.col_offset == 0 - assert n.end_lineno == 6 - assert n.end_col_offset == 3 + assert isinstance(n, ast.stmt | ast.expr) + for location in [ + (n.lineno, n.col_offset), + (n.end_lineno, n.end_col_offset), + ]: + assert (3, 0) <= location <= (6, 3) + + def test_positions_are_preserved(self) -> None: + """Ensure AST positions are preserved during rewriting (#12818).""" + + def preserved(code: str) -> None: + s = textwrap.dedent(code) + locations = [] + + def loc(msg: str | None = None) -> None: + frame = inspect.currentframe() + assert frame + frame = frame.f_back + assert frame + frame = frame.f_back + assert frame + + offset = frame.f_lasti + + instructions = {i.offset: i for i in dis.get_instructions(frame.f_code)} + + # skip CACHE instructions + while offset not in instructions and offset >= 0: + offset -= 1 + + instruction = instructions[offset] + if sys.version_info >= (3, 11): + position = instruction.positions + else: + position = instruction.starts_line + + locations.append((msg, instruction.opname, position)) + + globals = {"loc": loc} + + m = rewrite(s) + mod = compile(m, "", "exec") + exec(mod, globals, globals) + transformed_locations = locations + locations = [] + + mod = compile(s, "", "exec") + exec(mod, globals, globals) + original_locations = locations + + assert len(original_locations) > 0 + assert original_locations == transformed_locations + + preserved(""" + def f(): + loc() + return 8 + + assert f() in [8] + assert (f() + in + [8]) + """) + + preserved(""" + class T: + def __init__(self): + loc("init") + def __getitem__(self,index): + loc("getitem") + return index + + assert T()[5] == 5 + assert (T + () + [5] + == + 5) + """) + + for name, op in [ + ("pos", "+"), + ("neg", "-"), + ("invert", "~"), + ]: + preserved(f""" + class T: + def __{name}__(self): + loc("{name}") + return "{name}" + + assert {op}T() == "{name}" + assert ({op} + T + () + == + "{name}") + """) + + for name, op in [ + ("add", "+"), + ("sub", "-"), + ("mul", "*"), + ("truediv", "/"), + ("floordiv", "//"), + ("mod", "%"), + ("pow", "**"), + ("lshift", "<<"), + ("rshift", ">>"), + ("or", "|"), + ("xor", "^"), + ("and", "&"), + ("matmul", "@"), + ]: + preserved(f""" + class T: + def __{name}__(self,other): + loc("{name}") + return other + + def __r{name}__(self,other): + loc("r{name}") + return other + + assert T() {op} 2 == 2 + assert 2 {op} T() == 2 + + assert (T + () + {op} + 2 + == + 2) + + assert (2 + {op} + T + () + == + 2) + """) + + for name, op in [ + ("eq", "=="), + ("ne", "!="), + ("lt", "<"), + ("le", "<="), + ("gt", ">"), + ("ge", ">="), + ]: + preserved(f""" + class T: + def __{name}__(self,other): + loc() + return True + + assert T() {op} 5 + assert (T + () + {op} + 5) + """) + + for name, op in [ + ("eq", "=="), + ("ne", "!="), + ("lt", ">"), + ("le", ">="), + ("gt", "<"), + ("ge", "<="), + ("contains", "in"), + ]: + preserved(f""" + class T: + def __{name}__(self,other): + loc() + return True + + assert 5 {op} T() + assert (5 + {op} + T + ()) + """) + + preserved(""" + def func(value): + loc("func") + return value + + class T: + def __iter__(self): + loc("iter") + return iter([5]) + + assert func(*T()) == 5 + """) + + preserved(""" + class T: + def __getattr__(self,name): + loc() + return name + + assert T().attr == "attr" + """) def test_dont_rewrite(self) -> None: s = """'PYTEST_DONT_REWRITE'\nassert 14""" @@ -341,6 +544,34 @@ def test_assertion_messages_bytes(self, pytester: Pytester) -> None: assert result.ret == 1 result.stdout.fnmatch_lines(["*AssertionError: b'ohai!'", "*assert False"]) + def test_assertion_message_verbosity(self, pytester: Pytester) -> None: + """ + Obey verbosity levels when printing the "message" part of assertions, when they are + non-strings (#6682). + """ + pytester.makepyfile( + """ + class LongRepr: + + def __repr__(self): + return "A" * 500 + + def test_assertion_verbosity(): + assert False, LongRepr() + """ + ) + # Normal verbosity: assertion message gets abbreviated. + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.re_match_lines( + [r".*AssertionError: A+\.\.\.A+$", ".*assert False"] + ) + + # High-verbosity: do not abbreviate the assertion message. + result = pytester.runpytest("-vv") + assert result.ret == 1 + result.stdout.re_match_lines([r".*AssertionError: A+$", ".*assert False"]) + def test_boolop(self) -> None: def f1() -> None: f = g = False @@ -744,6 +975,23 @@ def __repr__(self): assert "UnicodeDecodeError" not in msg assert "UnicodeEncodeError" not in msg + def test_assert_fixture(self, pytester: Pytester) -> None: + pytester.makepyfile( + """\ + import pytest + @pytest.fixture + def fixt(): + return 42 + + def test_something(): # missing "fixt" argument + assert fixt == 42 + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + ["*assert )> == 42*"] + ) + class TestRewriteOnImport: def test_pycache_is_a_file(self, pytester: Pytester) -> None: @@ -787,10 +1035,6 @@ def test_zipfile(self, pytester: Pytester) -> None: ) assert pytester.runpytest().ret == ExitCode.NO_TESTS_COLLECTED - @pytest.mark.skipif( - sys.version_info < (3, 9), - reason="importlib.resources.files was introduced in 3.9", - ) def test_load_resource_via_files_with_rewrite(self, pytester: Pytester) -> None: example = pytester.path.joinpath("demo") / "example" init = pytester.path.joinpath("demo") / "__init__.py" @@ -953,7 +1197,23 @@ def test_rewrite_warning(self, pytester: Pytester) -> None: ) # needs to be a subprocess because pytester explicitly disables this warning result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines(["*Module already imported*: _pytest"]) + result.stdout.fnmatch_lines(["*Module already imported*; _pytest"]) + + def test_rewrite_warning_ignore(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + import pytest + pytest.register_assert_rewrite("_pytest") + """ + ) + # needs to be a subprocess because pytester explicitly disables this warning + result = pytester.runpytest_subprocess( + "-W", + "ignore:Module already imported so cannot be rewritten; _pytest:pytest.PytestAssertRewriteWarning", + ) + # Previously, when the message pattern used to contain an extra `:`, an error was raised. + assert not result.stderr.str().strip() + result.stdout.no_fnmatch_line("*Module already imported*; _pytest") def test_rewrite_module_imported_from_conftest(self, pytester: Pytester) -> None: pytester.makeconftest( @@ -1292,7 +1552,9 @@ def test_simple_failure(): result.stdout.fnmatch_lines(["*E*assert (1 + 1) == 3"]) -class TestIssue10743: +class TestAssertionRewriteWalrusOperator: + """See #10743""" + def test_assertion_walrus_operator(self, pytester: Pytester) -> None: pytester.makepyfile( """ @@ -1459,6 +1721,22 @@ def test_walrus_operator_not_override_value(): result = pytester.runpytest() assert result.ret == 0 + def test_assertion_namedexpr_compare_left_overwrite( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + """ + def test_namedexpr_compare_left_overwrite(): + a = "Hello" + b = "World" + c = "Test" + assert (a := b) == c and (a := "Test") == "Test" + """ + ) + result = pytester.runpytest() + assert result.ret == 1 + result.stdout.fnmatch_lines(["*assert ('World' == 'Test'*"]) + class TestIssue11028: def test_assertion_walrus_operator_in_operand(self, pytester: Pytester) -> None: @@ -1632,7 +1910,7 @@ class TestEarlyRewriteBailout: @pytest.fixture def hook( self, pytestconfig, monkeypatch, pytester: Pytester - ) -> Generator[AssertionRewritingHook, None, None]: + ) -> Generator[AssertionRewritingHook]: """Returns a patched AssertionRewritingHook instance so we can configure its initial paths and track if PathFinder.find_spec has been called. """ @@ -1937,9 +2215,9 @@ def test_simple(): ), ), ) -# fmt: on def test_get_assertion_exprs(src, expected) -> None: assert _get_assertion_exprs(src) == expected +# fmt: on def test_try_makedirs(monkeypatch, tmp_path: Path) -> None: @@ -2003,10 +2281,6 @@ def test_get_cache_dir(self, monkeypatch, prefix, source, expected) -> None: assert get_cache_dir(Path(source)) == Path(expected) - @pytest.mark.skipif( - sys.version_info[:2] == (3, 9) and sys.platform.startswith("win"), - reason="#9298", - ) def test_sys_pycache_prefix_integration( self, tmp_path, monkeypatch, pytester: Pytester ) -> None: diff --git a/testing/test_cacheprovider.py b/testing/test_cacheprovider.py index 72b4265cf75..ca417e86ee5 100644 --- a/testing/test_cacheprovider.py +++ b/testing/test_cacheprovider.py @@ -1,13 +1,13 @@ from __future__ import annotations +from collections.abc import Generator +from collections.abc import Sequence from enum import auto from enum import Enum import os from pathlib import Path import shutil from typing import Any -from typing import Generator -from typing import Sequence from _pytest.compat import assert_never from _pytest.config import ExitCode @@ -69,7 +69,7 @@ def test_cache_writefail_cachefile_silent(self, pytester: Pytester) -> None: cache.set("test/broken", []) @pytest.fixture - def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path, None, None]: + def unwritable_cache_dir(self, pytester: Pytester) -> Generator[Path]: cache_dir = pytester.path.joinpath(".pytest_cache") cache_dir.mkdir() mode = cache_dir.stat().st_mode @@ -104,7 +104,7 @@ def test_cache_failure_warns( pytester.makepyfile("def test_error(): raise Exception") result = pytester.runpytest() assert result.ret == 1 - # warnings from nodeids, lastfailed, and stepwise + # warnings from nodeids and lastfailed result.stdout.fnmatch_lines( [ # Validate location/stacklevel of warning from cacheprovider. @@ -113,7 +113,7 @@ def test_cache_failure_warns( " */cacheprovider.py:*: PytestCacheWarning: could not create cache path " f"{unwritable_cache_dir}/v/cache/nodeids: *", ' config.cache.set("cache/nodeids", sorted(self.cached_nodeids))', - "*1 failed, 3 warnings in*", + "*1 failed, 2 warnings in*", ] ) diff --git a/testing/test_capture.py b/testing/test_capture.py index fe6bd7d14fa..11fd18f08ff 100644 --- a/testing/test_capture.py +++ b/testing/test_capture.py @@ -1,16 +1,17 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator import contextlib import io from io import UnsupportedOperation import os +import re import subprocess import sys import textwrap from typing import BinaryIO from typing import cast -from typing import Generator from typing import TextIO from _pytest import capture @@ -75,7 +76,7 @@ def test_capturing_basic_api(self, method) -> None: assert outerr == ("", "") print("hello") capman.suspend_global_capture() - out, err = capman.read_global_capture() + out, _err = capman.read_global_capture() if method == "no": assert old == (sys.stdout, sys.stderr, sys.stdin) else: @@ -83,7 +84,7 @@ def test_capturing_basic_api(self, method) -> None: capman.resume_global_capture() print("hello") capman.suspend_global_capture() - out, err = capman.read_global_capture() + out, _err = capman.read_global_capture() if method != "no": assert out == "hello\n" capman.stop_global_capturing() @@ -445,6 +446,38 @@ def test_hello(capsys): ) reprec.assertoutcome(passed=1) + def test_capteesys(self, pytester: Pytester) -> None: + p = pytester.makepyfile( + """\ + import sys + def test_one(capteesys): + print("sTdoUt") + print("sTdeRr", file=sys.stderr) + out, err = capteesys.readouterr() + assert out == "sTdoUt\\n" + assert err == "sTdeRr\\n" + """ + ) + # -rN and --capture=tee-sys means we'll read them on stdout/stderr, + # as opposed to both being reported on stdout + result = pytester.runpytest(p, "--quiet", "--quiet", "-rN", "--capture=tee-sys") + assert result.ret == ExitCode.OK + result.stdout.fnmatch_lines(["sTdoUt"]) # tee'd out + result.stderr.fnmatch_lines(["sTdeRr"]) # tee'd out + + result = pytester.runpytest(p, "--quiet", "--quiet", "-rA", "--capture=tee-sys") + assert result.ret == ExitCode.OK + result.stdout.fnmatch_lines( + ["sTdoUt", "sTdoUt", "sTdeRr"] + ) # tee'd out, the next two reported + result.stderr.fnmatch_lines(["sTdeRr"]) # tee'd out + + # -rA and --capture=sys means we'll read them on stdout. + result = pytester.runpytest(p, "--quiet", "--quiet", "-rA", "--capture=sys") + assert result.ret == ExitCode.OK + result.stdout.fnmatch_lines(["sTdoUt", "sTdeRr"]) # no tee, just reported + assert not result.stderr.lines + def test_capsyscapfd(self, pytester: Pytester) -> None: p = pytester.makepyfile( """\ @@ -530,7 +563,7 @@ def test_hello(capfd): @pytest.mark.parametrize("nl", ("\n", "\r\n", "\r")) def test_cafd_preserves_newlines(self, capfd, nl) -> None: print("test", end=nl) - out, err = capfd.readouterr() + out, _err = capfd.readouterr() assert out.endswith(nl) def test_capfdbinary(self, pytester: Pytester) -> None: @@ -835,7 +868,7 @@ def bad_snap(self): FDCapture.snap = bad_snap """ ) - result = pytester.runpytest_subprocess("-p", "pytest_xyz", "--version") + result = pytester.runpytest_subprocess("-p", "pytest_xyz") result.stderr.fnmatch_lines( ["*in bad_snap", " raise Exception('boom')", "Exception: boom"] ) @@ -939,7 +972,7 @@ def test_captureresult() -> None: @pytest.fixture -def tmpfile(pytester: Pytester) -> Generator[BinaryIO, None, None]: +def tmpfile(pytester: Pytester) -> Generator[BinaryIO]: f = pytester.makepyfile("").open("wb+") yield f if not f.closed: @@ -950,8 +983,13 @@ def tmpfile(pytester: Pytester) -> Generator[BinaryIO, None, None]: def lsof_check(): pid = os.getpid() try: - out = subprocess.check_output(("lsof", "-p", str(pid))).decode() - except (OSError, subprocess.CalledProcessError, UnicodeDecodeError) as exc: + out = subprocess.check_output(("lsof", "-p", str(pid)), timeout=10).decode() + except ( + OSError, + UnicodeDecodeError, + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + ) as exc: # about UnicodeDecodeError, see note on pytester pytest.skip(f"could not run 'lsof' ({exc!r})") yield @@ -1115,7 +1153,7 @@ def test_capture_results_accessible_by_attribute(self) -> None: def test_capturing_readouterr_unicode(self) -> None: with self.getcapture() as cap: print("hxąć") - out, err = cap.readouterr() + out, _err = cap.readouterr() assert out == "hxąć\n" def test_reset_twice_error(self) -> None: @@ -1147,8 +1185,8 @@ def test_capturing_error_recursive(self) -> None: print("cap1") with self.getcapture() as cap2: print("cap2") - out2, err2 = cap2.readouterr() - out1, err1 = cap1.readouterr() + out2, _err2 = cap2.readouterr() + out1, _err1 = cap1.readouterr() assert out1 == "cap1\n" assert out2 == "cap2\n" @@ -1193,8 +1231,8 @@ def test_capturing_error_recursive(self) -> None: print("cap1") with self.getcapture() as cap2: print("cap2") - out2, err2 = cap2.readouterr() - out1, err1 = cap1.readouterr() + out2, _err2 = cap2.readouterr() + out1, _err1 = cap1.readouterr() assert out1 == "cap1\ncap2\n" assert out2 == "cap2\n" @@ -1666,3 +1704,32 @@ def test_logging(): ) result.stdout.no_fnmatch_line("*Captured stderr call*") result.stdout.no_fnmatch_line("*during collection*") + + +def test_libedit_workaround(pytester: Pytester) -> None: + pytester.makeconftest(""" + import pytest + + + def pytest_terminal_summary(config): + capture = config.pluginmanager.getplugin("capturemanager") + capture.suspend_global_capture(in_=True) + + print("Enter 'hi'") + value = input() + print(f"value: {value!r}") + + capture.resume_global_capture() + """) + readline = pytest.importorskip("readline") + backend = getattr(readline, "backend", readline.__doc__) # added in Python 3.13 + print(f"Readline backend: {backend}") + + child = pytester.spawn_pytest("") + child.expect(r"Enter 'hi'") + child.sendline("hi") + rest = child.read().decode("utf8") + print(rest) + match = re.search(r"^value: '(.*)'\r?$", rest, re.MULTILINE) + assert match is not None + assert match.group(1) == "hi" diff --git a/testing/test_collect_imported_tests.py b/testing/test_collect_imported_tests.py new file mode 100644 index 00000000000..28b92e17f6f --- /dev/null +++ b/testing/test_collect_imported_tests.py @@ -0,0 +1,102 @@ +"""Tests for the `collect_imported_tests` configuration value.""" + +from __future__ import annotations + +import textwrap + +from _pytest.pytester import Pytester +import pytest + + +def setup_files(pytester: Pytester) -> None: + src_dir = pytester.mkdir("src") + tests_dir = pytester.mkdir("tests") + src_file = src_dir / "foo.py" + + src_file.write_text( + textwrap.dedent("""\ + class Testament: + def test_collections(self): + pass + + def test_testament(): pass + """), + encoding="utf-8", + ) + + test_file = tests_dir / "foo_test.py" + test_file.write_text( + textwrap.dedent("""\ + from foo import Testament, test_testament + + class TestDomain: + def test(self): + testament = Testament() + assert testament + """), + encoding="utf-8", + ) + + pytester.syspathinsert(src_dir) + + +def test_collect_imports_disabled(pytester: Pytester) -> None: + """ + When collect_imported_tests is disabled, only objects in the + test modules are collected as tests, so the imported names (`Testament` and `test_testament`) + are not collected. + """ + pytester.makeini( + """ + [pytest] + collect_imported_tests = false + """ + ) + + setup_files(pytester) + result = pytester.runpytest("-v", "tests") + result.stdout.fnmatch_lines( + [ + "tests/foo_test.py::TestDomain::test PASSED*", + ] + ) + + # Ensure that the hooks were only called for the collected item. + reprec = result.reprec # type:ignore[attr-defined] + reports = reprec.getreports("pytest_collectreport") + [modified] = reprec.getcalls("pytest_collection_modifyitems") + [item_collected] = reprec.getcalls("pytest_itemcollected") + + assert [x.nodeid for x in reports] == [ + "", + "tests/foo_test.py::TestDomain", + "tests/foo_test.py", + "tests", + ] + assert [x.nodeid for x in modified.items] == ["tests/foo_test.py::TestDomain::test"] + assert item_collected.item.nodeid == "tests/foo_test.py::TestDomain::test" + + +@pytest.mark.parametrize("configure_ini", [False, True]) +def test_collect_imports_enabled(pytester: Pytester, configure_ini: bool) -> None: + """ + When collect_imported_tests is enabled (the default), all names in the + test modules are collected as tests. + """ + if configure_ini: + pytester.makeini( + """ + [pytest] + collect_imported_tests = true + """ + ) + + setup_files(pytester) + result = pytester.runpytest("-v", "tests") + result.stdout.fnmatch_lines( + [ + "tests/foo_test.py::Testament::test_collections PASSED*", + "tests/foo_test.py::test_testament PASSED*", + "tests/foo_test.py::TestDomain::test PASSED*", + ] + ) diff --git a/testing/test_collection.py b/testing/test_collection.py index f5822240335..39753d80cac 100644 --- a/testing/test_collection.py +++ b/testing/test_collection.py @@ -1,15 +1,17 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Sequence import os from pathlib import Path +from pathlib import PurePath import pprint import shutil import sys import tempfile import textwrap -from _pytest.assertion.util import running_on_ci +from _pytest.compat import running_on_ci from _pytest.config import ExitCode from _pytest.fixtures import FixtureRequest from _pytest.main import _in_venv @@ -152,8 +154,17 @@ def test_ignored_certain_directories(self, pytester: Pytester) -> None: assert "test_notfound" not in s assert "test_found" in s - def test_ignored_virtualenvs(self, pytester: Pytester) -> None: - ensure_file(pytester.path / "virtual" / "pyvenv.cfg") + known_environment_types = pytest.mark.parametrize( + "env_path", + [ + pytest.param(PurePath("pyvenv.cfg"), id="venv"), + pytest.param(PurePath("conda-meta", "history"), id="conda"), + ], + ) + + @known_environment_types + def test_ignored_virtualenvs(self, pytester: Pytester, env_path: PurePath) -> None: + ensure_file(pytester.path / "virtual" / env_path) testfile = ensure_file(pytester.path / "virtual" / "test_invenv.py") testfile.write_text("def test_hello(): pass", encoding="utf-8") @@ -167,11 +178,12 @@ def test_ignored_virtualenvs(self, pytester: Pytester) -> None: result = pytester.runpytest("virtual") assert "test_invenv" in result.stdout.str() + @known_environment_types def test_ignored_virtualenvs_norecursedirs_precedence( - self, pytester: Pytester + self, pytester: Pytester, env_path ) -> None: # norecursedirs takes priority - ensure_file(pytester.path / ".virtual" / "pyvenv.cfg") + ensure_file(pytester.path / ".virtual" / env_path) testfile = ensure_file(pytester.path / ".virtual" / "test_invenv.py") testfile.write_text("def test_hello(): pass", encoding="utf-8") result = pytester.runpytest("--collect-in-virtualenv") @@ -180,13 +192,14 @@ def test_ignored_virtualenvs_norecursedirs_precedence( result = pytester.runpytest("--collect-in-virtualenv", ".virtual") assert "test_invenv" in result.stdout.str() - def test__in_venv(self, pytester: Pytester) -> None: + @known_environment_types + def test__in_venv(self, pytester: Pytester, env_path: PurePath) -> None: """Directly test the virtual env detection function""" - # no pyvenv.cfg, not a virtualenv + # no env path, not a env base_path = pytester.mkdir("venv") assert _in_venv(base_path) is False - # with pyvenv.cfg, totally a virtualenv - base_path.joinpath("pyvenv.cfg").touch() + # with env path, totally a env + ensure_file(base_path.joinpath(env_path)) assert _in_venv(base_path) is True def test_custom_norecursedirs(self, pytester: Pytester) -> None: @@ -231,20 +244,20 @@ def test_testpaths_ini(self, pytester: Pytester, monkeypatch: MonkeyPatch) -> No # executing from rootdir only tests from `testpaths` directories # are collected - items, reprec = pytester.inline_genitems("-v") + items, _reprec = pytester.inline_genitems("-v") assert [x.name for x in items] == ["test_b", "test_c"] # check that explicitly passing directories in the command-line # collects the tests for dirname in ("a", "b", "c"): - items, reprec = pytester.inline_genitems(tmp_path.joinpath(dirname)) + items, _reprec = pytester.inline_genitems(tmp_path.joinpath(dirname)) assert [x.name for x in items] == [f"test_{dirname}"] # changing cwd to each subdirectory and running pytest without # arguments collects the tests in that directory normally for dirname in ("a", "b", "c"): monkeypatch.chdir(pytester.path.joinpath(dirname)) - items, reprec = pytester.inline_genitems() + items, _reprec = pytester.inline_genitems() assert [x.name for x in items] == [f"test_{dirname}"] def test_missing_permissions_on_unselected_directory_doesnt_crash( @@ -628,10 +641,10 @@ def test_collect_two_commandline_args(self, pytester: Pytester) -> None: def test_serialization_byid(self, pytester: Pytester) -> None: pytester.makepyfile("def test_func(): pass") - items, hookrec = pytester.inline_genitems() + items, _hookrec = pytester.inline_genitems() assert len(items) == 1 (item,) = items - items2, hookrec = pytester.inline_genitems(item.nodeid) + items2, _hookrec = pytester.inline_genitems(item.nodeid) (item2,) = items2 assert item2.name == item.name assert item2.path == item.path @@ -661,7 +674,7 @@ def test_collect_parametrized_order(self, pytester: Pytester) -> None: def test_param(i): ... """ ) - items, hookrec = pytester.inline_genitems(f"{p}::test_param") + items, _hookrec = pytester.inline_genitems(f"{p}::test_param") assert len(items) == 3 assert [item.nodeid for item in items] == [ "test_collect_parametrized_order.py::test_param[0]", @@ -720,7 +733,7 @@ def test_2(): """ ) shutil.copy(p, p.parent / (p.stem + "2" + ".py")) - items, reprec = pytester.inline_genitems(p.parent) + items, _reprec = pytester.inline_genitems(p.parent) assert len(items) == 4 for numi, i in enumerate(items): for numj, j in enumerate(items): @@ -746,7 +759,7 @@ def testmethod_two(self, arg0): pass """ ) - items, reprec = pytester.inline_genitems(p) + items, _reprec = pytester.inline_genitems(p) assert len(items) == 4 assert items[0].name == "testone" assert items[1].name == "testmethod_one" @@ -774,7 +787,7 @@ def test_classmethod(cls) -> None: pass """ ) - items, reprec = pytester.inline_genitems(p) + items, _reprec = pytester.inline_genitems(p) ids = [x.getmodpath() for x in items] # type: ignore[attr-defined] assert ids == ["TestCase.test_classmethod"] @@ -799,7 +812,7 @@ def test_y(self): pass """ ) - items, reprec = pytester.inline_genitems(p) + items, _reprec = pytester.inline_genitems(p) ids = [x.getmodpath() for x in items] # type: ignore[attr-defined] assert ids == ["MyTestSuite.x_test", "TestCase.test_y"] @@ -1272,7 +1285,7 @@ def test_1(): """ ) result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed in*"]) + result.assert_outcomes(passed=1) assert result.ret == 0 @@ -1336,11 +1349,11 @@ def test_collect_pyargs_with_testpaths( with monkeypatch.context() as mp: mp.chdir(root) result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines(["*1 passed in*"]) + result.assert_outcomes(passed=1) def test_initial_conftests_with_testpaths(pytester: Pytester) -> None: - """The testpaths ini option should load conftests in those paths as 'initial' (#10987).""" + """The testpaths config option should load conftests in those paths as 'initial' (#10987).""" p = pytester.mkdir("some_path") p.joinpath("conftest.py").write_text( textwrap.dedent( @@ -1603,7 +1616,7 @@ def __init__(self, name, parent, x): self.x = x @classmethod - def from_parent(cls, parent, *, name, x): + def from_parent(cls, parent, *, name, x): # type: ignore[override] return super().from_parent(parent=parent, name=name, x=x) collector = MyCollector.from_parent(parent=request.session, name="foo", x=10) @@ -1768,6 +1781,41 @@ def test_collect_short_file_windows(pytester: Pytester) -> None: assert result.parseoutcomes() == {"passed": 1} +def test_collect_short_file_windows_multi_level_symlink( + pytester: Pytester, + request: FixtureRequest, +) -> None: + """Regression test for multi-level Windows short-path comparison with + symlinks. + + Previously, when matching collection arguments against collected nodes on + Windows, the short path fallback resolved symlinks. With a chain a -> b -> + target, comparing 'a' against 'b' would incorrectly succeed because both + resolved to 'target', which could cause incorrect matching or duplicate + collection. + """ + # Prepare target directory with a test file. + short_path = Path(tempfile.mkdtemp()) + request.addfinalizer(lambda: shutil.rmtree(short_path, ignore_errors=True)) + target = short_path / "target" + target.mkdir() + (target / "test_chain.py").write_text("def test_chain(): pass", encoding="UTF-8") + + # Create multi-level symlink chain: a -> b -> target. + b = short_path / "b" + a = short_path / "a" + symlink_or_skip(target, b, target_is_directory=True) + symlink_or_skip(b, a, target_is_directory=True) + + # Collect via the first symlink; should find exactly one test. + result = pytester.runpytest(a) + result.assert_outcomes(passed=1) + + # Collect via the intermediate symlink; also exactly one test. + result = pytester.runpytest(b) + result.assert_outcomes(passed=1) + + def test_pyargs_collection_tree(pytester: Pytester, monkeypatch: MonkeyPatch) -> None: """When using `--pyargs`, the collection tree of a pyargs collection argument should only include parents in the import path, not up to confcutdir. @@ -1824,7 +1872,8 @@ def test_do_not_collect_symlink_siblings( """ # Use tmp_path because it creates a symlink with the name "current" next to the directory it creates. symlink_path = tmp_path.parent / (tmp_path.name[:-1] + "current") - assert symlink_path.is_symlink() is True + if not symlink_path.is_symlink(): # pragma: no cover + pytest.skip("Symlinks not supported in this environment") # Create test file. tmp_path.joinpath("test_foo.py").write_text("def test(): pass", encoding="UTF-8") @@ -1866,3 +1915,884 @@ def test_respect_system_exceptions( result.stdout.fnmatch_lines([f"*{head}*"]) result.stdout.fnmatch_lines([msg]) result.stdout.no_fnmatch_line(f"*{tail}*") + + +def test_yield_disallowed_in_tests(pytester: Pytester): + """Ensure generator test functions with 'yield' fail collection (#12960).""" + pytester.makepyfile( + """ + def test_with_yield(): + yield 1 + """ + ) + result = pytester.runpytest() + assert result.ret == 2 + result.stdout.fnmatch_lines( + ["*'yield' keyword is allowed in fixtures, but not in tests (test_with_yield)*"] + ) + # Assert that no tests were collected + result.stdout.fnmatch_lines(["*collected 0 items*"]) + + +def test_annotations_deferred_future(pytester: Pytester): + """Ensure stringified annotations don't raise any errors.""" + pytester.makepyfile( + """ + from __future__ import annotations + import pytest + + @pytest.fixture + def func() -> X: ... # X is undefined + + def test_func(): + assert True + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + +@pytest.mark.skipif( + sys.version_info < (3, 14), reason="Annotations are only skipped on 3.14+" +) +def test_annotations_deferred_314(pytester: Pytester): + """Ensure annotation eval is deferred.""" + pytester.makepyfile( + """ + import pytest + + @pytest.fixture + def func() -> X: ... # X is undefined + + def test_func(): + assert True + """ + ) + result = pytester.runpytest() + assert result.ret == 0 + result.stdout.fnmatch_lines(["*1 passed*"]) + + +@pytest.mark.parametrize("import_mode", ["prepend", "importlib", "append"]) +def test_namespace_packages(pytester: Pytester, import_mode: str): + pytester.makeini( + f""" + [pytest] + consider_namespace_packages = true + pythonpath = . + python_files = *.py + addopts = --import-mode {import_mode} + """ + ) + pytester.makepyfile( + **{ + "pkg/module1.py": "def test_module1(): pass", + "pkg/subpkg_namespace/module2.py": "def test_module1(): pass", + "pkg/subpkg_regular/__init__.py": "", + "pkg/subpkg_regular/module3": "def test_module3(): pass", + } + ) + + # should collect when called with top-level package correctly + result = pytester.runpytest("--collect-only", "--pyargs", "pkg") + result.stdout.fnmatch_lines( + [ + "collected 3 items", + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + ] + ) + + # should also work when called against a more specific subpackage/module + result = pytester.runpytest("--collect-only", "--pyargs", "pkg.subpkg_namespace") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "", + " ", + " ", + " ", + ] + ) + + result = pytester.runpytest("--collect-only", "--pyargs", "pkg.subpkg_regular") + result.stdout.fnmatch_lines( + [ + "collected 1 item", + "", + " ", + " ", + " ", + ] + ) + + +class TestOverlappingCollectionArguments: + """Test that overlapping collection arguments (e.g. `pytest a/b a + a/c::TestIt) are handled correctly (#12083).""" + + @pytest.mark.parametrize("args", [("a", "a/b"), ("a/b", "a")]) + def test_parent_child(self, pytester: Pytester, args: tuple[str, ...]) -> None: + """Test that 'pytest a a/b' and `pytest a/b a` collects all tests from 'a'.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a1(): pass + def test_a2(): pass + """, + "a/b/test_b.py": """ + def test_b1(): pass + def test_b2(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", *args) + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_multiple_nested_paths(self, pytester: Pytester) -> None: + """Test that 'pytest a/b a a/b/c' collects all tests from 'a'.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a(): pass + """, + "a/b/test_b.py": """ + def test_b(): pass + """, + "a/b/c/test_c.py": """ + def test_c(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", "a/b", "a", "a/b/c") + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_same_path_twice(self, pytester: Pytester) -> None: + """Test that 'pytest a a' doesn't duplicate tests.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", "a", "a") + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_keep_duplicates_flag(self, pytester: Pytester) -> None: + """Test that --keep-duplicates allows duplication.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a(): pass + """, + "a/b/test_b.py": """ + def test_b(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", "--keep-duplicates", "a", "a/b") + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_specific_file_then_parent_dir(self, pytester: Pytester) -> None: + """Test that 'pytest a/test_a.py a' collects all tests from 'a'.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a(): pass + """, + "a/test_other.py": """ + def test_other(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", "a/test_a.py", "a") + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_package_scope_fixture_with_overlapping_paths( + self, pytester: Pytester + ) -> None: + """Test that package-scoped fixtures work correctly with overlapping paths.""" + pytester.makepyfile( + **{ + "pkg/__init__.py": "", + "pkg/test_pkg.py": """ + import pytest + + counter = {"value": 0} + + @pytest.fixture(scope="package") + def pkg_fixture(): + counter["value"] += 1 + return counter["value"] + + def test_pkg1(pkg_fixture): + assert pkg_fixture == 1 + + def test_pkg2(pkg_fixture): + assert pkg_fixture == 1 + """, + "pkg/sub/__init__.py": "", + "pkg/sub/test_sub.py": """ + def test_sub(): pass + """, + } + ) + + # Package fixture should run only once even with overlapping paths. + result = pytester.runpytest("pkg", "pkg/sub", "pkg", "-v") + result.assert_outcomes(passed=3) + + def test_execution_order_preserved(self, pytester: Pytester) -> None: + """Test that test execution order follows argument order.""" + pytester.makepyfile( + **{ + "a/test_a.py": """ + def test_a(): pass + """, + "b/test_b.py": """ + def test_b(): pass + """, + } + ) + + result = pytester.runpytest("--collect-only", "b", "a", "b/test_b.py::test_b") + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_overlapping_node_ids_class_and_method(self, pytester: Pytester) -> None: + """Test that overlapping node IDs are handled correctly.""" + pytester.makepyfile( + test_nodeids=""" + class TestClass: + def test_method1(self): pass + def test_method2(self): pass + def test_method3(self): pass + + def test_function(): pass + """ + ) + + # Class then specific method. + result = pytester.runpytest( + "--collect-only", + "test_nodeids.py::TestClass", + "test_nodeids.py::TestClass::test_method2", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + # Specific method then class. + result = pytester.runpytest( + "--collect-only", + "test_nodeids.py::TestClass::test_method3", + "test_nodeids.py::TestClass", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_overlapping_node_ids_file_and_class(self, pytester: Pytester) -> None: + """Test that file-level and class-level selections work correctly.""" + pytester.makepyfile( + test_file=""" + class TestClass: + def test_method(self): pass + + class TestOther: + def test_other(self): pass + + def test_function(): pass + """ + ) + + # File then class. + result = pytester.runpytest( + "--collect-only", "test_file.py", "test_file.py::TestClass" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + # Class then file. + result = pytester.runpytest( + "--collect-only", "test_file.py::TestClass", "test_file.py" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_same_node_id_twice(self, pytester: Pytester) -> None: + """Test that the same node ID specified twice is collected only once.""" + pytester.makepyfile( + test_dup=""" + def test_one(): pass + def test_two(): pass + """ + ) + + result = pytester.runpytest( + "--collect-only", + "test_dup.py::test_one", + "test_dup.py::test_one", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_overlapping_with_parametrization(self, pytester: Pytester) -> None: + """Test overlapping with parametrized tests.""" + pytester.makepyfile( + test_param=""" + import pytest + + @pytest.mark.parametrize("n", [1, 2]) + def test_param(n): pass + + class TestClass: + @pytest.mark.parametrize("x", ["a", "b"]) + def test_method(self, x): pass + """ + ) + + result = pytester.runpytest( + "--collect-only", + "test_param.py::test_param[2]", + "test_param.py::TestClass::test_method[a]", + "test_param.py", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest( + "--collect-only", + "test_param.py::test_param[2]", + "test_param.py::test_param", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + @pytest.mark.parametrize("order", [(".", "a"), ("a", ".")]) + def test_root_and_subdir(self, pytester: Pytester, order: tuple[str, ...]) -> None: + """Test that '. a' and 'a .' both collect all tests.""" + pytester.makepyfile( + test_root=""" + def test_root(): pass + """, + **{ + "a/test_a.py": """ + def test_a(): pass + """, + }, + ) + + result = pytester.runpytest("--collect-only", *order) + + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + def test_complex_combined_handling(self, pytester: Pytester) -> None: + """Test some scenarios in a complex hierarchy.""" + pytester.makepyfile( + **{ + "top1/__init__.py": "", + "top1/test_1.py": ( + """ + def test_1(): pass + + class TestIt: + def test_2(): pass + + def test_3(): pass + """ + ), + "top1/test_2.py": ( + """ + def test_1(): pass + """ + ), + "top2/__init__.py": "", + "top2/test_1.py": ( + """ + def test_1(): pass + """ + ), + }, + ) + + result = pytester.runpytest_inprocess("--collect-only", ".") + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess("--collect-only", "top2", "top1") + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top1", "top1/test_2.py" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + # NOTE: Also sensible arguably even without --keep-duplicates. + # " ", + # " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top1/test_2.py", "top1" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + # NOTE: Ideally test_2 would come before test_1 here. + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "--keep-duplicates", "top1/test_2.py", "top1" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top1/test_2.py", "top1/test_2.py" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + # NOTE: Also sensible arguably even without --keep-duplicates. + # " ", + # " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess("--collect-only", "top2/", "top2/") + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + # NOTE: Also sensible arguably even without --keep-duplicates. + # " ", + # " ", + # " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top2/", "top2/", "top2/test_1.py" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + # NOTE: Also sensible arguably even without --keep-duplicates. + # " ", + # " ", + # " ", + # " ", + # " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top1/test_1.py", "top1/test_1.py::test_3" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + " ", + " ", + " ", + " ", + # NOTE: Also sensible arguably even without --keep-duplicates. + # " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", "top1/test_1.py::test_3", "top1/test_1.py" + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + " ", + # NOTE: Ideally test_3 would come before the others here. + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + result = pytester.runpytest_inprocess( + "--collect-only", + "--keep-duplicates", + "top1/test_1.py::test_3", + "top1/test_1.py", + ) + result.stdout.fnmatch_lines( + [ + "", + " ", + # NOTE: That is duplicated here is not great. + " ", + " ", + " ", + " ", + " ", + " ", + " ", + "", + ], + consecutive=True, + ) + + +@pytest.mark.parametrize( + ["x_y", "expected_duplicates"], + [ + ( + [(1, 1), (1, 1)], + ["1-1"], + ), + ( + [(1, 1), (1, 2), (1, 1)], + ["1-1"], + ), + ( + [(1, 1), (2, 2), (1, 1)], + ["1-1"], + ), + ( + [(1, 1), (2, 2), (1, 2), (2, 1), (1, 1), (2, 1)], + ["1-1", "2-1"], + ), + ], +) +@pytest.mark.parametrize("option_name", ["strict_parametrization_ids", "strict"]) +def test_strict_parametrization_ids( + pytester: Pytester, + x_y: Sequence[tuple[int, int]], + expected_duplicates: Sequence[str], + option_name: str, +) -> None: + pytester.makeini( + f""" + [pytest] + {option_name} = true + """ + ) + pytester.makepyfile( + f""" + import pytest + + @pytest.mark.parametrize(["x", "y"], {x_y}) + def test1(x, y): + pass + """ + ) + + result = pytester.runpytest() + + assert result.ret == ExitCode.INTERRUPTED + expected_parametersets = ", ".join(str(list(p)) for p in x_y) + expected_ids = ", ".join(f"{x}-{y}" for x, y in x_y) + result.stdout.fnmatch_lines( + [ + "Duplicate parametrization IDs detected*", + "", + "Test name: *::test1", + "Parameters: x, y", + f"Parameter sets: {expected_parametersets}", + f"IDs: {expected_ids}", + f"Duplicates: {', '.join(expected_duplicates)}", + "", + "You can fix this problem using *", + ] + ) + + +def test_strict_parametrization_ids_with_hidden_param(pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + strict_parametrization_ids = true + """ + ) + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize(["x"], ["a", pytest.param("a", id=pytest.HIDDEN_PARAM), "a"]) + def test1(x): + pass + """ + ) + + result = pytester.runpytest() + + assert result.ret == ExitCode.INTERRUPTED + result.stdout.fnmatch_lines( + [ + "Duplicate parametrization IDs detected*", + "IDs: a, , a", + "Duplicates: a", + ] + ) diff --git a/testing/test_compat.py b/testing/test_compat.py index 2c6b0269c27..fa9e259647f 100644 --- a/testing/test_compat.py +++ b/testing/test_compat.py @@ -5,33 +5,18 @@ from functools import cached_property from functools import partial from functools import wraps -import sys from typing import TYPE_CHECKING -from _pytest.compat import _PytestWrapper from _pytest.compat import assert_never from _pytest.compat import get_real_func -from _pytest.compat import is_generator from _pytest.compat import safe_getattr from _pytest.compat import safe_isclass from _pytest.outcomes import OutcomeException -from _pytest.pytester import Pytester import pytest if TYPE_CHECKING: - from typing_extensions import Literal - - -def test_is_generator() -> None: - def zap(): - yield # pragma: no cover - - def foo(): - pass # pragma: no cover - - assert is_generator(zap) - assert not is_generator(foo) + from typing import Literal def test_real_func_loop_limit() -> None: @@ -52,10 +37,7 @@ def __getattr__(self, attr): with pytest.raises( ValueError, - match=( - "could not find real function of \n" - "stopped at " - ), + match=("wrapper loop when unwrapping "), ): get_real_func(evil) @@ -79,10 +61,13 @@ def func(): wrapped_func2 = decorator(decorator(wrapped_func)) assert get_real_func(wrapped_func2) is func - # special case for __pytest_wrapped__ attribute: used to obtain the function up until the point - # a function was wrapped by pytest itself - wrapped_func2.__pytest_wrapped__ = _PytestWrapper(wrapped_func) - assert get_real_func(wrapped_func2) is wrapped_func + # obtain the function up until the point a function was wrapped by pytest itself + @pytest.fixture + def wrapped_func3(): + pass # pragma: no cover + + wrapped_func4 = decorator(wrapped_func3) + assert get_real_func(wrapped_func4) is wrapped_func3._get_wrapped_function() def test_get_real_func_partial() -> None: @@ -95,65 +80,6 @@ def foo(x): assert get_real_func(partial(foo)) is foo -@pytest.mark.skipif(sys.version_info >= (3, 11), reason="coroutine removed") -def test_is_generator_asyncio(pytester: Pytester) -> None: - pytester.makepyfile( - """ - from _pytest.compat import is_generator - import asyncio - @asyncio.coroutine - def baz(): - yield from [1,2,3] - - def test_is_generator_asyncio(): - assert not is_generator(baz) - """ - ) - # avoid importing asyncio into pytest's own process, - # which in turn imports logging (#8) - result = pytester.runpytest_subprocess() - result.stdout.fnmatch_lines(["*1 passed*"]) - - -def test_is_generator_async_syntax(pytester: Pytester) -> None: - pytester.makepyfile( - """ - from _pytest.compat import is_generator - def test_is_generator_py35(): - async def foo(): - await foo() - - async def bar(): - pass - - assert not is_generator(foo) - assert not is_generator(bar) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) - - -def test_is_generator_async_gen_syntax(pytester: Pytester) -> None: - pytester.makepyfile( - """ - from _pytest.compat import is_generator - def test_is_generator(): - async def foo(): - yield - await foo() - - async def bar(): - yield - - assert not is_generator(foo) - assert not is_generator(bar) - """ - ) - result = pytester.runpytest() - result.stdout.fnmatch_lines(["*1 passed*"]) - - class ErrorsHelper: @property def raise_baseexception(self): diff --git a/testing/test_config.py b/testing/test_config.py index 232839399e2..f086778ad1e 100644 --- a/testing/test_config.py +++ b/testing/test_config.py @@ -1,15 +1,16 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Sequence import dataclasses import importlib.metadata import os from pathlib import Path +import platform import re import sys import textwrap from typing import Any -from typing import Sequence import _pytest._code from _pytest.config import _get_plugin_specs_as_list @@ -22,6 +23,7 @@ from _pytest.config.argparsing import get_ini_default_for_type from _pytest.config.argparsing import Parser from _pytest.config.exceptions import UsageError +from _pytest.config.findpaths import ConfigValue from _pytest.config.findpaths import determine_setup from _pytest.config.findpaths import get_common_ancestor from _pytest.config.findpaths import locate_config @@ -55,10 +57,10 @@ def test_getcfg_and_config( ), encoding="utf-8", ) - _, _, cfg = locate_config(Path.cwd(), [sub]) - assert cfg["name"] == "value" + _, _, cfg, _ = locate_config(Path.cwd(), [sub]) + assert cfg["name"] == ConfigValue("value", origin="file", mode="ini") config = pytester.parseconfigure(str(sub)) - assert config.inicfg["name"] == "value" + assert config._inicfg["name"] == ConfigValue("value", origin="file", mode="ini") def test_setupcfg_uses_toolpytest_with_pytest(self, pytester: Pytester) -> None: p1 = pytester.makepyfile("def test(): pass") @@ -130,6 +132,20 @@ def test_ini_names(self, pytester: Pytester, name, section) -> None: config = pytester.parseconfig() assert config.getini("minversion") == "3.36" + @pytest.mark.parametrize("name", ["pytest.toml", ".pytest.toml"]) + def test_toml_config_names(self, pytester: Pytester, name: str) -> None: + pytester.path.joinpath(name).write_text( + textwrap.dedent( + """ + [pytest] + minversion = "3.36" + """ + ), + encoding="utf-8", + ) + config = pytester.parseconfig() + assert config.getini("minversion") == "3.36" + def test_pyproject_toml(self, pytester: Pytester) -> None: pyproject_toml = pytester.makepyprojecttoml( """ @@ -149,7 +165,7 @@ def test_empty_pyproject_toml(self, pytester: Pytester) -> None: def test_empty_pyproject_toml_found_many(self, pytester: Pytester) -> None: """ - In case we find multiple pyproject.toml files in our search, without a [tool.pytest.ini_options] + In case we find multiple pyproject.toml files in our search, without a [tool.pytest] table and without finding other candidates, the closest to where we started wins. """ pytester.makefile( @@ -163,9 +179,88 @@ def test_empty_pyproject_toml_found_many(self, pytester: Pytester) -> None: config = pytester.parseconfig(pytester.path / "foo/bar") assert config.inipath == pytester.path / "foo/bar/pyproject.toml" + def test_pytest_toml(self, pytester: Pytester) -> None: + pytest_toml = pytester.path.joinpath("pytest.toml") + pytest_toml = pytester.maketoml( + """ + [pytest] + minversion = "1.0" + """ + ) + config = pytester.parseconfig() + assert config.inipath == pytest_toml + assert config.getini("minversion") == "1.0" + + @pytest.mark.parametrize("name", ["pytest.toml", ".pytest.toml"]) + def test_empty_pytest_toml(self, pytester: Pytester, name: str) -> None: + """An empty pytest.toml is considered as config if no other option is found.""" + pytest_toml = pytester.path / name + pytest_toml.write_text("", encoding="utf-8") + config = pytester.parseconfig() + assert config.inipath == pytest_toml + + def test_pytest_toml_trumps_pyproject_toml(self, pytester: Pytester) -> None: + """A pytest.toml always takes precedence over a pyproject.toml file.""" + pytester.makepyprojecttoml( + """ + [tool.pytest] + minversion = "1.0" + """ + ) + pytest_toml = pytester.maketoml( + """ + [pytest] + minversion = "2.0" + """ + ) + config = pytester.parseconfig() + assert config.inipath == pytest_toml + assert config.getini("minversion") == "2.0" + + def test_pytest_toml_trumps_pytest_ini(self, pytester: Pytester) -> None: + """A pytest.toml always takes precedence over a pytest.ini file.""" + pytester.makeini( + """ + [pytest] + minversion = 1.0 + """, + ) + pytest_toml = pytester.maketoml( + """ + [pytest] + minversion = "2.0" + """, + ) + config = pytester.parseconfig() + assert config.inipath == pytest_toml + assert config.getini("minversion") == "2.0" + + def test_dot_pytest_toml_trumps_pytest_ini(self, pytester: Pytester) -> None: + """A .pytest.toml always takes precedence over a pytest.ini file.""" + pytester.makeini( + """ + [pytest] + minversion = 1.0 + """, + ) + pytest_toml = pytester.maketoml( + """ + [pytest] + minversion = "2.0" + """ + ) + config = pytester.parseconfig() + assert config.inipath == pytest_toml + assert config.getini("minversion") == "2.0" + def test_pytest_ini_trumps_pyproject_toml(self, pytester: Pytester) -> None: """A pytest.ini always take precedence over a pyproject.toml file.""" - pytester.makepyprojecttoml("[tool.pytest.ini_options]") + pytester.makepyprojecttoml( + """ + [tool.pytest] + minversion = "1.0" + """ + ) pytest_ini = pytester.makefile(".ini", pytest="") config = pytester.parseconfig() assert config.inipath == pytest_ini @@ -211,6 +306,17 @@ def test_toml_parse_error(self, pytester: Pytester) -> None: assert result.ret != 0 result.stderr.fnmatch_lines("ERROR: *pyproject.toml: Invalid statement*") + def test_pytest_toml_parse_error(self, pytester: Pytester) -> None: + pytester.path.joinpath("pytest.toml").write_text( + """ + \\" + """, + encoding="utf-8", + ) + result = pytester.runpytest() + assert result.ret != 0 + result.stderr.fnmatch_lines("ERROR: *pytest.toml: Invalid statement*") + def test_confcutdir_default_without_configfile(self, pytester: Pytester) -> None: # If --confcutdir is not specified, and there is no configfile, default # to the rootpath. @@ -353,6 +459,22 @@ def test_silence_unknown_key_warning(self, pytester: Pytester) -> None: result = pytester.runpytest() result.stdout.no_fnmatch_line("*PytestConfigWarning*") + @pytest.mark.parametrize("option_name", ["strict_config", "strict"]) + def test_strict_config_ini_option( + self, pytester: Pytester, option_name: str + ) -> None: + """Test that strict_config and strict ini options enable strict config checking.""" + pytester.makeini( + f""" + [pytest] + unknown_option = 1 + {option_name} = True + """ + ) + result = pytester.runpytest() + result.stderr.fnmatch_lines("ERROR: Unknown config option: unknown_option") + assert result.ret == pytest.ExitCode.USAGE_ERROR + @pytest.mark.filterwarnings("default::pytest.PytestConfigWarning") def test_disable_warnings_plugin_disables_config_warnings( self, pytester: Pytester @@ -611,20 +733,14 @@ def pytest_addoption(parser): assert config.getini("custom") == "1" def test_absolute_win32_path(self, pytester: Pytester) -> None: - temp_ini_file = pytester.makefile( - ".ini", - custom=""" - [pytest] - addopts = --version - """, - ) + temp_ini_file = pytester.makeini("[pytest]") from os.path import normpath temp_ini_file_norm = normpath(str(temp_ini_file)) ret = pytest.main(["-c", temp_ini_file_norm]) - assert ret == ExitCode.OK + assert ret == ExitCode.NO_TESTS_COLLECTED ret = pytest.main(["--config-file", temp_ini_file_norm]) - assert ret == ExitCode.OK + assert ret == ExitCode.NO_TESTS_COLLECTED class TestConfigAPI: @@ -636,7 +752,7 @@ def test_config_trace(self, pytester: Pytester) -> None: assert len(values) == 1 assert values[0] == "hello [config]\n" - def test_config_getoption(self, pytester: Pytester) -> None: + def test_config_getoption_declared_option_name(self, pytester: Pytester) -> None: pytester.makeconftest( """ def pytest_addoption(parser): @@ -648,6 +764,18 @@ def pytest_addoption(parser): assert config.getoption(x) == "this" pytest.raises(ValueError, config.getoption, "qweqwe") + config_novalue = pytester.parseconfig() + assert config_novalue.getoption("hello") is None + assert config_novalue.getoption("hello", default=1) is None + assert config_novalue.getoption("hello", default=1, skip=True) == 1 + + def test_config_getoption_undeclared_option_name(self, pytester: Pytester) -> None: + config = pytester.parseconfig() + with pytest.raises(ValueError): + config.getoption("x") + assert config.getoption("x", default=1) == 1 + assert config.getoption("x", default=1, skip=True) == 1 + def test_config_getoption_unicode(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -675,12 +803,6 @@ def pytest_addoption(parser): with pytest.raises(pytest.skip.Exception): config.getvalueorskip("hello") - def test_getoption(self, pytester: Pytester) -> None: - config = pytester.parseconfig() - with pytest.raises(ValueError): - config.getvalue("x") - assert config.getoption("x", 1) == 1 - def test_getconftest_pathlist(self, pytester: Pytester, tmp_path: Path) -> None: somepath = tmp_path.joinpath("x", "y", "z") p = tmp_path.joinpath("conftest.py") @@ -842,6 +964,82 @@ def pytest_addoption(parser): config = pytester.parseconfig() assert config.getini("strip") is bool_val + @pytest.mark.parametrize("str_val, int_val", [("10", 10), ("no-ini", 2)]) + def test_addini_int(self, pytester: Pytester, str_val: str, int_val: bool) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("ini_param", "", type="int", default=2) + """ + ) + if str_val != "no-ini": + pytester.makeini( + f""" + [pytest] + ini_param={str_val} + """ + ) + config = pytester.parseconfig() + assert config.getini("ini_param") == int_val + + def test_addini_int_invalid(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("ini_param", "", type="int", default=2) + """ + ) + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + ini_param=["foo"] + """ + ) + config = pytester.parseconfig() + with pytest.raises( + TypeError, match="Expected an int string for option ini_param" + ): + _ = config.getini("ini_param") + + @pytest.mark.parametrize("str_val, float_val", [("10.5", 10.5), ("no-ini", 2.2)]) + def test_addini_float( + self, pytester: Pytester, str_val: str, float_val: bool + ) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("ini_param", "", type="float", default=2.2) + """ + ) + if str_val != "no-ini": + pytester.makeini( + f""" + [pytest] + ini_param={str_val} + """ + ) + config = pytester.parseconfig() + assert config.getini("ini_param") == float_val + + def test_addini_float_invalid(self, pytester: Pytester) -> None: + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("ini_param", "", type="float", default=2.2) + """ + ) + pytester.makepyprojecttoml( + """ + [tool.pytest.ini_options] + ini_param=["foo"] + """ + ) + config = pytester.parseconfig() + with pytest.raises( + TypeError, match="Expected a float string for option ini_param" + ): + _ = config.getini("ini_param") + def test_addinivalue_line_existing(self, pytester: Pytester) -> None: pytester.makeconftest( """ @@ -917,7 +1115,7 @@ def pytest_addoption(parser): # default for string is "" value = config.getini("string1") assert value == "" - # should return None if None is explicity set as default value + # should return None if None is explicitly set as default value # irrespective of the type argument value = config.getini("none_1") assert value is None @@ -928,6 +1126,166 @@ def pytest_addoption(parser): value = config.getini("no_type") assert value == "" + def test_addini_with_aliases(self, pytester: Pytester) -> None: + """Test that ini options can have aliases.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("new_name", "my option", aliases=["old_name"]) + """ + ) + pytester.makeini( + """ + [pytest] + old_name = hello + """ + ) + config = pytester.parseconfig() + # Should be able to access via canonical name. + assert config.getini("new_name") == "hello" + # Should also be able to access via alias. + assert config.getini("old_name") == "hello" + + def test_addini_aliases_with_canonical_in_file(self, pytester: Pytester) -> None: + """Test that canonical name takes precedence over alias in configuration file.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("new_name", "my option", aliases=["old_name"]) + """ + ) + pytester.makeini( + """ + [pytest] + old_name = from_alias + new_name = from_canonical + """ + ) + config = pytester.parseconfig() + # Canonical name should take precedence. + assert config.getini("new_name") == "from_canonical" + assert config.getini("old_name") == "from_canonical" + + def test_addini_aliases_multiple(self, pytester: Pytester) -> None: + """Test that ini option can have multiple aliases.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("current_name", "my option", aliases=["old_name", "legacy_name"]) + """ + ) + pytester.makeini( + """ + [pytest] + old_name = value1 + """ + ) + config = pytester.parseconfig() + assert config.getini("current_name") == "value1" + assert config.getini("old_name") == "value1" + assert config.getini("legacy_name") == "value1" + + def test_addini_aliases_with_override_of_old(self, pytester: Pytester) -> None: + """Test that aliases work with --override-ini -- ini sets old.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("new_name", "my option", aliases=["old_name"]) + """ + ) + pytester.makeini( + """ + [pytest] + old_name = from_file + """ + ) + # Override using alias. + config = pytester.parseconfig("-o", "old_name=overridden") + assert config.getini("new_name") == "overridden" + assert config.getini("old_name") == "overridden" + + # Override using canonical name. + config = pytester.parseconfig("-o", "new_name=overridden2") + assert config.getini("new_name") == "overridden2" + + def test_addini_aliases_with_override_of_new(self, pytester: Pytester) -> None: + """Test that aliases work with --override-ini -- ini sets new.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("new_name", "my option", aliases=["old_name"]) + """ + ) + pytester.makeini( + """ + [pytest] + new_name = from_file + """ + ) + # Override using alias. + config = pytester.parseconfig("-o", "old_name=overridden") + assert config.getini("new_name") == "overridden" + assert config.getini("old_name") == "overridden" + + # Override using canonical name. + config = pytester.parseconfig("-o", "new_name=overridden2") + assert config.getini("new_name") == "overridden2" + + def test_addini_aliases_with_types(self, pytester: Pytester) -> None: + """Test that aliases work with different types.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("mylist", "list option", type="linelist", aliases=["oldlist"]) + parser.addini("mybool", "bool option", type="bool", aliases=["oldbool"]) + """ + ) + pytester.makeini( + """ + [pytest] + oldlist = line1 + line2 + oldbool = true + """ + ) + config = pytester.parseconfig() + assert config.getini("mylist") == ["line1", "line2"] + assert config.getini("oldlist") == ["line1", "line2"] + assert config.getini("mybool") is True + assert config.getini("oldbool") is True + + def test_addini_aliases_conflict_error(self, pytester: Pytester) -> None: + """Test that registering an alias that conflicts with an existing option raises an error.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("existing", "first option") + + try: + parser.addini("new_option", "second option", aliases=["existing"]) + except ValueError as e: + assert "alias 'existing' conflicts with existing configuration option" in str(e) + else: + assert False, "Should have raised ValueError" + """ + ) + pytester.parseconfig() + + def test_addini_aliases_duplicate_error(self, pytester: Pytester) -> None: + """Test that registering the same alias twice raises an error.""" + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("option1", "first option", aliases=["shared_alias"]) + try: + parser.addini("option2", "second option", aliases=["shared_alias"]) + raise AssertionError("Should have raised ValueError") + except ValueError as e: + assert "'shared_alias' is already an alias of 'option1'" in str(e) + """ + ) + pytester.parseconfig() + @pytest.mark.parametrize( "type, expected", [ @@ -977,6 +1335,37 @@ def test_confcutdir_check_isdir(self, pytester: Pytester) -> None: def test_iter_rewritable_modules(self, names, expected) -> None: assert list(_iter_rewritable_modules(names)) == expected + def test_add_cleanup(self, pytester: Pytester) -> None: + config = Config.fromdictargs({}, []) + config._do_configure() + report = [] + + class MyError(BaseException): + pass + + @config.add_cleanup + def cleanup_last(): + report.append("cleanup_last") + + @config.add_cleanup + def raise_2(): + report.append("raise_2") + raise MyError("raise_2") + + @config.add_cleanup + def raise_1(): + report.append("raise_1") + raise MyError("raise_1") + + @config.add_cleanup + def cleanup_first(): + report.append("cleanup_first") + + with pytest.raises(MyError, match=r"raise_2"): + config._ensure_unconfigure() + + assert report == ["cleanup_first", "raise_1", "raise_2", "cleanup_last"] + class TestConfigFromdictargs: def test_basic_behavior(self, _sys_snapshot) -> None: @@ -1036,7 +1425,7 @@ def test_inifilename(self, tmp_path: Path) -> None: ) with MonkeyPatch.context() as mp: mp.chdir(cwd) - config = Config.fromdictargs(option_dict, ()) + config = Config.fromdictargs(option_dict, []) inipath = absolutepath(inifilename) assert config.args == [str(cwd)] @@ -1045,8 +1434,10 @@ def test_inifilename(self, tmp_path: Path) -> None: # this indicates this is the file used for getting configuration values assert config.inipath == inipath - assert config.inicfg.get("name") == "value" - assert config.inicfg.get("should_not_be_set") is None + assert config._inicfg.get("name") == ConfigValue( + "value", origin="file", mode="ini" + ) + assert config._inicfg.get("should_not_be_set") is None def test_options_on_small_file_do_not_blow_up(pytester: Pytester) -> None: @@ -1201,14 +1592,13 @@ def distributions(): ) -@pytest.mark.parametrize( - "parse_args,should_load", [(("-p", "mytestplugin"), True), ((), False)] -) +@pytest.mark.parametrize("disable_plugin_method", ["env_var", "flag", ""]) +@pytest.mark.parametrize("enable_plugin_method", ["env_var", "flag", ""]) def test_disable_plugin_autoload( pytester: Pytester, monkeypatch: MonkeyPatch, - parse_args: tuple[str, str] | tuple[()], - should_load: bool, + enable_plugin_method: str, + disable_plugin_method: str, ) -> None: class DummyEntryPoint: project_name = name = "mytestplugin" @@ -1229,23 +1619,60 @@ class PseudoPlugin: attrs_used = [] def __getattr__(self, name): - assert name == "__loader__" + assert name in ("__loader__", "__spec__") self.attrs_used.append(name) return object() def distributions(): return (Distribution(),) - monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + parse_args: list[str] = [] + + if disable_plugin_method == "env_var": + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + elif disable_plugin_method == "flag": + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + parse_args.append("--disable-plugin-autoload") + else: + assert disable_plugin_method == "" + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + + if enable_plugin_method == "env_var": + monkeypatch.setenv("PYTEST_PLUGINS", "mytestplugin") + elif enable_plugin_method == "flag": + parse_args.extend(["-p", "mytestplugin"]) + else: + assert enable_plugin_method == "" + monkeypatch.setattr(importlib.metadata, "distributions", distributions) monkeypatch.setitem(sys.modules, "mytestplugin", PseudoPlugin()) config = pytester.parseconfig(*parse_args) + has_loaded = config.pluginmanager.get_plugin("mytestplugin") is not None - assert has_loaded == should_load - if should_load: - assert PseudoPlugin.attrs_used == ["__loader__"] - else: - assert PseudoPlugin.attrs_used == [] + # it should load if it's enabled, or we haven't disabled autoloading + assert has_loaded == (bool(enable_plugin_method) or not disable_plugin_method) + + # The reason for the discrepancy between 'has_loaded' and __loader__ being accessed + # appears to be the monkeypatching of importlib.metadata.distributions; where + # files being empty means that _mark_plugins_for_rewrite doesn't find the plugin. + # But enable_method==flag ends up in mark_rewrite being called and __loader__ + # being accessed. + assert ("__loader__" in PseudoPlugin.attrs_used) == ( + has_loaded + and not (enable_plugin_method in ("env_var", "") and not disable_plugin_method) + ) + + # __spec__ is accessed in AssertionRewritingHook.exec_module, which would be + # eventually called if we did a full pytest run; but it's only accessed with + # enable_plugin_method=="env_var" because that will early-load it. + # Except when autoloads aren't disabled, in which case PytestPluginManager.import_plugin + # bails out before importing it.. because it knows it'll be loaded later? + # The above seems a bit weird, but I *think* it's true. + if platform.python_implementation() != "PyPy": + assert ("__spec__" in PseudoPlugin.attrs_used) == bool( + enable_plugin_method == "env_var" and disable_plugin_method + ) + # __spec__ is present when testing locally on pypy, but not in CI ???? def test_plugin_loading_order(pytester: Pytester) -> None: @@ -1409,7 +1836,6 @@ def pytest_load_initial_conftests(self): ("_pytest.config", "nonwrapper"), (m.__module__, "nonwrapper"), ("_pytest.legacypath", "nonwrapper"), - ("_pytest.python_path", "nonwrapper"), ("_pytest.capture", "wrapper"), ("_pytest.warnings", "wrapper"), ] @@ -1486,33 +1912,39 @@ def test_with_ini(self, tmp_path: Path, name: str, contents: str) -> None: b = a / "b" b.mkdir() for args in ([str(tmp_path)], [str(a)], [str(b)]): - rootpath, parsed_inipath, _ = determine_setup( + rootpath, parsed_inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=args, rootdir_cmd_arg=None, invocation_dir=Path.cwd(), ) assert rootpath == tmp_path assert parsed_inipath == inipath - rootpath, parsed_inipath, ini_config = determine_setup( + rootpath, parsed_inipath, ini_config, _ = determine_setup( inifile=None, + override_ini=None, args=[str(b), str(a)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), ) assert rootpath == tmp_path assert parsed_inipath == inipath - assert ini_config == {"x": "10"} + assert ini_config["x"] == ConfigValue("10", origin="file", mode="ini") - @pytest.mark.parametrize("name", ["setup.cfg", "tox.ini"]) - def test_pytestini_overrides_empty_other(self, tmp_path: Path, name: str) -> None: - inipath = tmp_path / "pytest.ini" + @pytest.mark.parametrize("pytest_ini", ["pytest.ini", ".pytest.ini"]) + @pytest.mark.parametrize("other", ["setup.cfg", "tox.ini"]) + def test_pytestini_overrides_empty_other( + self, tmp_path: Path, pytest_ini: str, other: str + ) -> None: + inipath = tmp_path / pytest_ini inipath.touch() a = tmp_path / "a" a.mkdir() - (a / name).touch() - rootpath, parsed_inipath, _ = determine_setup( + (a / other).touch() + rootpath, parsed_inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=[str(a)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1525,8 +1957,9 @@ def test_setuppy_fallback(self, tmp_path: Path) -> None: a.mkdir() (a / "setup.cfg").touch() (tmp_path / "setup.py").touch() - rootpath, inipath, inicfg = determine_setup( + rootpath, inipath, inicfg, _ = determine_setup( inifile=None, + override_ini=None, args=[str(a)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1537,8 +1970,9 @@ def test_setuppy_fallback(self, tmp_path: Path) -> None: def test_nothing(self, tmp_path: Path, monkeypatch: MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) - rootpath, inipath, inicfg = determine_setup( + rootpath, inipath, inicfg, _ = determine_setup( inifile=None, + override_ini=None, args=[str(tmp_path)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1564,15 +1998,16 @@ def test_with_specific_inifile( p = tmp_path / name p.touch() p.write_text(contents, encoding="utf-8") - rootpath, inipath, ini_config = determine_setup( + rootpath, inipath, ini_config, _ = determine_setup( inifile=str(p), + override_ini=None, args=[str(tmp_path)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), ) assert rootpath == tmp_path assert inipath == p - assert ini_config == {"x": "10"} + assert ini_config["x"] == ConfigValue("10", origin="file", mode="ini") def test_explicit_config_file_sets_rootdir( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch @@ -1585,6 +2020,7 @@ def test_explicit_config_file_sets_rootdir( # No config file is explicitly given: rootdir is determined to be cwd. rootpath, found_inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=[str(tests_dir)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1597,6 +2033,7 @@ def test_explicit_config_file_sets_rootdir( inipath.touch() rootpath, found_inipath, *_ = determine_setup( inifile=str(inipath), + override_ini=None, args=[str(tests_dir)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1612,8 +2049,9 @@ def test_with_arg_outside_cwd_without_inifile( a.mkdir() b = tmp_path / "b" b.mkdir() - rootpath, inifile, _ = determine_setup( + rootpath, inifile, *_ = determine_setup( inifile=None, + override_ini=None, args=[str(a), str(b)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1628,8 +2066,9 @@ def test_with_arg_outside_cwd_with_inifile(self, tmp_path: Path) -> None: b.mkdir() inipath = a / "pytest.ini" inipath.touch() - rootpath, parsed_inipath, _ = determine_setup( + rootpath, parsed_inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=[str(a), str(b)], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1642,8 +2081,9 @@ def test_with_non_dir_arg( self, dirs: Sequence[str], tmp_path: Path, monkeypatch: MonkeyPatch ) -> None: monkeypatch.chdir(tmp_path) - rootpath, inipath, _ = determine_setup( + rootpath, inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=dirs, rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1658,8 +2098,9 @@ def test_with_existing_file_in_subdir( a.mkdir() (a / "exists").touch() monkeypatch.chdir(tmp_path) - rootpath, inipath, _ = determine_setup( + rootpath, inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=["a/exist"], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1677,8 +2118,9 @@ def test_with_config_also_in_parent_directory( (tmp_path / "myproject" / "tests").mkdir() monkeypatch.chdir(tmp_path / "myproject") - rootpath, inipath, _ = determine_setup( + rootpath, inipath, *_ = determine_setup( inifile=None, + override_ini=None, args=["tests/"], rootdir_cmd_arg=None, invocation_dir=Path.cwd(), @@ -1811,7 +2253,7 @@ def test_override_ini_usage_error_bad_style(self, pytester: Pytester) -> None: def test_override_ini_handled_asap( self, pytester: Pytester, with_ini: bool ) -> None: - """-o should be handled as soon as possible and always override what's in ini files (#2238)""" + """-o should be handled as soon as possible and always override what's in config files (#2238)""" if with_ini: pytester.makeini( """ @@ -1834,8 +2276,10 @@ def test_addopts_before_initini( cache_dir = ".custom_cache" monkeypatch.setenv("PYTEST_ADDOPTS", f"-o cache_dir={cache_dir}") config = _config_for_test - config._preparse([], addopts=True) - assert config._override_ini == [f"cache_dir={cache_dir}"] + config.parse([], addopts=True) + assert config._inicfg.get("cache_dir") == ConfigValue( + cache_dir, origin="override", mode="ini" + ) def test_addopts_from_env_not_concatenated( self, monkeypatch: MonkeyPatch, _config_for_test @@ -1844,14 +2288,15 @@ def test_addopts_from_env_not_concatenated( monkeypatch.setenv("PYTEST_ADDOPTS", "-o") config = _config_for_test with pytest.raises(UsageError) as excinfo: - config._preparse(["cache_dir=ignored"], addopts=True) + config.parse(["cache_dir=ignored"], addopts=True) assert ( - "error: argument -o/--override-ini: expected one argument (via PYTEST_ADDOPTS)" + "error: argument -o/--override-ini: expected one argument" in excinfo.value.args[0] ) + assert "via PYTEST_ADDOPTS" in excinfo.value.args[0] def test_addopts_from_ini_not_concatenated(self, pytester: Pytester) -> None: - """`addopts` from ini should not take values from normal args (#4265).""" + """`addopts` from configuration should not take values from normal args (#4265).""" pytester.makeini( """ [pytest] @@ -1861,8 +2306,8 @@ def test_addopts_from_ini_not_concatenated(self, pytester: Pytester) -> None: result = pytester.runpytest("cache_dir=ignored") result.stderr.fnmatch_lines( [ - f"{pytester._request.config._parser.optparser.prog}: error: " - f"argument -o/--override-ini: expected one argument (via addopts config)" + "*: error: argument -o/--override-ini: expected one argument", + " config source: via addopts config", ] ) assert result.ret == _pytest.config.ExitCode.USAGE_ERROR @@ -1872,8 +2317,10 @@ def test_override_ini_does_not_contain_paths( ) -> None: """Check that -o no longer swallows all options after it (#3103)""" config = _config_for_test - config._preparse(["-o", "cache_dir=/cache", "/some/test/path"]) - assert config._override_ini == ["cache_dir=/cache"] + config.parse(["-o", "cache_dir=/cache", "/some/test/path"]) + assert config._inicfg.get("cache_dir") == ConfigValue( + "/cache", origin="override", mode="ini" + ) def test_multiple_override_ini_options(self, pytester: Pytester) -> None: """Ensure a file path following a '-o' option does not generate an error (#3103)""" @@ -1909,7 +2356,16 @@ def test_override_ini_without_config_file(self, pytester: Pytester) -> None: } ) result = pytester.runpytest("--override-ini", "pythonpath=src") - assert result.parseoutcomes() == {"passed": 1} + result.assert_outcomes(passed=1) + + def test_override_ini_invalid_option(self, pytester: Pytester) -> None: + result = pytester.runpytest("--override-ini", "doesnotexist=true") + result.stdout.fnmatch_lines( + [ + "=*= warnings summary =*=", + "*PytestConfigWarning:*Unknown config option: doesnotexist", + ] + ) def test_help_via_addopts(pytester: Pytester) -> None: @@ -1962,8 +2418,7 @@ def pytest_addoption(parser): result.stderr.fnmatch_lines( [ "ERROR: usage: *", - f"{pytester._request.config._parser.optparser.prog}: error: " - f"argument --invalid-option-should-allow-for-help: expected one argument", + "*: error: argument --invalid-option-should-allow-for-help: expected one argument", ] ) # Does not display full/default help. @@ -1972,7 +2427,7 @@ def pytest_addoption(parser): result = pytester.runpytest("--version") result.stdout.fnmatch_lines([f"pytest {pytest.__version__}"]) - assert result.ret == ExitCode.USAGE_ERROR + assert result.ret == ExitCode.OK def test_help_formatter_uses_py_get_terminal_width(monkeypatch: MonkeyPatch) -> None: @@ -2005,6 +2460,10 @@ def test_config_does_not_load_blocked_plugin_from_args(pytester: Pytester) -> No result.stderr.fnmatch_lines(["*: error: unrecognized arguments: -s"]) assert result.ret == ExitCode.USAGE_ERROR + result = pytester.runpytest(str(p), "-p no:/path/to/conftest.py", "-s") + result.stderr.fnmatch_lines(["ERROR:*Blocking conftest files*"]) + assert result.ret == ExitCode.USAGE_ERROR + def test_invocation_args(pytester: Pytester) -> None: """Ensure that Config.invocation_* arguments are correctly defined""" @@ -2026,7 +2485,8 @@ class DummyPlugin: plugins = config.invocation_params.plugins assert len(plugins) == 2 assert plugins[0] is plugin - assert type(plugins[1]).__name__ == "Collect" # installed by pytester.inline_run() + # Installed by pytester.inline_run(). + assert type(plugins[1]).__name__ == "PytesterHelperPlugin" # args cannot be None with pytest.raises(TypeError): @@ -2250,8 +2710,6 @@ def test_parse_warning_filter( ":" * 5, # Invalid action. "FOO::", - # ImportError when importing the warning class. - "::test_parse_warning_filter_failure.NonExistentClass::", # Class is not a Warning subclass. "::list::", # Negative line number. @@ -2388,3 +2846,211 @@ def test_level_matches_specified_override( config.get_verbosity(TestVerbosity.SOME_OUTPUT_TYPE) == TestVerbosity.SOME_OUTPUT_VERBOSITY_LEVEL ) + + +class TestNativeTomlConfig: + """Test native TOML configuration parsing.""" + + def test_values(self, pytester: Pytester) -> None: + """Test that values are parsed as expected in TOML mode.""" + pytester.makepyprojecttoml( + """ + [tool.pytest] + test_bool = true + test_int = 5 + test_float = 30.5 + test_args = ["tests", "integration"] + test_paths = ["src", "lib"] + """ + ) + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("test_bool", "Test boolean config", type="bool", default=False) + parser.addini("test_int", "Test integer config", type="int", default=0) + parser.addini("test_float", "Test float config", type="float", default=0.0) + parser.addini("test_args", "Test args config", type="args") + parser.addini("test_paths", "Test paths config", type="paths") + """ + ) + config = pytester.parseconfig() + assert config.getini("test_bool") is True + assert config.getini("test_int") == 5 + assert config.getini("test_float") == 30.5 + assert config.getini("test_args") == ["tests", "integration"] + paths = config.getini("test_paths") + assert len(paths) == 2 + # Paths should be resolved relative to pyproject.toml location. + assert all(isinstance(p, Path) for p in paths) + + def test_override_with_list(self, pytester: Pytester) -> None: + """Test that -o overrides work with INI-style list syntax even when + config uses TOML mode.""" + pytester.makepyprojecttoml( + """ + [tool.pytest] + test_override_list = ["tests"] + """ + ) + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("test_override_list", "Test override list", type="args") + """ + ) + # -o uses INI mode, so uses space-separated syntax. + config = pytester.parseconfig("-o", "test_override_list=tests integration") + assert config.getini("test_override_list") == ["tests", "integration"] + + def test_conflict_between_native_and_ini_options(self, pytester: Pytester) -> None: + """Test that using both [tool.pytest] and [tool.pytest.ini_options] fails.""" + pytester.makepyprojecttoml( + """ + [tool.pytest] + test_conflict_1 = true + + [tool.pytest.ini_options] + test_conflict_2 = true + """, + ) + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("test_conflict_1", "Test conflict config 1", type="bool") + parser.addini("test_conflict_2", "Test conflict config 2", type="bool") + """ + ) + with pytest.raises(UsageError, match="Cannot use both"): + pytester.parseconfig() + + def test_type_errors(self, pytester: Pytester) -> None: + """Test all possible TypeError cases in getini.""" + pytester.maketoml( + """ + [pytest] + paths_not_list = "should_be_list" + paths_list_with_int = [1, 2] + + args_not_list = 123 + args_list_with_int = ["valid", 456] + + linelist_not_list = true + linelist_list_with_bool = ["valid", false] + + bool_not_bool = "true" + + int_not_int = "123" + int_is_bool = true + + float_not_float = "3.14" + float_is_bool = false + + string_not_string = 123 + """ + ) + pytester.makeconftest( + """ + def pytest_addoption(parser): + parser.addini("paths_not_list", "test", type="paths") + parser.addini("paths_list_with_int", "test", type="paths") + parser.addini("args_not_list", "test", type="args") + parser.addini("args_list_with_int", "test", type="args") + parser.addini("linelist_not_list", "test", type="linelist") + parser.addini("linelist_list_with_bool", "test", type="linelist") + parser.addini("bool_not_bool", "test", type="bool") + parser.addini("int_not_int", "test", type="int") + parser.addini("int_is_bool", "test", type="int") + parser.addini("float_not_float", "test", type="float") + parser.addini("float_is_bool", "test", type="float") + parser.addini("string_not_string", "test", type="string") + """ + ) + config = pytester.parseconfig() + + with pytest.raises( + TypeError, match=r"expects a list for type 'paths'.*got str" + ): + config.getini("paths_not_list") + + with pytest.raises( + TypeError, match=r"expects a list of strings.*item at index 0 is int" + ): + config.getini("paths_list_with_int") + + with pytest.raises(TypeError, match=r"expects a list for type 'args'.*got int"): + config.getini("args_not_list") + + with pytest.raises( + TypeError, match=r"expects a list of strings.*item at index 1 is int" + ): + config.getini("args_list_with_int") + + with pytest.raises( + TypeError, match=r"expects a list for type 'linelist'.*got bool" + ): + config.getini("linelist_not_list") + + with pytest.raises( + TypeError, match=r"expects a list of strings.*item at index 1 is bool" + ): + config.getini("linelist_list_with_bool") + + with pytest.raises(TypeError, match=r"expects a bool.*got str"): + config.getini("bool_not_bool") + + with pytest.raises(TypeError, match=r"expects an int.*got str"): + config.getini("int_not_int") + + with pytest.raises(TypeError, match=r"expects an int.*got bool"): + config.getini("int_is_bool") + + with pytest.raises(TypeError, match=r"expects a float.*got str"): + config.getini("float_not_float") + + with pytest.raises(TypeError, match=r"expects a float.*got bool"): + config.getini("float_is_bool") + + with pytest.raises(TypeError, match=r"expects a string.*got int"): + config.getini("string_not_string") + + +class TestInicfgDeprecation: + """Tests for the upcoming deprecation of config.inicfg.""" + + def test_inicfg_deprecated(self, pytester: Pytester) -> None: + """Test that accessing config.inicfg issues a deprecation warning (not yet).""" + pytester.makeini( + """ + [pytest] + minversion = 3.0 + """ + ) + config = pytester.parseconfig() + + inicfg = config.inicfg + + assert config.getini("minversion") == "3.0" + assert inicfg["minversion"] == "3.0" + assert inicfg.get("minversion") == "3.0" + del inicfg["minversion"] + inicfg["minversion"] = "4.0" + assert list(inicfg.keys()) == ["minversion"] + assert list(inicfg.items()) == [("minversion", "4.0")] + assert len(inicfg) == 1 + + def test_issue_13946_setting_bool_no_longer_crashes( + self, pytester: Pytester + ) -> None: + """Regression test for #13946 - setting inicfg doesn't cause a crash.""" + pytester.makepyfile( + """ + def pytest_configure(config): + config.inicfg["xfail_strict"] = True + + def test(): + pass + """ + ) + + result = pytester.runpytest() + assert result.ret == 0 diff --git a/testing/test_conftest.py b/testing/test_conftest.py index d51846f2f5f..4de61bceb90 100644 --- a/testing/test_conftest.py +++ b/testing/test_conftest.py @@ -1,13 +1,12 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator +from collections.abc import Sequence import os from pathlib import Path import textwrap from typing import cast -from typing import Generator -from typing import List -from typing import Sequence from _pytest.config import ExitCode from _pytest.config import PytestPluginManager @@ -44,9 +43,7 @@ def conftest_setinitial( @pytest.mark.usefixtures("_sys_snapshot") class TestConftestValueAccessGlobal: @pytest.fixture(scope="module", params=["global", "inpackage"]) - def basedir( - self, request, tmp_path_factory: TempPathFactory - ) -> Generator[Path, None, None]: + def basedir(self, request, tmp_path_factory: TempPathFactory) -> Generator[Path]: tmp_path = tmp_path_factory.mktemp("basedir", numbered=True) tmp_path.joinpath("adir/b").mkdir(parents=True) tmp_path.joinpath("adir/conftest.py").write_text( @@ -461,7 +458,7 @@ def impct(p, importmode, root, consider_namespace_packages): rootpath=pytester.path, consider_namespace_packages=False, ) - mods = cast(List[Path], conftest._getconftestmodules(sub)) + mods = cast(list[Path], conftest._getconftestmodules(sub)) expected = [ct1, ct2] assert mods == expected @@ -657,7 +654,7 @@ def test_parsefactories_relative_node_ids( def test_search_conftest_up_to_inifile( pytester: Pytester, confcutdir: str, passed: int, error: int ) -> None: - """Test that conftest files are detected only up to an ini file, unless + """Test that conftest files are detected only up to a configuration file, unless an explicit --confcutdir option is given. """ root = pytester.path @@ -702,9 +699,9 @@ def out_of_reach(): pass result = pytester.runpytest(*args) match = "" if passed: - match += "*%d passed*" % passed + match += f"*{passed} passed*" if error: - match += "*%d error*" % error + match += f"*{error} error*" result.stdout.fnmatch_lines(match) diff --git a/testing/test_debugging.py b/testing/test_debugging.py index 37032f92354..08ebf600253 100644 --- a/testing/test_debugging.py +++ b/testing/test_debugging.py @@ -52,6 +52,16 @@ def reset(self): def interaction(self, *args): called.append("interaction") + # Methods which we copy docstrings to. + def do_debug(self, *args): # pragma: no cover + pass + + def do_continue(self, *args): # pragma: no cover + pass + + def do_quit(self, *args): # pragma: no cover + pass + _pytest._CustomPdb = _CustomPdb # type: ignore return called @@ -75,6 +85,16 @@ def set_trace(self, frame): print("**CustomDebugger**") called.append("set_trace") + # Methods which we copy docstrings to. + def do_debug(self, *args): # pragma: no cover + pass + + def do_continue(self, *args): # pragma: no cover + pass + + def do_quit(self, *args): # pragma: no cover + pass + _pytest._CustomDebugger = _CustomDebugger # type: ignore yield called del _pytest._CustomDebugger # type: ignore @@ -103,7 +123,10 @@ def test_func(): ) assert rep.failed assert len(pdblist) == 1 - tb = _pytest._code.Traceback(pdblist[0][0]) + if sys.version_info < (3, 13): + tb = _pytest._code.Traceback(pdblist[0][0]) + else: + tb = _pytest._code.Traceback(pdblist[0][0].__traceback__) assert tb[-1].name == "test_func" def test_pdb_on_xfail(self, pytester: Pytester, pdblist) -> None: @@ -768,9 +791,13 @@ def test_pdb_used_outside_test(self, pytester: Pytester) -> None: x = 5 """ ) + if sys.version_info[:2] >= (3, 13): + break_line = "pytest.set_trace()" + else: + break_line = "x = 5" child = pytester.spawn(f"{sys.executable} {p1}") - child.expect("x = 5") - child.expect("Pdb") + child.expect_exact(break_line) + child.expect_exact("Pdb") child.sendeof() self.flush(child) @@ -785,9 +812,13 @@ def test_foo(a): pass """ ) + if sys.version_info[:2] >= (3, 13): + break_line = "pytest.set_trace()" + else: + break_line = "x = 5" child = pytester.spawn_pytest(str(p1)) - child.expect("x = 5") - child.expect("Pdb") + child.expect_exact(break_line) + child.expect_exact("Pdb") child.sendeof() self.flush(child) @@ -921,6 +952,67 @@ def test_foo(): child.expect("custom set_trace>") self.flush(child) + @pytest.mark.skipif( + sys.version_info < (3, 13), + reason="Navigating exception chains was introduced in 3.13", + ) + def test_pdb_exception_chain_navigation(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def inner_raise(): + is_inner = True + raise RuntimeError("Woops") + + def outer_raise(): + is_inner = False + try: + inner_raise() + except RuntimeError: + raise RuntimeError("Woopsie") + + def test_1(): + outer_raise() + assert True + """ + ) + child = pytester.spawn_pytest(f"--pdb {p1}") + child.expect("Pdb") + child.sendline("is_inner") + child.expect_exact("False") + child.sendline("exceptions 0") + child.sendline("is_inner") + child.expect_exact("True") + child.sendeof() + self.flush(child) + + def test_pdb_wrapped_commands_docstrings(self, pytester: Pytester) -> None: + p1 = pytester.makepyfile( + """ + def test_1(): + assert False + """ + ) + + child = pytester.spawn_pytest(f"--pdb {p1}") + child.expect("Pdb") + + # Verify no undocumented commands + child.sendline("help") + child.expect("Documented commands") + assert "Undocumented commands" not in child.before.decode() + + child.sendline("help continue") + child.expect("Continue execution") + child.expect("Pdb") + + child.sendline("help debug") + child.expect("Enter a recursive debugger") + child.expect("Pdb") + + child.sendline("c") + child.sendeof() + self.flush(child) + class TestDebuggingBreakpoints: @pytest.mark.parametrize("arg", ["--pdb", ""]) @@ -1244,6 +1336,16 @@ def set_trace(self, *args): def runcall(self, *args, **kwds): print("runcall_called", args, kwds) + + # Methods which we copy the docstring over. + def do_debug(self, *args): + pass + + def do_continue(self, *args): + pass + + def do_quit(self, *args): + pass """, ) result = pytester.runpytest( @@ -1271,6 +1373,10 @@ def runcall(self, *args, **kwds): result.stdout.fnmatch_lines(["*runcall_called*", "* 1 passed in *"]) +@pytest.mark.xfail( + sys.version_info >= (3, 14), + reason="C-D now quits the test session, rather than failing the test. See https://github.com/python/cpython/issues/124703", +) def test_raises_bdbquit_with_eoferror(pytester: Pytester) -> None: """It is not guaranteed that DontReadFromInput's read is called.""" p1 = pytester.makepyfile( @@ -1285,6 +1391,7 @@ def test(monkeypatch): """ ) result = pytester.runpytest(str(p1)) + result.assert_outcomes(failed=1) result.stdout.fnmatch_lines(["E *BdbQuit", "*= 1 failed in*"]) assert result.ret == 1 @@ -1310,6 +1417,16 @@ def __init__(self, *args, **kwargs): def set_trace(self, *args): print("set_trace_called", args) + + # Methods which we copy the docstring over. + def do_debug(self, *args): + pass + + def do_continue(self, *args): + pass + + def do_quit(self, *args): + pass """, ) result = pytester.runpytest(str(p1), "--pdbcls=mypdb:MyPdb", syspathinsert=True) diff --git a/testing/test_doctest.py b/testing/test_doctest.py index 4aa4876c711..8b71dabbc77 100644 --- a/testing/test_doctest.py +++ b/testing/test_doctest.py @@ -1,11 +1,11 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Callable import inspect from pathlib import Path import sys import textwrap -from typing import Callable from _pytest.doctest import _get_checker from _pytest.doctest import _is_main_py @@ -33,24 +33,24 @@ def test_collect_testtextfile(self, pytester: Pytester): for x in (pytester.path, checkfile): # print "checking that %s returns custom items" % (x,) - items, reprec = pytester.inline_genitems(x) + items, _reprec = pytester.inline_genitems(x) assert len(items) == 1 assert isinstance(items[0], DoctestItem) assert isinstance(items[0].parent, DoctestTextfile) # Empty file has no items. - items, reprec = pytester.inline_genitems(w) + items, _reprec = pytester.inline_genitems(w) assert len(items) == 0 def test_collect_module_empty(self, pytester: Pytester): path = pytester.makepyfile(whatever="#") for p in (path, pytester.path): - items, reprec = pytester.inline_genitems(p, "--doctest-modules") + items, _reprec = pytester.inline_genitems(p, "--doctest-modules") assert len(items) == 0 def test_collect_module_single_modulelevel_doctest(self, pytester: Pytester): path = pytester.makepyfile(whatever='""">>> pass"""') for p in (path, pytester.path): - items, reprec = pytester.inline_genitems(p, "--doctest-modules") + items, _reprec = pytester.inline_genitems(p, "--doctest-modules") assert len(items) == 1 assert isinstance(items[0], DoctestItem) assert isinstance(items[0].parent, DoctestModule) @@ -64,7 +64,7 @@ def my_func(): """ ) for p in (path, pytester.path): - items, reprec = pytester.inline_genitems(p, "--doctest-modules") + items, _reprec = pytester.inline_genitems(p, "--doctest-modules") assert len(items) == 2 assert isinstance(items[0], DoctestItem) assert isinstance(items[1], DoctestItem) @@ -97,7 +97,7 @@ def another(): }, ) for p in (path, pytester.path): - items, reprec = pytester.inline_genitems(p, "--doctest-modules") + items, _reprec = pytester.inline_genitems(p, "--doctest-modules") assert len(items) == 2 assert isinstance(items[0], DoctestItem) assert isinstance(items[1], DoctestItem) @@ -223,9 +223,12 @@ def test_doctest_unexpected_exception(self, pytester: Pytester): "002 >>> 0 / i", "UNEXPECTED EXCEPTION: ZeroDivisionError*", "Traceback (most recent call last):", - ' File "*/doctest.py", line *, in __run', - " *", - *((" *^^^^*", " *", " *") if sys.version_info >= (3, 13) else ()), + *( + (' File "*/doctest.py", line *, in __run', " *") + if sys.version_info <= (3, 14) + else () + ), + *((" *^^^^*", " *", " *") if sys.version_info[:2] == (3, 13) else ()), ' File "", line 1, in ', "ZeroDivisionError: division by zero", "*/test_doctest_unexpected_exception.txt:2: UnexpectedException", @@ -837,7 +840,7 @@ def foo(x): return 'c' """ ) - items, reprec = pytester.inline_genitems(p, "--doctest-modules") + items, _reprec = pytester.inline_genitems(p, "--doctest-modules") reportinfo = items[0].reportinfo() assert reportinfo[1] == 1 @@ -904,7 +907,7 @@ class TestLiterals: def test_allow_unicode(self, pytester, config_mode): """Test that doctests which output unicode work in all python versions tested by pytest when the ALLOW_UNICODE option is used (either in - the ini file or by an inline comment). + the configuration file or by an inline comment). """ if config_mode == "ini": pytester.makeini( @@ -939,7 +942,7 @@ def foo(): def test_allow_bytes(self, pytester, config_mode): """Test that doctests which output bytes work in all python versions tested by pytest when the ALLOW_BYTES option is used (either in - the ini file or by an inline comment)(#1287). + the configuration file or by an inline comment)(#1287). """ if config_mode == "ini": pytester.makeini( @@ -1328,7 +1331,7 @@ def test_bar(): params = ("--doctest-modules",) if enable_doctest else () passes = 3 if enable_doctest else 2 result = pytester.runpytest(*params) - result.stdout.fnmatch_lines(["*=== %d passed in *" % passes]) + result.stdout.fnmatch_lines([f"*=== {passes} passed in *"]) @pytest.mark.parametrize("scope", SCOPES) @pytest.mark.parametrize("autouse", [True, False]) @@ -1593,7 +1596,14 @@ def __getattr__(self, _): @pytest.mark.parametrize( # pragma: no branch (lambdas are not called) - "stop", [None, _is_mocked, lambda f: None, lambda f: False, lambda f: True] + "stop", + [ + None, + pytest.param(_is_mocked, id="is_mocked"), + pytest.param(lambda f: None, id="lambda_none"), + pytest.param(lambda f: False, id="lambda_false"), + pytest.param(lambda f: True, id="lambda_true"), + ], ) def test_warning_on_unwrap_of_broken_object( stop: Callable[[object], object] | None, diff --git a/testing/test_faulthandler.py b/testing/test_faulthandler.py index c416e81d2d9..67ca221f3f2 100644 --- a/testing/test_faulthandler.py +++ b/testing/test_faulthandler.py @@ -2,6 +2,7 @@ from __future__ import annotations import io +import os import sys from _pytest.pytester import Pytester @@ -71,11 +72,18 @@ def test_disabled(): assert result.ret == 0 +@pytest.mark.keep_ci_var @pytest.mark.parametrize( "enabled", [ pytest.param( - True, marks=pytest.mark.skip(reason="sometimes crashes on CI (#7022)") + True, + marks=pytest.mark.skipif( + bool(os.environ.get("CI")) + and sys.platform == "linux" + and sys.version_info >= (3, 14), + reason="sometimes crashes on CI because of truncated outputs (#7022)", + ), ), False, ], @@ -110,6 +118,43 @@ def test_timeout(): assert result.ret == 0 +@pytest.mark.keep_ci_var +@pytest.mark.skipif( + "CI" in os.environ and sys.platform == "linux" and sys.version_info >= (3, 14), + reason="sometimes crashes on CI because of truncated outputs (#7022)", +) +@pytest.mark.parametrize("exit_on_timeout", [True, False]) +def test_timeout_and_exit(pytester: Pytester, exit_on_timeout: bool) -> None: + """Test option to force exit pytest process after a certain timeout.""" + pytester.makepyfile( + """ + import os, time + def test_long_sleep_and_raise(): + time.sleep(1 if "CI" in os.environ else 0.1) + raise AssertionError( + "This test should have been interrupted before reaching this point." + ) + """ + ) + pytester.makeini( + f""" + [pytest] + faulthandler_timeout = 0.01 + faulthandler_exit_on_timeout = {"true" if exit_on_timeout else "false"} + """ + ) + result = pytester.runpytest_subprocess() + tb_output = "most recent call first" + result.stderr.fnmatch_lines([f"*{tb_output}*"]) + if exit_on_timeout: + result.stdout.no_fnmatch_line("*1 failed*") + result.stdout.no_fnmatch_line("*AssertionError*") + else: + result.stdout.fnmatch_lines(["*1 failed*"]) + result.stdout.fnmatch_lines(["*AssertionError*"]) + assert result.ret == 1 + + @pytest.mark.parametrize("hook_name", ["pytest_enter_pdb", "pytest_exception_interact"]) def test_cancel_timeout_on_hook(monkeypatch, hook_name) -> None: """Make sure that we are cancelling any scheduled traceback dumping due diff --git a/testing/test_findpaths.py b/testing/test_findpaths.py index 9532f1eef75..aea7b1f9a4d 100644 --- a/testing/test_findpaths.py +++ b/testing/test_findpaths.py @@ -6,6 +6,7 @@ from textwrap import dedent from _pytest.config import UsageError +from _pytest.config.findpaths import ConfigValue from _pytest.config.findpaths import get_common_ancestor from _pytest.config.findpaths import get_dirs_from_args from _pytest.config.findpaths import is_fs_root @@ -14,9 +15,10 @@ class TestLoadConfigDictFromFile: - def test_empty_pytest_ini(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("filename", ["pytest.ini", ".pytest.ini"]) + def test_empty_pytest_ini(self, tmp_path: Path, filename: str) -> None: """pytest.ini files are always considered for configuration, even if empty""" - fn = tmp_path / "pytest.ini" + fn = tmp_path / filename fn.write_text("", encoding="utf-8") assert load_config_dict_from_file(fn) == {} @@ -24,13 +26,17 @@ def test_pytest_ini(self, tmp_path: Path) -> None: """[pytest] section in pytest.ini files is read correctly""" fn = tmp_path / "pytest.ini" fn.write_text("[pytest]\nx=1", encoding="utf-8") - assert load_config_dict_from_file(fn) == {"x": "1"} + assert load_config_dict_from_file(fn) == { + "x": ConfigValue("1", origin="file", mode="ini") + } def test_custom_ini(self, tmp_path: Path) -> None: """[pytest] section in any .ini file is read correctly""" fn = tmp_path / "custom.ini" fn.write_text("[pytest]\nx=1", encoding="utf-8") - assert load_config_dict_from_file(fn) == {"x": "1"} + assert load_config_dict_from_file(fn) == { + "x": ConfigValue("1", origin="file", mode="ini") + } def test_custom_ini_without_section(self, tmp_path: Path) -> None: """Custom .ini files without [pytest] section are not considered for configuration""" @@ -48,7 +54,9 @@ def test_valid_cfg_file(self, tmp_path: Path) -> None: """Custom .cfg files with [tool:pytest] section are read correctly""" fn = tmp_path / "custom.cfg" fn.write_text("[tool:pytest]\nx=1", encoding="utf-8") - assert load_config_dict_from_file(fn) == {"x": "1"} + assert load_config_dict_from_file(fn) == { + "x": ConfigValue("1", origin="file", mode="ini") + } def test_unsupported_pytest_section_in_cfg_file(self, tmp_path: Path) -> None: """.cfg files with [pytest] section are no longer supported and should fail to alert users""" @@ -65,7 +73,7 @@ def test_invalid_toml_file(self, tmp_path: Path) -> None: load_config_dict_from_file(fn) def test_custom_toml_file(self, tmp_path: Path) -> None: - """.toml files without [tool.pytest.ini_options] are not considered for configuration.""" + """.toml files without [tool.pytest] are not considered for configuration.""" fn = tmp_path / "myconfig.toml" fn.write_text( dedent( @@ -96,13 +104,70 @@ def test_valid_toml_file(self, tmp_path: Path) -> None: encoding="utf-8", ) assert load_config_dict_from_file(fn) == { - "x": "1", - "y": "20.0", - "values": ["tests", "integration"], - "name": "foo", - "heterogeneous_array": [1, "str"], + "x": ConfigValue("1", origin="file", mode="ini"), + "y": ConfigValue("20.0", origin="file", mode="ini"), + "values": ConfigValue(["tests", "integration"], origin="file", mode="ini"), + "name": ConfigValue("foo", origin="file", mode="ini"), + "heterogeneous_array": ConfigValue([1, "str"], origin="file", mode="ini"), + } + + def test_native_toml_config(self, tmp_path: Path) -> None: + """[tool.pytest] sections with native types are parsed correctly without coercion.""" + fn = tmp_path / "pyproject.toml" + fn.write_text( + dedent( + """ + [tool.pytest] + minversion = "7.0" + xfail_strict = true + testpaths = ["tests", "integration"] + python_files = ["test_*.py", "*_test.py"] + verbosity_assertions = 2 + maxfail = 5 + timeout = 300.5 + """ + ), + encoding="utf-8", + ) + result = load_config_dict_from_file(fn) + assert result == { + "minversion": ConfigValue("7.0", origin="file", mode="toml"), + "xfail_strict": ConfigValue(True, origin="file", mode="toml"), + "testpaths": ConfigValue( + ["tests", "integration"], origin="file", mode="toml" + ), + "python_files": ConfigValue( + ["test_*.py", "*_test.py"], origin="file", mode="toml" + ), + "verbosity_assertions": ConfigValue(2, origin="file", mode="toml"), + "maxfail": ConfigValue(5, origin="file", mode="toml"), + "timeout": ConfigValue(300.5, origin="file", mode="toml"), } + def test_native_and_ini_conflict(self, tmp_path: Path) -> None: + """Using both [tool.pytest] and [tool.pytest.ini_options] should raise an error.""" + fn = tmp_path / "pyproject.toml" + fn.write_text( + dedent( + """ + [tool.pytest] + xfail_strict = true + + [tool.pytest.ini_options] + minversion = "7.0" + """ + ), + encoding="utf-8", + ) + with pytest.raises(UsageError, match="Cannot use both"): + load_config_dict_from_file(fn) + + def test_invalid_suffix(self, tmp_path: Path) -> None: + """A file with an unknown suffix is ignored.""" + fn = tmp_path / "pytest.config" + fn.write_text("", encoding="utf-8") + assert load_config_dict_from_file(fn) is None + class TestCommonAncestor: def test_has_ancestor(self, tmp_path: Path) -> None: diff --git a/testing/test_helpconfig.py b/testing/test_helpconfig.py index 7fcf5804ace..b01a6fa1559 100644 --- a/testing/test_helpconfig.py +++ b/testing/test_helpconfig.py @@ -8,21 +8,23 @@ def test_version_verbose(pytester: Pytester, pytestconfig, monkeypatch) -> None: monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + monkeypatch.delenv("PYTEST_PLUGINS", raising=False) result = pytester.runpytest("--version", "--version") - assert result.ret == 0 + assert result.ret == ExitCode.OK result.stdout.fnmatch_lines([f"*pytest*{pytest.__version__}*imported from*"]) if pytestconfig.pluginmanager.list_plugin_distinfo(): result.stdout.fnmatch_lines(["*registered third-party plugins:", "*at*"]) -def test_version_less_verbose(pytester: Pytester, pytestconfig, monkeypatch) -> None: - monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") - result = pytester.runpytest("--version") - assert result.ret == 0 - result.stdout.fnmatch_lines([f"pytest {pytest.__version__}"]) +def test_version_less_verbose(pytester: Pytester) -> None: + """Single ``--version`` parameter should display only the pytest version, without loading plugins (#13574).""" + pytester.makeconftest("print('This should not be printed')") + result = pytester.runpytest_subprocess("--version") + assert result.ret == ExitCode.OK + assert result.stdout.str().strip() == f"pytest {pytest.__version__}" -def test_versions(): +def test_versions() -> None: """Regression check for the public version attributes in pytest.""" assert isinstance(pytest.__version__, str) assert isinstance(pytest.version_tuple, tuple) @@ -30,7 +32,7 @@ def test_versions(): def test_help(pytester: Pytester) -> None: result = pytester.runpytest("--help") - assert result.ret == 0 + assert result.ret == ExitCode.OK result.stdout.fnmatch_lines( """ -m MARKEXPR Only run tests matching given mark expression. For @@ -71,7 +73,7 @@ def pytest_addoption(parser): """ ) result = pytester.runpytest("--help") - assert result.ret == 0 + assert result.ret == ExitCode.OK lines = [ " required_plugins (args):", " Plugins that must be present for pytest to run*", @@ -81,6 +83,14 @@ def pytest_addoption(parser): result.stdout.fnmatch_lines(lines, consecutive=True) +def test_parse_known_args_doesnt_quit_on_help(pytester: Pytester) -> None: + """`parse_known_args` shouldn't exit on `--help`, unlike `parse`.""" + config = pytester.parseconfig() + # Doesn't raise or exit! + config._parser.parse_known_args(["--help"]) + config._parser.parse_known_and_unknown_args(["--help"]) + + def test_hookvalidation_unknown(pytester: Pytester) -> None: pytester.makeconftest( """ @@ -89,7 +99,7 @@ def pytest_hello(xyz): """ ) result = pytester.runpytest() - assert result.ret != 0 + assert result.ret != ExitCode.OK result.stdout.fnmatch_lines(["*unknown hook*pytest_hello*"]) diff --git a/testing/test_junitxml.py b/testing/test_junitxml.py index fd1fecb54f1..5a603c05bc8 100644 --- a/testing/test_junitxml.py +++ b/testing/test_junitxml.py @@ -1,4 +1,3 @@ -# mypy: allow-untyped-defs from __future__ import annotations from datetime import datetime @@ -6,6 +5,7 @@ import os from pathlib import Path import platform +from typing import Any from typing import cast from typing import TYPE_CHECKING from xml.dom import minidom @@ -21,6 +21,7 @@ from _pytest.reports import BaseReport from _pytest.reports import TestReport from _pytest.stash import Stash +import _pytest.timing import pytest @@ -39,7 +40,7 @@ def __init__(self, pytester: Pytester, schema: xmlschema.XMLSchema) -> None: def __call__( self, *args: str | os.PathLike[str], family: str | None = "xunit1" - ) -> tuple[RunResult, DomNode]: + ) -> tuple[RunResult, DomDocument]: if family: args = ("-o", "junit_family=" + family, *args) xml_path = self.pytester.path.joinpath("junit.xml") @@ -48,7 +49,7 @@ def __call__( with xml_path.open(encoding="utf-8") as f: self.schema.validate(f) xmldoc = minidom.parse(str(xml_path)) - return result, DomNode(xmldoc) + return result, DomDocument(xmldoc) @pytest.fixture @@ -62,78 +63,133 @@ def run_and_parse(pytester: Pytester, schema: xmlschema.XMLSchema) -> RunAndPars return RunAndParse(pytester, schema) -def assert_attr(node, **kwargs): +def assert_attr(node: minidom.Element, **kwargs: object) -> None: __tracebackhide__ = True - def nodeval(node, name): + def nodeval(node: minidom.Element, name: str) -> str | None: anode = node.getAttributeNode(name) - if anode is not None: - return anode.value + return anode.value if anode is not None else None expected = {name: str(value) for name, value in kwargs.items()} on_node = {name: nodeval(node, name) for name in expected} assert on_node == expected -class DomNode: - def __init__(self, dom): - self.__node = dom +class DomDocument: + _node: minidom.Document | minidom.Element - def __repr__(self): - return self.__node.toxml() + def __init__(self, dom: minidom.Document) -> None: + self._node = dom - def find_first_by_tag(self, tag): + def find_first_by_tag(self, tag: str) -> DomNode | None: return self.find_nth_by_tag(tag, 0) - def _by_tag(self, tag): - return self.__node.getElementsByTagName(tag) + def get_first_by_tag(self, tag: str) -> DomNode: + maybe = self.find_first_by_tag(tag) + if maybe is None: + raise LookupError(tag) + else: + return maybe + + def find_nth_by_tag(self, tag: str, n: int) -> DomNode | None: + items = self._node.getElementsByTagName(tag) + try: + nth = items[n] + except IndexError: + return None + else: + return DomNode(nth) + + def find_by_tag(self, tag: str) -> list[DomNode]: + return [DomNode(x) for x in self._node.getElementsByTagName(tag)] @property - def children(self): - return [type(self)(x) for x in self.__node.childNodes] + def children(self) -> list[DomNode]: + return [ + DomNode(x) for x in self._node.childNodes if isinstance(x, minidom.Element) + ] @property - def get_unique_child(self): + def get_unique_child(self) -> DomNode: children = self.children assert len(children) == 1 return children[0] - def find_nth_by_tag(self, tag, n): - items = self._by_tag(tag) - try: - nth = items[n] - except IndexError: - pass - else: - return type(self)(nth) + def toxml(self) -> str: + return self._node.toxml() - def find_by_tag(self, tag): - t = type(self) - return [t(x) for x in self.__node.getElementsByTagName(tag)] - def __getitem__(self, key): - node = self.__node.getAttributeNode(key) +class DomNode(DomDocument): + _node: minidom.Element + + def __init__(self, dom: minidom.Element) -> None: + self._node = dom + + def __repr__(self) -> str: + return self.toxml() + + def __getitem__(self, key: str) -> str: + node = self._node.getAttributeNode(key) if node is not None: return node.value + else: + raise KeyError(key) - def assert_attr(self, **kwargs): + def assert_attr(self, **kwargs: object) -> None: __tracebackhide__ = True - return assert_attr(self.__node, **kwargs) - - def toxml(self): - return self.__node.toxml() + return assert_attr(self._node, **kwargs) @property - def text(self): - return self.__node.childNodes[0].wholeText + def text(self) -> str: + text = self._node.childNodes[0] + assert isinstance(text, minidom.Text) + return text.wholeText @property - def tag(self): - return self.__node.tagName + def tag(self) -> str: + return self._node.tagName - @property - def next_sibling(self): - return type(self)(self.__node.nextSibling) + +class TestJunitHelpers: + """minimal test to increase coverage for methods that are used in debugging""" + + @pytest.fixture + def document(self) -> DomDocument: + doc = minidom.parseString(""" + + + + +""") + return DomDocument(doc) + + def test_uc_root(self, document: DomDocument) -> None: + assert document.get_unique_child.tag == "root" + + def test_node_assert_attr(self, document: DomDocument) -> None: + item = document.get_first_by_tag("item") + + item.assert_attr(name="a") + + with pytest.raises(AssertionError): + item.assert_attr(missing="foo") + + def test_node_getitem(self, document: DomDocument) -> None: + item = document.get_first_by_tag("item") + assert item["name"] == "a" + + with pytest.raises(KeyError, match="missing"): + item["missing"] + + def test_node_get_first_lookup(self, document: DomDocument) -> None: + with pytest.raises(LookupError, match="missing"): + document.get_first_by_tag("missing") + + def test_node_repr(self, document: DomDocument) -> None: + item = document.get_first_by_tag("item") + + assert repr(item) == item.toxml() + assert item.toxml() == '' parametrize_families = pytest.mark.parametrize("xunit_family", ["xunit1", "xunit2"]) @@ -163,7 +219,7 @@ def test_xpass(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=0, failures=1, skipped=2, tests=5) @parametrize_families @@ -192,7 +248,7 @@ def test_xpass(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(name="pytest", errors=1, failures=2, skipped=1, tests=5) @parametrize_families @@ -205,8 +261,8 @@ def test_pass(): pass """ ) - result, dom = run_and_parse(family=xunit_family) - node = dom.find_first_by_tag("testsuite") + _result, dom = run_and_parse(family=xunit_family) + node = dom.get_first_by_tag("testsuite") node.assert_attr(hostname=platform.node()) @parametrize_families @@ -220,13 +276,16 @@ def test_pass(): """ ) start_time = datetime.now(timezone.utc) - result, dom = run_and_parse(family=xunit_family) - node = dom.find_first_by_tag("testsuite") - timestamp = datetime.strptime(node["timestamp"], "%Y-%m-%dT%H:%M:%S.%f%z") + _result, dom = run_and_parse(family=xunit_family) + node = dom.get_first_by_tag("testsuite") + timestamp = datetime.fromisoformat(node["timestamp"]) assert start_time <= timestamp < datetime.now(timezone.utc) def test_timing_function( - self, pytester: Pytester, run_and_parse: RunAndParse, mock_timing + self, + pytester: Pytester, + run_and_parse: RunAndParse, + mock_timing: _pytest.timing.MockTiming, ) -> None: pytester.makepyfile( """ @@ -239,10 +298,11 @@ def test_sleep(): timing.sleep(4) """ ) - result, dom = run_and_parse() - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse() + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") val = tnode["time"] + assert val is not None assert float(val) == 7.0 @pytest.mark.parametrize("duration_report", ["call", "total"]) @@ -256,7 +316,7 @@ def test_junit_duration_report( # mock LogXML.node_reporter so it always sets a known duration to each test report object original_node_reporter = LogXML.node_reporter - def node_reporter_wrapper(s, report): + def node_reporter_wrapper(s: Any, report: TestReport) -> Any: report.duration = 1.0 reporter = original_node_reporter(s, report) return reporter @@ -269,9 +329,9 @@ def test_foo(): pass """ ) - result, dom = run_and_parse("-o", f"junit_duration_report={duration_report}") - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_duration_report={duration_report}") + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") val = float(tnode["time"]) if duration_report == "total": assert val == 3.0 @@ -296,11 +356,11 @@ def test_function(arg): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_setup_error", name="test_function") - fnode = tnode.find_first_by_tag("error") + fnode = tnode.get_first_by_tag("error") fnode.assert_attr(message='failed on setup with "ValueError: Error reason"') assert "ValueError" in fnode.toxml() @@ -322,10 +382,10 @@ def test_function(arg): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_teardown_error", name="test_function") - fnode = tnode.find_first_by_tag("error") + fnode = tnode.get_first_by_tag("error") fnode.assert_attr(message='failed on teardown with "ValueError: Error reason"') assert "ValueError" in fnode.toxml() @@ -347,15 +407,15 @@ def test_function(arg): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, failures=1, tests=1) first, second = dom.find_by_tag("testcase") assert first assert second assert first != second - fnode = first.find_first_by_tag("failure") + fnode = first.get_first_by_tag("failure") fnode.assert_attr(message="Exception: Call Exception") - snode = second.find_first_by_tag("error") + snode = second.get_first_by_tag("error") snode.assert_attr( message='failed on teardown with "Exception: Teardown Exception"' ) @@ -373,11 +433,11 @@ def test_skip(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_skip_contains_name_reason", name="test_skip") - snode = tnode.find_first_by_tag("skipped") + snode = tnode.get_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello23") @parametrize_families @@ -394,13 +454,13 @@ def test_skip(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr( classname="test_mark_skip_contains_name_reason", name="test_skip" ) - snode = tnode.find_first_by_tag("skipped") + snode = tnode.get_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello24") @parametrize_families @@ -418,13 +478,13 @@ def test_skip(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr( classname="test_mark_skipif_contains_name_reason", name="test_skip" ) - snode = tnode.find_first_by_tag("skipped") + snode = tnode.get_first_by_tag("skipped") snode.assert_attr(type="pytest.skip", message="hello25") @parametrize_families @@ -441,7 +501,7 @@ def test_skip(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node_xml = dom.find_first_by_tag("testsuite").toxml() + node_xml = dom.get_first_by_tag("testsuite").toxml() assert "bar!" not in node_xml @parametrize_families @@ -457,9 +517,9 @@ def test_method(self): ) result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr( classname="test_classname_instance.TestClass", name="test_method" ) @@ -472,9 +532,9 @@ def test_classname_nested_dir( p.write_text("def test_func(): 0/0", encoding="utf-8") result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="sub.test_hello", name="test_func") @parametrize_families @@ -485,11 +545,11 @@ def test_internal_error( pytester.makepyfile("def test_function(): pass") result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="pytest", name="internal") - fnode = tnode.find_first_by_tag("error") + fnode = tnode.get_first_by_tag("error") fnode.assert_attr(message="internal error") assert "Division" in fnode.toxml() @@ -500,9 +560,9 @@ def test_internal_error( def test_failure_function( self, pytester: Pytester, - junit_logging, + junit_logging: str, run_and_parse: RunAndParse, - xunit_family, + xunit_family: str, ) -> None: pytester.makepyfile( """ @@ -522,47 +582,47 @@ def test_fail(): "-o", f"junit_logging={junit_logging}", family=xunit_family ) assert result.ret, "Expected ret > 0" - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_failure_function", name="test_fail") - fnode = tnode.find_first_by_tag("failure") + fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="ValueError: 42") assert "ValueError" in fnode.toxml(), "ValueError not included" if junit_logging in ["log", "all"]: - logdata = tnode.find_first_by_tag("system-out") + logdata = tnode.get_first_by_tag("system-out") log_xml = logdata.toxml() assert logdata.tag == "system-out", "Expected tag: system-out" assert "info msg" not in log_xml, "Unexpected INFO message" assert "warning msg" in log_xml, "Missing WARN message" if junit_logging in ["system-out", "out-err", "all"]: - systemout = tnode.find_first_by_tag("system-out") + systemout = tnode.get_first_by_tag("system-out") systemout_xml = systemout.toxml() assert systemout.tag == "system-out", "Expected tag: system-out" assert "info msg" not in systemout_xml, "INFO message found in system-out" - assert ( - "hello-stdout" in systemout_xml - ), "Missing 'hello-stdout' in system-out" + assert "hello-stdout" in systemout_xml, ( + "Missing 'hello-stdout' in system-out" + ) if junit_logging in ["system-err", "out-err", "all"]: - systemerr = tnode.find_first_by_tag("system-err") + systemerr = tnode.get_first_by_tag("system-err") systemerr_xml = systemerr.toxml() assert systemerr.tag == "system-err", "Expected tag: system-err" assert "info msg" not in systemerr_xml, "INFO message found in system-err" - assert ( - "hello-stderr" in systemerr_xml - ), "Missing 'hello-stderr' in system-err" - assert ( - "warning msg" not in systemerr_xml - ), "WARN message found in system-err" + assert "hello-stderr" in systemerr_xml, ( + "Missing 'hello-stderr' in system-err" + ) + assert "warning msg" not in systemerr_xml, ( + "WARN message found in system-err" + ) if junit_logging == "no": assert not tnode.find_by_tag("log"), "Found unexpected content: log" - assert not tnode.find_by_tag( - "system-out" - ), "Found unexpected content: system-out" - assert not tnode.find_by_tag( - "system-err" - ), "Found unexpected content: system-err" + assert not tnode.find_by_tag("system-out"), ( + "Found unexpected content: system-out" + ) + assert not tnode.find_by_tag("system-err"), ( + "Found unexpected content: system-err" + ) @parametrize_families def test_failure_verbose_message( @@ -575,10 +635,10 @@ def test_fail(): assert 0, "An error" """ ) - result, dom = run_and_parse(family=xunit_family) - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") - fnode = tnode.find_first_by_tag("failure") + _result, dom = run_and_parse(family=xunit_family) + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") + fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="AssertionError: An error\nassert 0") @parametrize_families @@ -598,15 +658,14 @@ def test_func(arg1): "-o", "junit_logging=system-out", family=xunit_family ) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=3, tests=3) - - for index, char in enumerate("<&'"): - tnode = node.find_nth_by_tag("testcase", index) + tnodes = node.find_by_tag("testcase") + for tnode, char in zip(tnodes, "<&'", strict=True): tnode.assert_attr( classname="test_failure_escape", name=f"test_func[{char}]" ) - sysout = tnode.find_first_by_tag("system-out") + sysout = tnode.get_first_by_tag("system-out") text = sysout.text assert f"{char}\n" in text @@ -625,11 +684,11 @@ def test_hello(self): ) result, dom = run_and_parse("--junitprefix=xyz", family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(failures=1, tests=2) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="xyz.test_junit_prefixing", name="test_func") - tnode = node.find_nth_by_tag("testcase", 1) + tnode = node.find_by_tag("testcase")[1] tnode.assert_attr( classname="xyz.test_junit_prefixing.TestHello", name="test_hello" ) @@ -647,11 +706,11 @@ def test_xfail(): ) result, dom = run_and_parse(family=xunit_family) assert not result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_xfailure_function", name="test_xfail") - fnode = tnode.find_first_by_tag("skipped") + fnode = tnode.get_first_by_tag("skipped") fnode.assert_attr(type="pytest.xfail", message="42") @parametrize_families @@ -668,11 +727,11 @@ def test_xfail(): ) result, dom = run_and_parse(family=xunit_family) assert not result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=1, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_xfailure_marker", name="test_xfail") - fnode = tnode.find_first_by_tag("skipped") + fnode = tnode.get_first_by_tag("skipped") fnode.assert_attr(type="pytest.xfail", message="42") @pytest.mark.parametrize( @@ -693,18 +752,18 @@ def test_fail(): assert 0 """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") - if junit_logging in ["system-err", "out-err", "all"]: - assert len(tnode.find_by_tag("system-err")) == 1 - else: - assert len(tnode.find_by_tag("system-err")) == 0 + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") - if junit_logging in ["log", "system-out", "out-err", "all"]: - assert len(tnode.find_by_tag("system-out")) == 1 - else: - assert len(tnode.find_by_tag("system-out")) == 0 + has_err_logging = junit_logging in ["system-err", "out-err", "all"] + expected_err_output_len = 1 if has_err_logging else 0 + assert len(tnode.find_by_tag("system-err")) == expected_err_output_len + + has_out_logigng = junit_logging in ("log", "system-out", "out-err", "all") + expected_out_output_len = 1 if has_out_logigng else 0 + + assert len(tnode.find_by_tag("system-out")) == expected_out_output_len @parametrize_families def test_xfailure_xpass( @@ -718,11 +777,11 @@ def test_xpass(): pass """ ) - result, dom = run_and_parse(family=xunit_family) + _result, dom = run_and_parse(family=xunit_family) # assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_xfailure_xpass", name="test_xpass") @parametrize_families @@ -737,13 +796,13 @@ def test_xpass(): pass """ ) - result, dom = run_and_parse(family=xunit_family) + _result, dom = run_and_parse(family=xunit_family) # assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(skipped=0, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(classname="test_xfailure_xpass_strict", name="test_xpass") - fnode = tnode.find_first_by_tag("failure") + fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="[XPASS(strict)] This needs to fail!") @parametrize_families @@ -753,10 +812,10 @@ def test_collect_error( pytester.makepyfile("syntax error") result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=1, tests=1) - tnode = node.find_first_by_tag("testcase") - fnode = tnode.find_first_by_tag("error") + tnode = node.get_first_by_tag("testcase") + fnode = tnode.get_first_by_tag("error") fnode.assert_attr(message="collection failure") assert "SyntaxError" in fnode.toxml() @@ -772,8 +831,8 @@ def test_hello(): ) result, dom = run_and_parse() assert result.ret == 1 - tnode = dom.find_first_by_tag("testcase") - fnode = tnode.find_first_by_tag("failure") + tnode = dom.get_first_by_tag("testcase") + fnode = tnode.get_first_by_tag("failure") assert "hx" in fnode.toxml() def test_assertion_binchars( @@ -790,7 +849,7 @@ def test_str_compare(): assert M1 == M2 """ ) - result, dom = run_and_parse() + _result, dom = run_and_parse() print(dom.toxml()) @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) @@ -803,18 +862,18 @@ def test_pass(): print('hello-stdout') """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - pnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + pnode = node.get_first_by_tag("testcase") if junit_logging == "no": - assert not node.find_by_tag( - "system-out" - ), "system-out should not be generated" + assert not node.find_by_tag("system-out"), ( + "system-out should not be generated" + ) if junit_logging == "system-out": - systemout = pnode.find_first_by_tag("system-out") - assert ( - "hello-stdout" in systemout.toxml() - ), "'hello-stdout' should be in system-out" + systemout = pnode.get_first_by_tag("system-out") + assert "hello-stdout" in systemout.toxml(), ( + "'hello-stdout' should be in system-out" + ) @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) def test_pass_captures_stderr( @@ -827,18 +886,18 @@ def test_pass(): sys.stderr.write('hello-stderr') """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - pnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + pnode = node.get_first_by_tag("testcase") if junit_logging == "no": - assert not node.find_by_tag( - "system-err" - ), "system-err should not be generated" + assert not node.find_by_tag("system-err"), ( + "system-err should not be generated" + ) if junit_logging == "system-err": - systemerr = pnode.find_first_by_tag("system-err") - assert ( - "hello-stderr" in systemerr.toxml() - ), "'hello-stderr' should be in system-err" + systemerr = pnode.get_first_by_tag("system-err") + assert "hello-stderr" in systemerr.toxml(), ( + "'hello-stderr' should be in system-err" + ) @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_setup_error_captures_stdout( @@ -856,18 +915,18 @@ def test_function(arg): pass """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - pnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + pnode = node.get_first_by_tag("testcase") if junit_logging == "no": - assert not node.find_by_tag( - "system-out" - ), "system-out should not be generated" + assert not node.find_by_tag("system-out"), ( + "system-out should not be generated" + ) if junit_logging == "system-out": - systemout = pnode.find_first_by_tag("system-out") - assert ( - "hello-stdout" in systemout.toxml() - ), "'hello-stdout' should be in system-out" + systemout = pnode.get_first_by_tag("system-out") + assert "hello-stdout" in systemout.toxml(), ( + "'hello-stdout' should be in system-out" + ) @pytest.mark.parametrize("junit_logging", ["no", "system-err"]) def test_setup_error_captures_stderr( @@ -886,18 +945,18 @@ def test_function(arg): pass """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - pnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + pnode = node.get_first_by_tag("testcase") if junit_logging == "no": - assert not node.find_by_tag( - "system-err" - ), "system-err should not be generated" + assert not node.find_by_tag("system-err"), ( + "system-err should not be generated" + ) if junit_logging == "system-err": - systemerr = pnode.find_first_by_tag("system-err") - assert ( - "hello-stderr" in systemerr.toxml() - ), "'hello-stderr' should be in system-err" + systemerr = pnode.get_first_by_tag("system-err") + assert "hello-stderr" in systemerr.toxml(), ( + "'hello-stderr' should be in system-err" + ) @pytest.mark.parametrize("junit_logging", ["no", "system-out"]) def test_avoid_double_stdout( @@ -917,15 +976,15 @@ def test_function(arg): sys.stdout.write('hello-stdout call') """ ) - result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") - node = dom.find_first_by_tag("testsuite") - pnode = node.find_first_by_tag("testcase") + _result, dom = run_and_parse("-o", f"junit_logging={junit_logging}") + node = dom.get_first_by_tag("testsuite") + pnode = node.get_first_by_tag("testcase") if junit_logging == "no": - assert not node.find_by_tag( - "system-out" - ), "system-out should not be generated" + assert not node.find_by_tag("system-out"), ( + "system-out should not be generated" + ) if junit_logging == "system-out": - systemout = pnode.find_first_by_tag("system-out") + systemout = pnode.get_first_by_tag("system-out") assert "hello-stdout call" in systemout.toxml() assert "hello-stdout teardown" in systemout.toxml() @@ -945,12 +1004,12 @@ class FakeConfig: if TYPE_CHECKING: workerinput = None - def __init__(self): + def __init__(self) -> None: self.pluginmanager = self self.option = self self.stash = Stash() - def getini(self, name): + def getini(self, name: str) -> str: return "pytest" junitprefix = None @@ -989,11 +1048,11 @@ def repr_failure(self, excinfo): pytester.path.joinpath("myfile.xyz").write_text("hello", encoding="utf-8") result, dom = run_and_parse(family=xunit_family) assert result.ret - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(errors=0, failures=1, skipped=0, tests=1) - tnode = node.find_first_by_tag("testcase") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(name="myfile.xyz") - fnode = tnode.find_first_by_tag("failure") + fnode = tnode.get_first_by_tag("failure") fnode.assert_attr(message="custom item runtest failed") assert "custom item runtest failed" in fnode.toxml() @@ -1134,7 +1193,7 @@ def test_func(char): ) result, dom = run_and_parse() assert result.ret == 0 - node = dom.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testcase") node.assert_attr(name="test_func[\\x00]") @@ -1151,7 +1210,7 @@ def test_func(param): ) result, dom = run_and_parse() assert result.ret == 0 - node = dom.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_function_issue469") node.assert_attr(name="test_func[double::colon]") @@ -1170,7 +1229,7 @@ def test_func(self, param): ) result, dom = run_and_parse() assert result.ret == 0 - node = dom.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testcase") node.assert_attr(classname="test_double_colon_split_method_issue469.TestClass") node.assert_attr(name="test_func[double::colon]") @@ -1218,9 +1277,9 @@ def test_record(record_property, other): """ ) result, dom = run_and_parse() - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") - psnode = tnode.find_first_by_tag("properties") + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") + psnode = tnode.get_first_by_tag("properties") pnodes = psnode.find_by_tag("property") pnodes[0].assert_attr(name="bar", value="1") pnodes[1].assert_attr(name="foo", value="<1") @@ -1246,10 +1305,10 @@ def test_record(record_property, other): """ ) result, dom = run_and_parse() - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") tnodes = node.find_by_tag("testcase") for tnode in tnodes: - psnode = tnode.find_first_by_tag("properties") + psnode = tnode.get_first_by_tag("properties") assert psnode, f"testcase didn't had expected properties:\n{tnode}" pnodes = psnode.find_by_tag("property") pnodes[0].assert_attr(name="bar", value="1") @@ -1267,10 +1326,10 @@ def test_record_with_same_name(record_property): record_property("foo", "baz") """ ) - result, dom = run_and_parse() - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") - psnode = tnode.find_first_by_tag("properties") + _result, dom = run_and_parse() + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") + psnode = tnode.get_first_by_tag("properties") pnodes = psnode.find_by_tag("property") pnodes[0].assert_attr(name="foo", value="bar") pnodes[1].assert_attr(name="foo", value="baz") @@ -1310,8 +1369,8 @@ def test_record(record_xml_attribute, other): """ ) result, dom = run_and_parse() - node = dom.find_first_by_tag("testsuite") - tnode = node.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testsuite") + tnode = node.get_first_by_tag("testcase") tnode.assert_attr(bar="1") tnode.assert_attr(foo="<1") result.stdout.fnmatch_lines( @@ -1343,7 +1402,7 @@ def test_record({fixture_name}, other): """ ) - result, dom = run_and_parse(family=None) + result, _dom = run_and_parse(family=None) expected_lines = [] if fixture_name == "record_xml_attribute": expected_lines.append( @@ -1373,7 +1432,7 @@ def test_x(i): """ ) _, dom = run_and_parse("-n2") - suite_node = dom.find_first_by_tag("testsuite") + suite_node = dom.get_first_by_tag("testsuite") failed = [] for case_node in suite_node.find_by_tag("testcase"): if case_node.find_first_by_tag("failure"): @@ -1395,6 +1454,7 @@ def test_x(): _, dom = run_and_parse(family=xunit_family) root = dom.get_unique_child assert root.tag == "testsuites" + root.assert_attr(name="pytest tests") suite_node = root.get_unique_child assert suite_node.tag == "testsuite" @@ -1407,7 +1467,7 @@ def test_pass(): """ ) - result, dom = run_and_parse(f, f) + result, dom = run_and_parse("--keep-duplicates", f, f) result.stdout.no_fnmatch_line("*INTERNALERROR*") first, second = (x["classname"] for x in dom.find_by_tag("testcase")) assert first == second @@ -1469,7 +1529,7 @@ def test_pass(): result.stdout.no_fnmatch_line("*INTERNALERROR*") items = sorted( - "%(classname)s %(name)s" % x # noqa: UP031 + f"{x['classname']} {x['name']}" # dom is a DomNode not a mapping, it's not possible to ** it. for x in dom.find_by_tag("testcase") ) @@ -1544,9 +1604,9 @@ class Report(BaseReport): test_case = minidom.parse(str(path)).getElementsByTagName("testcase")[0] - assert ( - test_case.getAttribute("url") == test_url - ), "The URL did not get written to the xml" + assert test_case.getAttribute("url") == test_url, ( + "The URL did not get written to the xml" + ) @parametrize_families @@ -1564,10 +1624,11 @@ def test_func2(record_testsuite_property): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testsuite") - properties_node = node.find_first_by_tag("properties") - p1_node = properties_node.find_nth_by_tag("property", 0) - p2_node = properties_node.find_nth_by_tag("property", 1) + node = dom.get_first_by_tag("testsuite") + properties_node = node.get_first_by_tag("properties") + p1_node, p2_node = properties_node.find_by_tag( + "property", + )[:2] p1_node.assert_attr(name="stats", value="all good") p2_node.assert_attr(name="stats", value="10") @@ -1627,7 +1688,7 @@ def test_func(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testsuite") + node = dom.get_first_by_tag("testsuite") node.assert_attr(name=expected) @@ -1643,8 +1704,8 @@ def test_skip(): """ ) _, dom = run_and_parse() - node = dom.find_first_by_tag("testcase") - snode = node.find_first_by_tag("skipped") + node = dom.get_first_by_tag("testcase") + snode = node.get_first_by_tag("skipped") assert "1 <> 2" in snode.text snode.assert_attr(message="1 <> 2") @@ -1660,8 +1721,8 @@ def test_skip(): """ ) _, dom = run_and_parse() - node = dom.find_first_by_tag("testcase") - snode = node.find_first_by_tag("skipped") + node = dom.get_first_by_tag("testcase") + snode = node.get_first_by_tag("skipped") assert "#x1B[31;1mred#x1B[0m" in snode.text snode.assert_attr(message="#x1B[31;1mred#x1B[0m") @@ -1682,8 +1743,8 @@ def test_esc(my_setup): """ ) _, dom = run_and_parse() - node = dom.find_first_by_tag("testcase") - snode = node.find_first_by_tag("error") + node = dom.get_first_by_tag("testcase") + snode = node.get_first_by_tag("error") assert "#x1B[31mred#x1B[m" in snode["message"] assert "#x1B[31mred#x1B[m" in snode.text @@ -1714,7 +1775,7 @@ def test_func(): ) result, dom = run_and_parse(family=xunit_family) assert result.ret == 0 - node = dom.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testcase") assert len(node.find_by_tag("system-err")) == 0 assert len(node.find_by_tag("system-out")) == 0 @@ -1749,7 +1810,7 @@ def test_func(): "-o", f"junit_logging={junit_logging}", family=xunit_family ) assert result.ret == 1 - node = dom.find_first_by_tag("testcase") + node = dom.get_first_by_tag("testcase") if junit_logging == "system-out": assert len(node.find_by_tag("system-err")) == 0 assert len(node.find_by_tag("system-out")) == 1 @@ -1760,3 +1821,13 @@ def test_func(): assert junit_logging == "no" assert len(node.find_by_tag("system-err")) == 0 assert len(node.find_by_tag("system-out")) == 0 + + +def test_no_message_quiet(pytester: Pytester) -> None: + """Do not show the summary banner when --quiet is given (#13700).""" + pytester.makepyfile("def test(): pass") + result = pytester.runpytest("--junitxml=pytest.xml") + result.stdout.fnmatch_lines("* generated xml file: *") + + result = pytester.runpytest("--junitxml=pytest.xml", "--quiet") + result.stdout.no_fnmatch_line("* generated xml file: *") diff --git a/testing/test_legacypath.py b/testing/test_legacypath.py index 72854e4e5c0..d1f2255f30f 100644 --- a/testing/test_legacypath.py +++ b/testing/test_legacypath.py @@ -12,10 +12,10 @@ def test_item_fspath(pytester: pytest.Pytester) -> None: pytester.makepyfile("def test_func(): pass") - items, hookrec = pytester.inline_genitems() + items, _hookrec = pytester.inline_genitems() assert len(items) == 1 (item,) = items - items2, hookrec = pytester.inline_genitems(item.nodeid) + items2, _hookrec = pytester.inline_genitems(item.nodeid) (item2,) = items2 assert item2.name == item.name assert item2.fspath == item.fspath @@ -113,7 +113,7 @@ def test_session_scoped_unavailable_attributes(self, session_request): _ = session_request.fspath -@pytest.mark.parametrize("config_type", ["ini", "pyproject"]) +@pytest.mark.parametrize("config_type", ["ini", "toml"]) def test_addini_paths(pytester: pytest.Pytester, config_type: str) -> None: pytester.makeconftest( """ @@ -126,15 +126,15 @@ def pytest_addoption(parser): inipath = pytester.makeini( """ [pytest] - paths=hello world/sub.py - """ + paths = hello world/sub.py + """ ) - elif config_type == "pyproject": - inipath = pytester.makepyprojecttoml( + else: + inipath = pytester.maketoml( + """ + [pytest] + paths = ["hello", "world/sub.py"] """ - [tool.pytest.ini_options] - paths=["hello", "world/sub.py"] - """ ) config = pytester.parseconfig() values = config.getini("paths") diff --git a/testing/test_main.py b/testing/test_main.py index 94eac02ce63..41d7055df26 100644 --- a/testing/test_main.py +++ b/testing/test_main.py @@ -121,100 +121,151 @@ def invocation_path(self, pytester: Pytester) -> Path: def test_file(self, invocation_path: Path) -> None: """File and parts.""" assert resolve_collection_argument( - invocation_path, "src/pkg/test.py" + invocation_path, "src/pkg/test.py", 0 ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=[], + parametrization=None, module_name=None, + original_index=0, ) assert resolve_collection_argument( - invocation_path, "src/pkg/test.py::" + invocation_path, "src/pkg/test.py::", 10 ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=[""], + parametrization=None, module_name=None, + original_index=10, ) assert resolve_collection_argument( - invocation_path, "src/pkg/test.py::foo::bar" + invocation_path, "src/pkg/test.py::foo::bar", 20 ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=["foo", "bar"], + parametrization=None, module_name=None, + original_index=20, ) assert resolve_collection_argument( - invocation_path, "src/pkg/test.py::foo::bar::" + invocation_path, "src/pkg/test.py::foo::bar::", 30 ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=["foo", "bar", ""], + parametrization=None, module_name=None, + original_index=30, + ) + assert resolve_collection_argument( + invocation_path, "src/pkg/test.py::foo::bar[a,b,c]", 40 + ) == CollectionArgument( + path=invocation_path / "src/pkg/test.py", + parts=["foo", "bar"], + parametrization="[a,b,c]", + module_name=None, + original_index=40, ) def test_dir(self, invocation_path: Path) -> None: """Directory and parts.""" assert resolve_collection_argument( - invocation_path, "src/pkg" + invocation_path, "src/pkg", 0 ) == CollectionArgument( path=invocation_path / "src/pkg", parts=[], + parametrization=None, module_name=None, + original_index=0, ) with pytest.raises( UsageError, match=r"directory argument cannot contain :: selection parts" ): - resolve_collection_argument(invocation_path, "src/pkg::") + resolve_collection_argument(invocation_path, "src/pkg::", 0) with pytest.raises( UsageError, match=r"directory argument cannot contain :: selection parts" ): - resolve_collection_argument(invocation_path, "src/pkg::foo::bar") + resolve_collection_argument(invocation_path, "src/pkg::foo::bar", 0) - def test_pypath(self, invocation_path: Path) -> None: + @pytest.mark.parametrize("namespace_package", [False, True]) + def test_pypath(self, namespace_package: bool, invocation_path: Path) -> None: """Dotted name and parts.""" + if namespace_package: + # Namespace package doesn't have to contain __init__py + (invocation_path / "src/pkg/__init__.py").unlink() + assert resolve_collection_argument( - invocation_path, "pkg.test", as_pypath=True + invocation_path, "pkg.test", 0, as_pypath=True ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=[], + parametrization=None, module_name="pkg.test", + original_index=0, ) assert resolve_collection_argument( - invocation_path, "pkg.test::foo::bar", as_pypath=True + invocation_path, "pkg.test::foo::bar", 0, as_pypath=True ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", parts=["foo", "bar"], + parametrization=None, module_name="pkg.test", + original_index=0, ) assert resolve_collection_argument( - invocation_path, "pkg", as_pypath=True + invocation_path, + "pkg", + 0, + as_pypath=True, + consider_namespace_packages=namespace_package, ) == CollectionArgument( path=invocation_path / "src/pkg", parts=[], + parametrization=None, module_name="pkg", + original_index=0, ) with pytest.raises( UsageError, match=r"package argument cannot contain :: selection parts" ): resolve_collection_argument( - invocation_path, "pkg::foo::bar", as_pypath=True + invocation_path, + "pkg::foo::bar", + 0, + as_pypath=True, + consider_namespace_packages=namespace_package, ) def test_parametrized_name_with_colons(self, invocation_path: Path) -> None: assert resolve_collection_argument( - invocation_path, "src/pkg/test.py::test[a::b]" + invocation_path, "src/pkg/test.py::test[a::b]", 0 ) == CollectionArgument( path=invocation_path / "src/pkg/test.py", - parts=["test[a::b]"], + parts=["test"], + parametrization="[a::b]", module_name=None, + original_index=0, ) + @pytest.mark.parametrize( + "arg", ["x.py[a]", "x.py[a]::foo", "x/y.py[a]::foo::bar", "x.py[a]::foo[b]"] + ) + def test_path_parametrization_not_allowed( + self, invocation_path: Path, arg: str + ) -> None: + with pytest.raises( + UsageError, match=r"path cannot contain \[\] parametrization" + ): + resolve_collection_argument(invocation_path, arg, 0) + def test_does_not_exist(self, invocation_path: Path) -> None: """Given a file/module that does not exist raises UsageError.""" with pytest.raises( UsageError, match=re.escape("file or directory not found: foobar") ): - resolve_collection_argument(invocation_path, "foobar") + resolve_collection_argument(invocation_path, "foobar", 0) with pytest.raises( UsageError, @@ -222,28 +273,32 @@ def test_does_not_exist(self, invocation_path: Path) -> None: "module or package not found: foobar (missing __init__.py?)" ), ): - resolve_collection_argument(invocation_path, "foobar", as_pypath=True) + resolve_collection_argument(invocation_path, "foobar", 0, as_pypath=True) def test_absolute_paths_are_resolved_correctly(self, invocation_path: Path) -> None: """Absolute paths resolve back to absolute paths.""" full_path = str(invocation_path / "src") assert resolve_collection_argument( - invocation_path, full_path + invocation_path, full_path, 0 ) == CollectionArgument( path=Path(os.path.abspath("src")), parts=[], + parametrization=None, module_name=None, + original_index=0, ) # ensure full paths given in the command-line without the drive letter resolve # to the full path correctly (#7628) - drive, full_path_without_drive = os.path.splitdrive(full_path) + _drive, full_path_without_drive = os.path.splitdrive(full_path) assert resolve_collection_argument( - invocation_path, full_path_without_drive + invocation_path, full_path_without_drive, 0 ) == CollectionArgument( path=Path(os.path.abspath("src")), parts=[], + parametrization=None, module_name=None, + original_index=0, ) @@ -275,7 +330,7 @@ def test(fix): fn = pytester.path.joinpath("project/tests/dummy_test.py") assert fn.is_file() - drive, path = os.path.splitdrive(str(fn)) + _drive, path = os.path.splitdrive(str(fn)) result = pytester.runpytest(path, "-v") result.stdout.fnmatch_lines( diff --git a/testing/test_mark.py b/testing/test_mark.py index 89eef7920cf..8d76ea310eb 100644 --- a/testing/test_mark.py +++ b/testing/test_mark.py @@ -59,7 +59,7 @@ def test_1(self, abc): """ ) file_name = os.path.basename(py_file) - rec = pytester.inline_run(file_name, file_name) + rec = pytester.inline_run("--keep-duplicates", file_name, file_name) rec.assertoutcome(passed=6) @@ -183,7 +183,9 @@ def test_hello(): reprec.assertoutcome(passed=1) -@pytest.mark.parametrize("option_name", ["--strict-markers", "--strict"]) +@pytest.mark.parametrize( + "option_name", ["--strict-markers", "--strict", "strict_markers", "strict"] +) def test_strict_prohibits_unregistered_markers( pytester: Pytester, option_name: str ) -> None: @@ -195,7 +197,16 @@ def test_hello(): pass """ ) - result = pytester.runpytest(option_name) + if option_name in ("strict_markers", "strict"): + pytester.makeini( + f""" + [pytest] + {option_name} = true + """ + ) + result = pytester.runpytest() + else: + result = pytester.runpytest(option_name) assert result.ret != 0 result.stdout.fnmatch_lines( ["'unregisteredmark' not found in `markers` configuration option"] @@ -228,7 +239,7 @@ def test_two(): """ ) rec = pytester.inline_run("-m", expr) - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() passed_str = [x.nodeid.split("::")[-1] for x in passed] assert passed_str == expected_passed @@ -276,7 +287,7 @@ def test_three(): """ ) rec = pytester.inline_run("-m", expr) - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() passed_str = [x.nodeid.split("::")[-1] for x in passed] assert passed_str == expected_passed @@ -306,7 +317,7 @@ def test_nointer(): """ ) rec = pytester.inline_run("-m", expr) - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() passed_str = [x.nodeid.split("::")[-1] for x in passed] assert passed_str == expected_passed @@ -341,7 +352,7 @@ def test_2(): """ ) rec = pytester.inline_run("-k", expr) - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() passed_str = [x.nodeid.split("::")[-1] for x in passed] assert passed_str == expected_passed @@ -373,7 +384,7 @@ def test_func(arg): """ ) rec = pytester.inline_run("-k", expr) - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() passed_str = [x.nodeid.split("::")[-1] for x in passed] assert passed_str == expected_passed @@ -388,7 +399,7 @@ def test_func(arg): """ ) rec = pytester.inline_run() - passed, skipped, fail = rec.listoutcomes() + passed, _skipped, _fail = rec.listoutcomes() expected_id = "test_func[" + pytest.__name__ + "]" assert passed[0].nodeid.split("::")[-1] == expected_id @@ -537,7 +548,7 @@ def test_d(self): assert True """ ) - items, rec = pytester.inline_genitems(p) + items, _rec = pytester.inline_genitems(p) for item in items: print(item, item.keywords) assert [x for x in item.iter_markers() if x.name == "a"] @@ -560,7 +571,7 @@ class Test2(Base): def test_bar(self): pass """ ) - items, rec = pytester.inline_genitems(p) + items, _rec = pytester.inline_genitems(p) self.assert_markers(items, test_foo=("a", "b"), test_bar=("a",)) def test_mark_should_not_pass_to_siebling_class(self, pytester: Pytester) -> None: @@ -583,7 +594,7 @@ class TestOtherSub(TestBase): """ ) - items, rec = pytester.inline_genitems(p) + items, _rec = pytester.inline_genitems(p) base_item, sub_item, sub_item_other = items print(items, [x.nodeid for x in items]) # new api segregates @@ -611,7 +622,7 @@ class Test2(Base2): def test_bar(self): pass """ ) - items, rec = pytester.inline_genitems(p) + items, _rec = pytester.inline_genitems(p) self.assert_markers(items, test_foo=("a", "b", "c"), test_bar=("a", "b", "d")) def test_mark_closest(self, pytester: Pytester) -> None: @@ -630,7 +641,7 @@ def test_has_inherited(self): """ ) - items, rec = pytester.inline_genitems(p) + items, _rec = pytester.inline_genitems(p) has_own, has_inherited = items has_own_marker = has_own.get_closest_marker("c") has_inherited_marker = has_inherited.get_closest_marker("c") @@ -826,7 +837,7 @@ def test_method_one(self): def check(keyword, name): reprec = pytester.inline_run("-s", "-k", keyword, file_test) - passed, skipped, failed = reprec.listoutcomes() + _passed, _skipped, failed = reprec.listoutcomes() assert len(failed) == 1 assert failed[0].nodeid.split("::")[-1] == name assert len(reprec.getcalls("pytest_deselected")) == 1 @@ -869,7 +880,7 @@ def pytest_pycollect_makeitem(name): ) reprec = pytester.inline_run(p.parent, "-s", "-k", keyword) print("keyword", repr(keyword)) - passed, skipped, failed = reprec.listoutcomes() + passed, _skipped, _failed = reprec.listoutcomes() assert len(passed) == 1 assert passed[0].nodeid.endswith("test_2") dlist = reprec.getcalls("pytest_deselected") @@ -885,7 +896,7 @@ def test_one(): """ ) reprec = pytester.inline_run("-k", "mykeyword", p) - passed, skipped, failed = reprec.countoutcomes() + _passed, _skipped, failed = reprec.countoutcomes() assert failed == 1 @pytest.mark.xfail @@ -1048,6 +1059,32 @@ def test(): assert result.ret == ExitCode.INTERRUPTED +def test_paramset_empty_no_idfunc( + pytester: Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + """An empty parameter set should not call the user provided id function (#13031).""" + p1 = pytester.makepyfile( + """ + import pytest + + def idfunc(value): + raise ValueError() + @pytest.mark.parametrize("param", [], ids=idfunc) + def test(param): + pass + """ + ) + result = pytester.runpytest(p1, "-v", "-rs") + result.stdout.fnmatch_lines( + [ + "* collected 1 item", + "test_paramset_empty_no_idfunc* SKIPPED *", + "SKIPPED [1] test_paramset_empty_no_idfunc.py:5: got empty parameter set for (param)", + "*= 1 skipped in *", + ] + ) + + def test_parameterset_for_parametrize_bad_markname(pytester: Pytester) -> None: with pytest.raises(pytest.UsageError): test_parameterset_for_parametrize_marks(pytester, "bad") @@ -1144,7 +1181,11 @@ def test_pytest_param_id_requires_string() -> None: with pytest.raises(TypeError) as excinfo: pytest.param(id=True) # type: ignore[arg-type] (msg,) = excinfo.value.args - assert msg == "Expected id to be a string, got : True" + expected = ( + "Expected id to be a string or a `pytest.HIDDEN_PARAM` sentinel, " + "got : True" + ) + assert msg == expected @pytest.mark.parametrize("s", (None, "hello world")) @@ -1226,3 +1267,38 @@ def test_attrs(self): ) result = pytester.runpytest(foo) result.assert_outcomes(passed=1) + + +def test_mark_parametrize_over_staticmethod(pytester: Pytester) -> None: + """Check that applying marks works as intended on classmethods and staticmethods. + + Regression test for #12863. + """ + pytester.makepyfile( + """ + import pytest + + class TestClass: + @pytest.mark.parametrize("value", [1, 2]) + @classmethod + def test_classmethod_wrapper(cls, value: int): + assert value in [1, 2] + + @classmethod + @pytest.mark.parametrize("value", [1, 2]) + def test_classmethod_wrapper_on_top(cls, value: int): + assert value in [1, 2] + + @pytest.mark.parametrize("value", [1, 2]) + @staticmethod + def test_staticmethod_wrapper(value: int): + assert value in [1, 2] + + @staticmethod + @pytest.mark.parametrize("value", [1, 2]) + def test_staticmethod_wrapper_on_top(value: int): + assert value in [1, 2] + """ + ) + result = pytester.runpytest() + result.assert_outcomes(passed=8) diff --git a/testing/test_mark_expression.py b/testing/test_mark_expression.py index a61a9f21560..1e3c769347c 100644 --- a/testing/test_mark_expression.py +++ b/testing/test_mark_expression.py @@ -1,30 +1,25 @@ from __future__ import annotations -from typing import Callable -from typing import cast - from _pytest.mark import MarkMatcher from _pytest.mark.expression import Expression -from _pytest.mark.expression import MatcherCall -from _pytest.mark.expression import ParseError +from _pytest.mark.expression import ExpressionMatcher import pytest -def evaluate(input: str, matcher: Callable[[str], bool]) -> bool: - return Expression.compile(input).evaluate(cast(MatcherCall, matcher)) +def evaluate(input: str, matcher: ExpressionMatcher) -> bool: + return Expression.compile(input).evaluate(matcher) def test_empty_is_false() -> None: - assert not evaluate("", lambda ident: False) - assert not evaluate("", lambda ident: True) - assert not evaluate(" ", lambda ident: False) - assert not evaluate("\t", lambda ident: False) + assert not evaluate("", lambda ident, /, **kwargs: False) + assert not evaluate("", lambda ident, /, **kwargs: True) + assert not evaluate(" ", lambda ident, /, **kwargs: False) + assert not evaluate("\t", lambda ident, /, **kwargs: False) @pytest.mark.parametrize( ("expr", "expected"), ( - ("true", True), ("true", True), ("false", False), ("not true", False), @@ -51,7 +46,9 @@ def test_empty_is_false() -> None: ), ) def test_basic(expr: str, expected: bool) -> None: - matcher = {"true": True, "false": False}.__getitem__ + def matcher(name: str, /, **kwargs: str | int | bool | None) -> bool: + return {"true": True, "false": False}[name] + assert evaluate(expr, matcher) is expected @@ -67,7 +64,9 @@ def test_basic(expr: str, expected: bool) -> None: ), ) def test_syntax_oddities(expr: str, expected: bool) -> None: - matcher = {"true": True, "false": False}.__getitem__ + def matcher(name: str, /, **kwargs: str | int | bool | None) -> bool: + return {"true": True, "false": False}[name] + assert evaluate(expr, matcher) is expected @@ -77,11 +76,13 @@ def test_backslash_not_treated_specially() -> None: user will never need to insert a literal newline, only \n (two chars). So mark expressions themselves do not support escaping, instead they treat backslashes as regular identifier characters.""" - matcher = {r"\nfoo\n"}.__contains__ + + def matcher(name: str, /, **kwargs: str | int | bool | None) -> bool: + return {r"\nfoo\n"}.__contains__(name) assert evaluate(r"\nfoo\n", matcher) assert not evaluate(r"foo", matcher) - with pytest.raises(ParseError): + with pytest.raises(SyntaxError): evaluate("\nfoo\n", matcher) @@ -134,10 +135,10 @@ def test_backslash_not_treated_specially() -> None: ), ) def test_syntax_errors(expr: str, column: int, message: str) -> None: - with pytest.raises(ParseError) as excinfo: - evaluate(expr, lambda ident: True) - assert excinfo.value.column == column - assert excinfo.value.message == message + with pytest.raises(SyntaxError) as excinfo: + evaluate(expr, lambda ident, /, **kwargs: True) + assert excinfo.value.offset == column + assert excinfo.value.msg == message @pytest.mark.parametrize( @@ -172,7 +173,10 @@ def test_syntax_errors(expr: str, column: int, message: str) -> None: ), ) def test_valid_idents(ident: str) -> None: - assert evaluate(ident, {ident: True}.__getitem__) + def matcher(name: str, /, **kwargs: str | int | bool | None) -> bool: + return name == ident + + assert evaluate(ident, matcher) @pytest.mark.parametrize( @@ -198,13 +202,14 @@ def test_valid_idents(ident: str) -> None: ), ) def test_invalid_idents(ident: str) -> None: - with pytest.raises(ParseError): - evaluate(ident, lambda ident: True) + with pytest.raises(SyntaxError): + evaluate(ident, lambda ident, /, **kwargs: True) @pytest.mark.parametrize( "expr, expected_error_msg", ( + ("mark()", "expected identifier; got right parenthesis"), ("mark(True=False)", "unexpected reserved python keyword `True`"), ("mark(def=False)", "unexpected reserved python keyword `def`"), ("mark(class=False)", "unexpected reserved python keyword `class`"), @@ -234,7 +239,7 @@ def test_invalid_idents(ident: str) -> None: def test_invalid_kwarg_name_or_value( expr: str, expected_error_msg: str, mark_matcher: MarkMatcher ) -> None: - with pytest.raises(ParseError, match=expected_error_msg): + with pytest.raises(SyntaxError, match=expected_error_msg): assert evaluate(expr, mark_matcher) diff --git a/testing/test_monkeypatch.py b/testing/test_monkeypatch.py index 079d8ff60ad..c321439e398 100644 --- a/testing/test_monkeypatch.py +++ b/testing/test_monkeypatch.py @@ -1,12 +1,13 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator import os from pathlib import Path import re import sys import textwrap -from typing import Generator +import warnings from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester @@ -14,7 +15,7 @@ @pytest.fixture -def mp() -> Generator[MonkeyPatch, None, None]: +def mp() -> Generator[MonkeyPatch]: cwd = os.getcwd() sys_path = list(sys.path) yield MonkeyPatch() @@ -428,10 +429,16 @@ class A: assert A.x == 1 -@pytest.mark.filterwarnings(r"ignore:.*\bpkg_resources\b:DeprecationWarning") +@pytest.mark.filterwarnings( + r"ignore:.*\bpkg_resources\b:DeprecationWarning", + r"ignore:.*\bpkg_resources\b:UserWarning", +) def test_syspath_prepend_with_namespace_packages( pytester: Pytester, monkeypatch: MonkeyPatch ) -> None: + # Needs to be in sys.modules. + pytest.importorskip("pkg_resources") + for dirname in "hello", "world": d = pytester.mkdir(dirname) ns = d.joinpath("ns_pkg") @@ -445,7 +452,9 @@ def test_syspath_prepend_with_namespace_packages( f"def check(): return {dirname!r}", encoding="utf-8" ) + # First call should not warn - namespace package not registered yet. monkeypatch.syspath_prepend("hello") + # This registers ns_pkg as a namespace package. import ns_pkg.hello assert ns_pkg.hello.check() == "hello" @@ -454,13 +463,19 @@ def test_syspath_prepend_with_namespace_packages( import ns_pkg.world # Prepending should call fixup_namespace_packages. - monkeypatch.syspath_prepend("world") + # This call should warn - ns_pkg is now registered and "world" contains it + with pytest.warns(pytest.PytestRemovedIn10Warning, match="legacy namespace"): + monkeypatch.syspath_prepend("world") import ns_pkg.world assert ns_pkg.world.check() == "world" # Should invalidate caches via importlib.invalidate_caches. + # Should not warn for path without namespace packages. modules_tmpdir = pytester.mkdir("modules_tmpdir") - monkeypatch.syspath_prepend(str(modules_tmpdir)) + with warnings.catch_warnings(): + warnings.simplefilter("error") + monkeypatch.syspath_prepend(str(modules_tmpdir)) + modules_tmpdir.joinpath("main_app.py").write_text("app = True", encoding="utf-8") from main_app import app # noqa: F401 diff --git a/testing/test_nodes.py b/testing/test_nodes.py index f039acf243b..de7875ca427 100644 --- a/testing/test_nodes.py +++ b/testing/test_nodes.py @@ -3,7 +3,6 @@ from pathlib import Path import re -from typing import cast import warnings from _pytest import nodes @@ -25,10 +24,10 @@ def test_node_direct_construction_deprecated() -> None: with pytest.raises( OutcomeException, match=( - "Direct construction of _pytest.nodes.Node has been deprecated, please " - "use _pytest.nodes.Node.from_parent.\nSee " - "https://docs.pytest.org/en/stable/deprecations.html#node-construction-changed-to-node-from-parent" - " for more details." + r"Direct construction of _pytest\.nodes\.Node has been deprecated, please " + r"use _pytest\.nodes\.Node\.from_parent.\nSee " + r"https://docs\.pytest\.org/en/stable/deprecations\.html#node-construction-changed-to-node-from-parent" + r" for more details\." ), ): nodes.Node(None, session=None) # type: ignore[arg-type] @@ -103,24 +102,15 @@ def test__check_initialpaths_for_relpath() -> None: """Ensure that it handles dirs, and does not always use dirname.""" cwd = Path.cwd() - class FakeSession1: - _initialpaths = frozenset({cwd}) + initial_paths = frozenset({cwd}) - session = cast(pytest.Session, FakeSession1) - - assert nodes._check_initialpaths_for_relpath(session, cwd) == "" + assert nodes._check_initialpaths_for_relpath(initial_paths, cwd) == "" sub = cwd / "file" - - class FakeSession2: - _initialpaths = frozenset({cwd}) - - session = cast(pytest.Session, FakeSession2) - - assert nodes._check_initialpaths_for_relpath(session, sub) == "file" + assert nodes._check_initialpaths_for_relpath(initial_paths, sub) == "file" outside = Path("/outside-this-does-not-exist") - assert nodes._check_initialpaths_for_relpath(session, outside) is None + assert nodes._check_initialpaths_for_relpath(initial_paths, outside) is None def test_failure_with_changed_cwd(pytester: Pytester) -> None: diff --git a/testing/test_parseopt.py b/testing/test_parseopt.py index 14e2b5f69fb..30370d3d673 100644 --- a/testing/test_parseopt.py +++ b/testing/test_parseopt.py @@ -28,9 +28,10 @@ def test_no_help_by_default(self) -> None: def test_custom_prog(self, parser: parseopt.Parser) -> None: """Custom prog can be set for `argparse.ArgumentParser`.""" - assert parser._getparser().prog == os.path.basename(sys.argv[0]) + assert parser.optparser.prog == argparse.ArgumentParser().prog parser.prog = "custom-prog" - assert parser._getparser().prog == "custom-prog" + assert parser.prog == "custom-prog" + assert parser.optparser.prog == "custom-prog" def test_argument(self) -> None: with pytest.raises(parseopt.ArgumentError): @@ -71,14 +72,12 @@ def test_argument_processopt(self) -> None: assert res["dest"] == "abc" def test_group_add_and_get(self, parser: parseopt.Parser) -> None: - group = parser.getgroup("hello", description="desc") + group = parser.getgroup("hello") assert group.name == "hello" - assert group.description == "desc" def test_getgroup_simple(self, parser: parseopt.Parser) -> None: - group = parser.getgroup("hello", description="desc") + group = parser.getgroup("hello") assert group.name == "hello" - assert group.description == "desc" group2 = parser.getgroup("hello") assert group2 is group @@ -88,16 +87,20 @@ def test_group_ordering(self, parser: parseopt.Parser) -> None: parser.getgroup("3", after="1") groups = parser._groups groups_names = [x.name for x in groups] - assert groups_names == list("132") + assert groups_names == ["_anonymous", "1", "3", "2"] def test_group_addoption(self) -> None: - group = parseopt.OptionGroup("hello", _ispytest=True) + optparser = argparse.ArgumentParser() + arggroup = optparser.add_argument_group("hello") + group = parseopt.OptionGroup(arggroup, "hello", None, _ispytest=True) group.addoption("--option1", action="store_true") assert len(group.options) == 1 assert isinstance(group.options[0], parseopt.Argument) def test_group_addoption_conflict(self) -> None: - group = parseopt.OptionGroup("hello again", _ispytest=True) + optparser = argparse.ArgumentParser() + arggroup = optparser.add_argument_group("hello again") + group = parseopt.OptionGroup(arggroup, "hello again", None, _ispytest=True) group.addoption("--option1", "--option-1", action="store_true") with pytest.raises(ValueError) as err: group.addoption("--option1", "--option-one", action="store_true") @@ -142,35 +145,32 @@ def test_parse_known_args(self, parser: parseopt.Parser) -> None: parser.parse_known_args([Path(".")]) parser.addoption("--hello", action="store_true") ns = parser.parse_known_args(["x", "--y", "--hello", "this"]) - assert ns.hello - assert ns.file_or_dir == ["x"] + assert ns.hello is True + assert ns.file_or_dir == ["x", "this"] def test_parse_known_and_unknown_args(self, parser: parseopt.Parser) -> None: parser.addoption("--hello", action="store_true") ns, unknown = parser.parse_known_and_unknown_args( ["x", "--y", "--hello", "this"] ) - assert ns.hello - assert ns.file_or_dir == ["x"] - assert unknown == ["--y", "this"] + assert ns.hello is True + assert ns.file_or_dir == ["x", "this"] + assert unknown == ["--y"] def test_parse_will_set_default(self, parser: parseopt.Parser) -> None: parser.addoption("--hello", dest="hello", default="x", action="store") option = parser.parse([]) assert option.hello == "x" - del option.hello - parser.parse_setoption([], option) - assert option.hello == "x" - def test_parse_setoption(self, parser: parseopt.Parser) -> None: + def test_parse_set_options(self, parser: parseopt.Parser) -> None: parser.addoption("--hello", dest="hello", action="store") parser.addoption("--world", dest="world", default=42) option = argparse.Namespace() - args = parser.parse_setoption(["--hello", "world"], option) + parser.parse(["--hello", "world"], option) assert option.hello == "world" assert option.world == 42 - assert not args + assert getattr(option, parseopt.FILE_OR_DIR) == [] def test_parse_special_destination(self, parser: parseopt.Parser) -> None: parser.addoption("--ultimate-answer", type=int) diff --git a/testing/test_pastebin.py b/testing/test_pastebin.py index 8fdd60bac75..9b928e00c06 100644 --- a/testing/test_pastebin.py +++ b/testing/test_pastebin.py @@ -3,6 +3,7 @@ import email.message import io +from unittest import mock from _pytest.monkeypatch import MonkeyPatch from _pytest.pytester import Pytester @@ -90,23 +91,6 @@ class TestPaste: def pastebin(self, request): return request.config.pluginmanager.getplugin("pastebin") - @pytest.fixture - def mocked_urlopen_fail(self, monkeypatch: MonkeyPatch): - """Monkeypatch the actual urlopen call to emulate a HTTP Error 400.""" - calls = [] - - import urllib.error - import urllib.request - - def mocked(url, data): - calls.append((url, data)) - raise urllib.error.HTTPError( - url, 400, "Bad request", email.message.Message(), io.BytesIO() - ) - - monkeypatch.setattr(urllib.request, "urlopen", mocked) - return calls - @pytest.fixture def mocked_urlopen_invalid(self, monkeypatch: MonkeyPatch): """Monkeypatch the actual urlopen calls done by the internal plugin @@ -158,10 +142,33 @@ def test_pastebin_invalid_url(self, pastebin, mocked_urlopen_invalid) -> None: ) assert len(mocked_urlopen_invalid) == 1 - def test_pastebin_http_error(self, pastebin, mocked_urlopen_fail) -> None: - result = pastebin.create_new_paste(b"full-paste-contents") + def test_pastebin_http_error(self, pastebin) -> None: + import urllib.error + + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.HTTPError( + url="https://bpa.st", + code=400, + msg="Bad request", + hdrs=email.message.Message(), + fp=io.BytesIO(), + ), + ) as mock_urlopen: + result = pastebin.create_new_paste(b"full-paste-contents") assert result == "bad response: HTTP Error 400: Bad request" - assert len(mocked_urlopen_fail) == 1 + assert len(mock_urlopen.mock_calls) == 1 + + def test_pastebin_url_error(self, pastebin) -> None: + import urllib.error + + with mock.patch( + "urllib.request.urlopen", + side_effect=urllib.error.URLError("the url was bad"), + ) as mock_urlopen: + result = pastebin.create_new_paste(b"full-paste-contents") + assert result == "bad response: " + assert len(mock_urlopen.mock_calls) == 1 def test_create_new_paste(self, pastebin, mocked_urlopen) -> None: result = pastebin.create_new_paste(b"full-paste-contents") diff --git a/testing/test_pathlib.py b/testing/test_pathlib.py index 81aba25f78f..0880c355557 100644 --- a/testing/test_pathlib.py +++ b/testing/test_pathlib.py @@ -1,6 +1,9 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Generator +from collections.abc import Iterator +from collections.abc import Sequence import errno import importlib.abc import importlib.machinery @@ -12,12 +15,11 @@ from textwrap import dedent from types import ModuleType from typing import Any -from typing import Generator -from typing import Iterator -from typing import Sequence import unittest.mock +from _pytest.config import ExitCode from _pytest.monkeypatch import MonkeyPatch +from _pytest.pathlib import _import_module_using_spec from _pytest.pathlib import bestrelpath from _pytest.pathlib import commonpath from _pytest.pathlib import compute_module_name @@ -36,6 +38,8 @@ from _pytest.pathlib import resolve_package_path from _pytest.pathlib import resolve_pkg_root_and_module_name from _pytest.pathlib import safe_exists +from _pytest.pathlib import scandir +from _pytest.pathlib import spec_matches_module_path from _pytest.pathlib import symlink_or_skip from _pytest.pathlib import visit from _pytest.pytester import Pytester @@ -125,7 +129,7 @@ class TestImportPath: """ @pytest.fixture(scope="session") - def path1(self, tmp_path_factory: TempPathFactory) -> Generator[Path, None, None]: + def path1(self, tmp_path_factory: TempPathFactory) -> Generator[Path]: path = tmp_path_factory.mktemp("path") self.setuptestfs(path) yield path @@ -416,7 +420,7 @@ def test_no_meta_path_found( del sys.modules[module.__name__] monkeypatch.setattr( - importlib.util, "spec_from_file_location", lambda *args: None + importlib.util, "spec_from_file_location", lambda *args, **kwargs: None ) with pytest.raises(ImportError): import_path( @@ -566,6 +570,29 @@ def test_samefile_false_negatives(tmp_path: Path, monkeypatch: MonkeyPatch) -> N assert getattr(module, "foo")() == 42 +def test_scandir_with_non_existent_directory() -> None: + # Test with a directory that does not exist + non_existent_dir = "path_to_non_existent_dir" + result = scandir(non_existent_dir) + # Assert that the result is an empty list + assert result == [] + + +def test_scandir_handles_os_error() -> None: + # Create a mock entry that will raise an OSError when is_file is called + mock_entry = unittest.mock.MagicMock() + mock_entry.is_file.side_effect = OSError("some permission error") + # Mock os.scandir to return an iterator with our mock entry + with unittest.mock.patch("os.scandir") as mock_scandir: + mock_scandir.return_value.__enter__.return_value = [mock_entry] + # Call the scandir function with a path + # We expect an OSError to be raised here + with pytest.raises(OSError, match="some permission error"): + scandir("/fake/path") + # Verify that the is_file method was called on the mock entry + mock_entry.is_file.assert_called_once() + + class TestImportLibMode: def test_importmode_importlib_with_dataclass( self, tmp_path: Path, ns_param: bool @@ -780,6 +807,62 @@ def test_insert_missing_modules( insert_missing_modules(modules, "") assert modules == {} + @pytest.mark.parametrize("b_is_package", [True, False]) + @pytest.mark.parametrize("insert_modules", [True, False]) + def test_import_module_using_spec( + self, b_is_package, insert_modules, tmp_path: Path + ): + """ + Verify that `_import_module_using_spec` can obtain a spec based on the path, thereby enabling the import. + When importing, not only the target module is imported, but also the parent modules are recursively imported. + """ + file_path = tmp_path / "a/b/c/demo.py" + file_path.parent.mkdir(parents=True) + file_path.write_text("my_name='demo'", encoding="utf-8") + + if b_is_package: + (tmp_path / "a/b/__init__.py").write_text( + "my_name='b.__init__'", encoding="utf-8" + ) + + mod = _import_module_using_spec( + "a.b.c.demo", + file_path, + file_path.parent, + insert_modules=insert_modules, + ) + + # target module is imported + assert mod is not None + assert spec_matches_module_path(mod.__spec__, file_path) is True + + mod_demo = sys.modules["a.b.c.demo"] + assert "demo.py" in str(mod_demo) + assert mod_demo.my_name == "demo" # Imported and available for use + + # parent modules are recursively imported. + mod_a = sys.modules["a"] + mod_b = sys.modules["a.b"] + mod_c = sys.modules["a.b.c"] + + assert mod_a.b is mod_b + assert mod_a.b.c is mod_c + assert mod_a.b.c.demo is mod_demo + + assert "namespace" in str(mod_a).lower() + assert "namespace" in str(mod_c).lower() + + # Compatibility package and namespace package. + if b_is_package: + assert "namespace" not in str(mod_b).lower() + assert "__init__.py" in str(mod_b).lower() # Imported __init__.py + assert mod_b.my_name == "b.__init__" # Imported and available for use + + else: + assert "namespace" in str(mod_b).lower() + with pytest.raises(AttributeError): # Not imported __init__.py + assert mod_b.my_name + def test_parent_contains_child_module_attribute( self, monkeypatch: MonkeyPatch, tmp_path: Path ): @@ -863,6 +946,37 @@ def test_my_test(): result = pytester.runpytest("--import-mode=importlib") result.stdout.fnmatch_lines("* 1 passed *") + @pytest.mark.parametrize("name", ["code", "time", "math"]) + def test_importlib_same_name_as_stl( + self, pytester, ns_param: bool, tmp_path: Path, name: str + ): + """Import a namespace package with the same name as the standard library (#13026).""" + file_path = pytester.path / f"{name}/foo/test_demo.py" + file_path.parent.mkdir(parents=True) + file_path.write_text( + dedent( + """ + def test_demo(): + pass + """ + ), + encoding="utf-8", + ) + + # unit test + __import__(name) # import standard library + + import_path( # import user files + file_path, + mode=ImportMode.importlib, + root=pytester.path, + consider_namespace_packages=ns_param, + ) + + # E2E test + result = pytester.runpytest("--import-mode=importlib") + result.stdout.fnmatch_lines("* 1 passed *") + def create_installed_doctests_and_tests_dir( self, path: Path, monkeypatch: MonkeyPatch ) -> tuple[Path, Path, Path]: @@ -1372,6 +1486,70 @@ def test_resolve_pkg_root_and_module_name_ns_multiple_levels( ) assert mod is mod2 + def test_ns_multiple_levels_import_rewrite_assertions( + self, + tmp_path: Path, + monkeypatch: MonkeyPatch, + pytester: Pytester, + ) -> None: + """Check assert rewriting with `--import-mode=importlib` (#12659).""" + self.setup_directories(tmp_path, monkeypatch, pytester) + code = dedent(""" + def test(): + assert "four lights" == "five lights" + """) + + # A case is in a subdirectory with an `__init__.py` file. + test_py = tmp_path / "src/dist2/com/company/calc/algo/test_demo.py" + test_py.write_text(code, encoding="UTF-8") + + pkg_root, module_name = resolve_pkg_root_and_module_name( + test_py, consider_namespace_packages=True + ) + assert (pkg_root, module_name) == ( + tmp_path / "src/dist2", + "com.company.calc.algo.test_demo", + ) + + result = pytester.runpytest("--import-mode=importlib", test_py) + + result.stdout.fnmatch_lines( + [ + "E AssertionError: assert 'four lights' == 'five lights'", + "E *", + "E - five lights*", + "E + four lights", + ] + ) + + def test_ns_multiple_levels_import_error( + self, + tmp_path: Path, + pytester: Pytester, + ) -> None: + # Trigger condition 1: ns and file with the same name + file = pytester.path / "cow/moo/moo.py" + file.parent.mkdir(parents=True) + file.write_text("data=123", encoding="utf-8") + + # Trigger condition 2: tests are located in ns + tests = pytester.path / "cow/moo/test_moo.py" + + tests.write_text( + dedent( + """ + from cow.moo.moo import data + + def test_moo(): + print(data) + """ + ), + encoding="utf-8", + ) + + result = pytester.runpytest("--import-mode=importlib") + assert result.ret == ExitCode.OK + @pytest.mark.parametrize("import_mode", ["prepend", "append", "importlib"]) def test_incorrect_namespace_package( self, @@ -1446,7 +1624,7 @@ def find_spec( return None # Setup directories without configuring sys.path. - models_py, algorithms_py = self.setup_directories( + models_py, _algorithms_py = self.setup_directories( tmp_path, monkeypatch=None, pytester=pytester ) com_root_1 = tmp_path / "src/dist1/com" @@ -1506,6 +1684,19 @@ def test_full_ns_packages_without_init_files( ) == (tmp_path / "src/dist2", "ns.a.core.foo.m") +def test_ns_import_same_name_directory_12592( + tmp_path: Path, pytester: Pytester +) -> None: + """Regression for `--import-mode=importlib` with directory parent and child with same name (#12592).""" + y_dir = tmp_path / "x/y/y" + y_dir.mkdir(parents=True) + test_y = tmp_path / "x/y/test_y.py" + test_y.write_text("def test(): pass", encoding="UTF-8") + + result = pytester.runpytest("--import-mode=importlib", test_y) + assert result.ret == ExitCode.OK + + def test_is_importable(pytester: Pytester) -> None: pytester.syspathinsert() diff --git a/testing/test_pluginmanager.py b/testing/test_pluginmanager.py index db85124bf0d..24700c07c80 100644 --- a/testing/test_pluginmanager.py +++ b/testing/test_pluginmanager.py @@ -436,7 +436,7 @@ def test_preparse_args(self, pytestpm: PytestPluginManager) -> None: # Handles -p without following arg (when used without argparse). pytestpm.consider_preparse(["-p"]) - with pytest.raises(UsageError, match="^plugin main cannot be disabled$"): + with pytest.raises(UsageError, match=r"^plugin main cannot be disabled$"): pytestpm.consider_preparse(["-p", "no:main"]) def test_plugin_prevent_register(self, pytestpm: PytestPluginManager) -> None: diff --git a/testing/test_pytester.py b/testing/test_pytester.py index 87714b4708f..5e2e22f111b 100644 --- a/testing/test_pytester.py +++ b/testing/test_pytester.py @@ -4,7 +4,6 @@ import os import subprocess import sys -import time from types import ModuleType from _pytest.config import ExitCode @@ -16,6 +15,7 @@ from _pytest.pytester import Pytester from _pytest.pytester import SysModulesSnapshot from _pytest.pytester import SysPathsSnapshot +import _pytest.timing import pytest @@ -451,13 +451,12 @@ def test_pytester_run_with_timeout(pytester: Pytester) -> None: timeout = 120 - start = time.time() + instant = _pytest.timing.Instant() result = pytester.runpytest_subprocess(testfile, timeout=timeout) - end = time.time() - duration = end - start + duration = instant.elapsed() assert result.ret == ExitCode.OK - assert duration < timeout + assert duration.seconds < timeout def test_pytester_run_timeout_expires(pytester: Pytester) -> None: @@ -800,7 +799,7 @@ def test_pytester_makefile_dot_prefixes_extension_with_warning( ) -> None: with pytest.raises( ValueError, - match="pytester.makefile expects a file extension, try .foo.bar instead of foo.bar", + match=r"pytester\.makefile expects a file extension, try \.foo\.bar instead of foo\.bar", ): pytester.makefile("foo.bar", "") @@ -835,3 +834,25 @@ def test_two(): result.assert_outcomes(passed=1, deselected=1) # If deselected is not passed, it is not checked at all. result.assert_outcomes(passed=1) + + +def test_pytester_subprocess_with_string_plugins(pytester: Pytester) -> None: + """Test that pytester.runpytest_subprocess is OK with named (string) + `.plugins`.""" + pytester.plugins = ["pytester"] + + result = pytester.runpytest_subprocess() + assert result.ret == ExitCode.NO_TESTS_COLLECTED + + +def test_pytester_subprocess_with_non_string_plugins(pytester: Pytester) -> None: + """Test that pytester.runpytest_subprocess fails with a proper error given + non-string `.plugins`.""" + + class MyPlugin: + pass + + pytester.plugins = [MyPlugin()] + + with pytest.raises(ValueError, match="plugins as objects is not supported"): + pytester.runpytest_subprocess() diff --git a/testing/test_python_path.py b/testing/test_python_path.py index 1db02252d22..f75bcb6bb57 100644 --- a/testing/test_python_path.py +++ b/testing/test_python_path.py @@ -3,7 +3,6 @@ import sys from textwrap import dedent -from typing import Generator from _pytest.pytester import Pytester import pytest @@ -62,6 +61,27 @@ def test_two_dirs(pytester: Pytester, file_structure) -> None: result.assert_outcomes(passed=2) +def test_local_plugin(pytester: Pytester, file_structure) -> None: + """`pythonpath` kicks early enough to load plugins via -p (#11118).""" + localplugin_py = pytester.path / "sub" / "localplugin.py" + content = dedent( + """ + def pytest_load_initial_conftests(): + print("local plugin load") + + def pytest_unconfigure(): + print("local plugin unconfig") + """ + ) + localplugin_py.write_text(content, encoding="utf-8") + + pytester.makeini("[pytest]\npythonpath=sub\n") + result = pytester.runpytest("-plocalplugin", "-s", "test_foo.py") + result.stdout.fnmatch_lines(["local plugin load", "local plugin unconfig"]) + assert result.ret == 0 + result.assert_outcomes(passed=1) + + def test_module_not_found(pytester: Pytester, file_structure) -> None: """Without the pythonpath setting, the module should not be found.""" pytester.makefile(".ini", pytest="[pytest]\n") @@ -72,8 +92,8 @@ def test_module_not_found(pytester: Pytester, file_structure) -> None: result.stdout.fnmatch_lines([expected_error]) -def test_no_ini(pytester: Pytester, file_structure) -> None: - """If no ini file, test should error.""" +def test_no_config_file(pytester: Pytester, file_structure) -> None: + """If no configuration file, test should error.""" result = pytester.runpytest("test_foo.py") assert result.ret == pytest.ExitCode.INTERRUPTED result.assert_outcomes(errors=1) @@ -95,16 +115,13 @@ def test_clean_up(pytester: Pytester) -> None: after: list[str] | None = None class Plugin: - @pytest.hookimpl(wrapper=True, tryfirst=True) - def pytest_unconfigure(self) -> Generator[None, None, None]: - nonlocal before, after + @pytest.hookimpl(tryfirst=True) + def pytest_unconfigure(self) -> None: + nonlocal before before = sys.path.copy() - try: - return (yield) - finally: - after = sys.path.copy() result = pytester.runpytest_inprocess(plugins=[Plugin()]) + after = sys.path.copy() assert result.ret == 0 assert before is not None diff --git a/testing/test_reports.py b/testing/test_reports.py index 3e314d2aade..b81371587d9 100644 --- a/testing/test_reports.py +++ b/testing/test_reports.py @@ -1,7 +1,7 @@ # mypy: allow-untyped-defs from __future__ import annotations -from typing import Sequence +from collections.abc import Sequence from _pytest._code.code import ExceptionChainRepr from _pytest._code.code import ExceptionRepr @@ -101,8 +101,7 @@ def test_repr_entry(): rep_entries = rep.longrepr.reprtraceback.reprentries a_entries = a.longrepr.reprtraceback.reprentries - assert len(rep_entries) == len(a_entries) # python < 3.10 zip(strict=True) - for a_entry, rep_entry in zip(a_entries, rep_entries): + for a_entry, rep_entry in zip(a_entries, rep_entries, strict=True): assert isinstance(rep_entry, ReprEntry) assert rep_entry.reprfileloc is not None assert rep_entry.reprfuncargs is not None @@ -146,8 +145,7 @@ def test_repr_entry_native(): rep_entries = rep.longrepr.reprtraceback.reprentries a_entries = a.longrepr.reprtraceback.reprentries - assert len(rep_entries) == len(a_entries) # python < 3.10 zip(strict=True) - for rep_entry, a_entry in zip(rep_entries, a_entries): + for rep_entry, a_entry in zip(rep_entries, a_entries, strict=True): assert isinstance(rep_entry, ReprEntryNative) assert rep_entry.lines == a_entry.lines @@ -319,8 +317,8 @@ def check_longrepr(longrepr: ExceptionChainRepr) -> None: assert longrepr.sections == [("title", "contents", "=")] assert len(longrepr.chain) == 2 entry1, entry2 = longrepr.chain - tb1, fileloc1, desc1 = entry1 - tb2, fileloc2, desc2 = entry2 + tb1, _fileloc1, desc1 = entry1 + tb2, _fileloc2, desc2 = entry2 assert "ValueError('value error')" in str(tb1) assert "RuntimeError('runtime error')" in str(tb2) @@ -377,8 +375,8 @@ def check_longrepr(longrepr: object) -> None: assert isinstance(longrepr, ExceptionChainRepr) assert len(longrepr.chain) == 2 entry1, entry2 = longrepr.chain - tb1, fileloc1, desc1 = entry1 - tb2, fileloc2, desc2 = entry2 + tb1, fileloc1, _desc1 = entry1 + tb2, fileloc2, _desc2 = entry2 assert "RemoteTraceback" in str(tb1) assert "ValueError: value error" in str(tb2) @@ -436,6 +434,83 @@ def test_1(fixture_): timing.sleep(10) loaded_report = TestReport._from_json(data) assert loaded_report.stop - loaded_report.start == approx(report.duration) + @pytest.mark.parametrize( + "first_skip_reason, second_skip_reason, skip_reason_output", + [("A", "B", "(A; B)"), ("A", "A", "(A)")], + ) + def test_exception_group_with_only_skips( + self, + pytester: Pytester, + first_skip_reason: str, + second_skip_reason: str, + skip_reason_output: str, + ): + """ + Test that when an ExceptionGroup with only Skipped exceptions is raised in teardown, + it is reported as a single skipped test, not as an error. + This is a regression test for issue #13537. + """ + pytester.makepyfile( + test_it=f""" + import pytest + @pytest.fixture + def fixA(): + yield + pytest.skip(reason="{first_skip_reason}") + @pytest.fixture + def fixB(): + yield + pytest.skip(reason="{second_skip_reason}") + def test_skip(fixA, fixB): + assert True + """ + ) + result = pytester.runpytest("-v") + result.assert_outcomes(passed=1, skipped=1) + out = result.stdout.str() + assert skip_reason_output in out + assert "ERROR at teardown" not in out + + @pytest.mark.parametrize( + "use_item_location, skip_file_location", + [(True, "test_it.py"), (False, "runner.py")], + ) + def test_exception_group_skips_use_item_location( + self, pytester: Pytester, use_item_location: bool, skip_file_location: str + ): + """ + Regression for #13537: + If any skip inside an ExceptionGroup has _use_item_location=True, + the report location should point to the test item, not the fixture teardown. + """ + pytester.makepyfile( + test_it=f""" + import pytest + @pytest.fixture + def fix_item1(): + yield + exc = pytest.skip.Exception("A") + exc._use_item_location = True + raise exc + @pytest.fixture + def fix_item2(): + yield + exc = pytest.skip.Exception("B") + exc._use_item_location = {use_item_location} + raise exc + def test_both(fix_item1, fix_item2): + assert True + """ + ) + result = pytester.runpytest("-rs") + result.assert_outcomes(passed=1, skipped=1) + + out = result.stdout.str() + # Both reasons should appear + assert "A" and "B" in out + # Crucially, the skip should be attributed to the test item, not teardown + assert skip_file_location in out + class TestHooks: """Test that the hooks are working correctly for plugins""" diff --git a/testing/test_runner.py b/testing/test_runner.py index 1b59ff78633..0245438a47d 100644 --- a/testing/test_runner.py +++ b/testing/test_runner.py @@ -137,8 +137,8 @@ def raiser(exc): ss.teardown_exact(None) mod, func = e.value.exceptions assert isinstance(mod, KeyError) - assert isinstance(func.exceptions[0], TypeError) # type: ignore - assert isinstance(func.exceptions[1], ValueError) # type: ignore + assert isinstance(func.exceptions[0], TypeError) + assert isinstance(func.exceptions[1], ValueError) def test_cached_exception_doesnt_get_longer(self, pytester: Pytester) -> None: """Regression test for #12204 (the "BTW" case).""" @@ -1030,7 +1030,7 @@ def runtest(self): assert sys.last_type is IndexError assert isinstance(sys.last_value, IndexError) if sys.version_info >= (3, 12, 0): - assert isinstance(sys.last_exc, IndexError) + assert isinstance(sys.last_exc, IndexError) # type:ignore[attr-defined] assert sys.last_value.args[0] == "TEST" assert sys.last_traceback diff --git a/testing/test_session.py b/testing/test_session.py index ba904916033..e3db9a1b690 100644 --- a/testing/test_session.py +++ b/testing/test_session.py @@ -66,7 +66,7 @@ def test_raises_doesnt(): pytest.raises(ValueError, int, "3") """ ) - passed, skipped, failed = reprec.listoutcomes() + _passed, _skipped, failed = reprec.listoutcomes() assert len(failed) == 1 out = failed[0].longrepr.reprcrash.message # type: ignore[union-attr] assert "DID NOT RAISE" in out diff --git a/testing/test_skipping.py b/testing/test_skipping.py index d1a63b1d920..e1e25e45468 100644 --- a/testing/test_skipping.py +++ b/testing/test_skipping.py @@ -1,7 +1,6 @@ # mypy: allow-untyped-defs from __future__ import annotations -import sys import textwrap from _pytest.pytester import Pytester @@ -448,8 +447,8 @@ def test_this_false(): result = pytester.runpytest(p, "-rx") result.stdout.fnmatch_lines( [ - "*test_one*test_this - reason: *NOTRUN* noway", - "*test_one*test_this_true - reason: *NOTRUN* condition: True", + "*test_one*test_this - *NOTRUN* noway", + "*test_one*test_this_true - *NOTRUN* condition: True", "*1 passed*", ] ) @@ -492,7 +491,7 @@ def test_this(): result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*1 xfailed*"]) result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines(["*XFAIL*test_this*reason:*hello*"]) + result.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) result = pytester.runpytest(p, "--runxfail") result.stdout.fnmatch_lines(["*1 pass*"]) @@ -510,7 +509,7 @@ def test_this(): result = pytester.runpytest(p) result.stdout.fnmatch_lines(["*1 xfailed*"]) result = pytester.runpytest(p, "-rx") - result.stdout.fnmatch_lines(["*XFAIL*test_this*reason:*hello*"]) + result.stdout.fnmatch_lines(["*XFAIL*test_this*hello*"]) result = pytester.runpytest(p, "--runxfail") result.stdout.fnmatch_lines( """ @@ -602,13 +601,12 @@ def test_xfail_raises( self, expected, actual, matchline, pytester: Pytester ) -> None: p = pytester.makepyfile( - """ + f""" import pytest - @pytest.mark.xfail(raises=%s) + @pytest.mark.xfail(raises={expected}) def test_raises(): - raise %s() - """ # noqa: UP031 (python syntax issues) - % (expected, actual) + raise {actual}() + """ ) result = pytester.runpytest(p) result.stdout.fnmatch_lines([matchline]) @@ -686,13 +684,14 @@ def test_foo(): assert result.ret == 0 @pytest.mark.parametrize("strict_val", ["true", "false"]) + @pytest.mark.parametrize("option_name", ["strict_xfail", "strict"]) def test_strict_xfail_default_from_file( - self, pytester: Pytester, strict_val + self, pytester: Pytester, strict_val: str, option_name: str ) -> None: pytester.makeini( f""" [pytest] - xfail_strict = {strict_val} + {option_name} = {strict_val} """ ) p = pytester.makepyfile( @@ -900,13 +899,12 @@ def test_func(): ) def test_skipif_reporting(self, pytester: Pytester, params) -> None: p = pytester.makepyfile( - test_foo=""" + test_foo=f""" import pytest - @pytest.mark.skipif(%(params)s) + @pytest.mark.skipif({params}) def test_that(): assert 0 - """ # noqa: UP031 (python syntax issues) - % dict(params=params) + """ ) result = pytester.runpytest(p, "-s", "-rs") result.stdout.fnmatch_lines(["*SKIP*1*test_foo.py*platform*", "*1 skipped*"]) @@ -1138,22 +1136,13 @@ def test_func(): """ ) result = pytester.runpytest() - markline = " ^" - pypy_version_info = getattr(sys, "pypy_version_info", None) - if pypy_version_info is not None: - markline = markline[7:] - - if sys.version_info >= (3, 10): - expected = [ - "*ERROR*test_nameerror*", - "*asd*", - "", - "During handling of the above exception, another exception occurred:", - ] - else: - expected = [ - "*ERROR*test_nameerror*", - ] + + expected = [ + "*ERROR*test_nameerror*", + "*asd*", + "", + "During handling of the above exception, another exception occurred:", + ] expected += [ "*evaluating*skipif*condition*", @@ -1161,7 +1150,7 @@ def test_func(): "*ERROR*test_syntax*", "*evaluating*xfail*condition*", " syntax error", - markline, + " ^", "SyntaxError: invalid syntax", "*1 pass*2 errors*", ] @@ -1190,7 +1179,7 @@ def test_default_markers(pytester: Pytester) -> None: result.stdout.fnmatch_lines( [ "*skipif(condition, ..., [*], reason=...)*skip*", - "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=xfail_strict)*expected failure*", + "*xfail(condition, ..., [*], reason=..., run=True, raises=None, strict=strict_xfail)*expected failure*", ] ) @@ -1316,7 +1305,7 @@ def pytest_collect_file(file_path, parent): """ ) result = pytester.inline_run() - passed, skipped, failed = result.listoutcomes() + _passed, skipped, failed = result.listoutcomes() assert not failed xfailed = [r for r in skipped if hasattr(r, "wasxfail")] assert xfailed @@ -1390,7 +1379,7 @@ def pytest_collect_file(file_path, parent): """ ) result = pytester.inline_run() - passed, skipped, failed = result.listoutcomes() + _passed, skipped, failed = result.listoutcomes() assert not failed xfailed = [r for r in skipped if hasattr(r, "wasxfail")] assert xfailed @@ -1418,7 +1407,7 @@ def test_fail(): def test_importorskip() -> None: with pytest.raises( pytest.skip.Exception, - match="^could not import 'doesnotexist': No module named .*", + match=r"^could not import 'doesnotexist': No module named .*", ): pytest.importorskip("doesnotexist") diff --git a/testing/test_stepwise.py b/testing/test_stepwise.py index affdb73375e..d2ad3bae500 100644 --- a/testing/test_stepwise.py +++ b/testing/test_stepwise.py @@ -1,6 +1,8 @@ -# mypy: allow-untyped-defs +# mypy: disallow-untyped-defs from __future__ import annotations +from collections.abc import Sequence +import json from pathlib import Path from _pytest.cacheprovider import Cache @@ -84,7 +86,7 @@ def broken_pytester(pytester: Pytester) -> Pytester: return pytester -def _strip_resource_warnings(lines): +def _strip_resource_warnings(lines: Sequence[str]) -> Sequence[str]: # Strip unreliable ResourceWarnings, so no-output assertions on stderr can work. # (https://github.com/pytest-dev/pytest/issues/5088) return [ @@ -114,7 +116,10 @@ def test_data(expected): result.stdout.fnmatch_lines(["stepwise: no previously failed tests, not skipping."]) result = pytester.runpytest("-v", "--stepwise") result.stdout.fnmatch_lines( - ["stepwise: skipping 4 already passed items.", "*1 failed, 4 deselected*"] + [ + "stepwise: skipping 4 already passed items (cache from * ago, use --sw-reset to discard).", + "*1 failed, 4 deselected*", + ] ) @@ -358,3 +363,183 @@ def test_one(): with stepwise_cache_file.open(encoding="utf-8") as file_handle: observed_value = file_handle.readlines() assert [expected_value] == observed_value + + +def test_do_not_reset_cache_if_disabled(pytester: Pytester) -> None: + """ + If pytest is run without --stepwise, do not clear the stepwise cache. + + Keeping the cache around is important for this workflow: + + 1. Run tests with --stepwise + 2. Stop at the failing test, and iterate over it changing the code and running it in isolation + (in the IDE for example). + 3. Run tests with --stepwise again - at this point we expect to start from the failing test, which should now pass, + and continue with the next tests. + """ + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + """ + ) + result = pytester.runpytest("--stepwise") + result.stdout.fnmatch_lines( + [ + "*::test_2 - assert False*", + "*failed, continuing from this test next run*", + "=* 1 failed, 1 passed in *", + ] + ) + + # Run a specific test without passing `--stepwise`. + result = pytester.runpytest("-k", "test_1") + result.stdout.fnmatch_lines(["*1 passed*"]) + + # Running with `--stepwise` should continue from the last failing test. + result = pytester.runpytest("--stepwise") + result.stdout.fnmatch_lines( + [ + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", + "*::test_2 - assert False*", + "*failed, continuing from this test next run*", + "=* 1 failed, 1 deselected in *", + ] + ) + + +def test_reset(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 deselected in *", + ] + ) + + # Running with --stepwise-reset restarts the stepwise workflow. + result = pytester.runpytest("-v", "--stepwise-reset") + result.stdout.fnmatch_lines( + [ + "stepwise: resetting state, not skipping.", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + +def test_change_test_count(pytester: Pytester) -> None: + # Run initially with 3 tests. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + # Change the number of tests, which invalidates the test cache. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): assert False + def test_3(): pass + def test_4(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: test count changed, not skipping (now 4 tests, previously 3).", + "*::test_1 *PASSED*", + "*::test_2 *FAILED*", + "*failed, continuing from this test next run*", + "* 1 failed, 1 passed in *", + ] + ) + + # Fix the failing test and run again. + pytester.makepyfile( + """ + def test_1(): pass + def test_2(): pass + def test_3(): pass + def test_4(): pass + """ + ) + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: skipping 1 already passed items (cache from *, use --sw-reset to discard).", + "*::test_2 *PASSED*", + "*::test_3 *PASSED*", + "*::test_4 *PASSED*", + "* 3 passed, 1 deselected in *", + ] + ) + + +def test_cache_error(pytester: Pytester) -> None: + pytester.makepyfile( + """ + def test_1(): pass + """ + ) + # Run stepwise normally to generate the cache information. + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "* 1 passed in *", + ] + ) + + # Corrupt the cache. + cache_file = pytester.path / f".pytest_cache/v/{STEPWISE_CACHE_DIR}" + assert cache_file.is_file() + cache_file.write_text(json.dumps({"invalid": True}), encoding="UTF-8") + + # Check we run as if the cache did not exist, but also show an error message. + result = pytester.runpytest("--stepwise", "-v") + result.stdout.fnmatch_lines( + [ + "stepwise: error reading cache, discarding (KeyError: *", + "stepwise: no previously failed tests, not skipping.", + "*::test_1 *PASSED*", + "* 1 passed in *", + ] + ) diff --git a/testing/test_subtests.py b/testing/test_subtests.py new file mode 100644 index 00000000000..06de9f009d8 --- /dev/null +++ b/testing/test_subtests.py @@ -0,0 +1,1034 @@ +from __future__ import annotations + +import sys +from typing import Literal + +from _pytest.subtests import SubtestContext +from _pytest.subtests import SubtestReport +import pytest + + +IS_PY311 = sys.version_info[:2] >= (3, 11) + + +def test_failures(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + def test_foo(subtests): + with subtests.test("foo subtest"): + assert False, "foo subtest failure" + + def test_bar(subtests): + with subtests.test("bar subtest"): + assert False, "bar subtest failure" + assert False, "test_bar also failed" + + def test_zaz(subtests): + with subtests.test("zaz subtest"): + pass + """ + ) + summary_lines = [ + "*=== FAILURES ===*", + # + "*___ test_foo [[]foo subtest[]] ___*", + "*AssertionError: foo subtest failure", + # + "*___ test_foo ___*", + "contains 1 failed subtest", + # + "*___ test_bar [[]bar subtest[]] ___*", + "*AssertionError: bar subtest failure", + # + "*___ test_bar ___*", + "*AssertionError: test_bar also failed", + # + "*=== short test summary info ===*", + "SUBFAILED[[]foo subtest[]] test_*.py::test_foo - AssertionError*", + "FAILED test_*.py::test_foo - contains 1 failed subtest", + "SUBFAILED[[]bar subtest[]] test_*.py::test_bar - AssertionError*", + "FAILED test_*.py::test_bar - AssertionError*", + ] + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "test_*.py uFuF. * [[]100%[]]", + *summary_lines, + "* 4 failed, 1 passed in *", + ] + ) + + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "test_*.py::test_foo SUBFAILED[[]foo subtest[]] * [[] 33%[]]", + "test_*.py::test_foo FAILED * [[] 33%[]]", + "test_*.py::test_bar SUBFAILED[[]bar subtest[]] * [[] 66%[]]", + "test_*.py::test_bar FAILED * [[] 66%[]]", + "test_*.py::test_zaz SUBPASSED[[]zaz subtest[]] * [[]100%[]]", + "test_*.py::test_zaz PASSED * [[]100%[]]", + *summary_lines, + "* 4 failed, 1 passed, 1 subtests passed in *", + ] + ) + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "test_*.py::test_foo SUBFAILED[[]foo subtest[]] * [[] 33%[]]", + "test_*.py::test_foo FAILED * [[] 33%[]]", + "test_*.py::test_bar SUBFAILED[[]bar subtest[]] * [[] 66%[]]", + "test_*.py::test_bar FAILED * [[] 66%[]]", + "test_*.py::test_zaz PASSED * [[]100%[]]", + *summary_lines, + "* 4 failed, 1 passed in *", + ] + ) + result.stdout.no_fnmatch_line("test_*.py::test_zaz SUBPASSED[[]zaz subtest[]]*") + + +def test_passes(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + def test_foo(subtests): + with subtests.test("foo subtest"): + pass + + def test_bar(subtests): + with subtests.test("bar subtest"): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "test_*.py .. * [[]100%[]]", + "* 2 passed in *", + ] + ) + + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo SUBPASSED[[]foo subtest[]] * [[] 50%[]]", + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar SUBPASSED[[]bar subtest[]] * [[]100%[]]", + "*.py::test_bar PASSED * [[]100%[]]", + "* 2 passed, 2 subtests passed in *", + ] + ) + + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar PASSED * [[]100%[]]", + "* 2 passed in *", + ] + ) + result.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") + result.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") + + +def test_skip(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + import pytest + def test_foo(subtests): + with subtests.test("foo subtest"): + pytest.skip("skip foo subtest") + + def test_bar(subtests): + with subtests.test("bar subtest"): + pytest.skip("skip bar subtest") + pytest.skip("skip test_bar") + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "test_*.py .s * [[]100%[]]", + "*=== short test summary info ===*", + "SKIPPED [[]1[]] test_skip.py:9: skip test_bar", + "* 1 passed, 1 skipped in *", + ] + ) + + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo SUBSKIPPED[[]foo subtest[]] (skip foo subtest) * [[] 50%[]]", + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar SUBSKIPPED[[]bar subtest[]] (skip bar subtest) * [[]100%[]]", + "*.py::test_bar SKIPPED (skip test_bar) * [[]100%[]]", + "*=== short test summary info ===*", + "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip foo subtest", + "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip bar subtest", + "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip test_bar", + "* 1 passed, 3 skipped in *", + ] + ) + + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar SKIPPED (skip test_bar) * [[]100%[]]", + "*=== short test summary info ===*", + "* 1 passed, 1 skipped in *", + ] + ) + result.stdout.no_fnmatch_line("*.py::test_foo SUBPASSED[[]foo subtest[]]*") + result.stdout.no_fnmatch_line("*.py::test_bar SUBPASSED[[]bar subtest[]]*") + result.stdout.no_fnmatch_line( + "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip foo subtest" + ) + result.stdout.no_fnmatch_line( + "SUBSKIPPED[[]foo subtest[]] [[]1[]] *.py:*: skip test_bar" + ) + + +def test_xfail(pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + import pytest + def test_foo(subtests): + with subtests.test("foo subtest"): + pytest.xfail("xfail foo subtest") + + def test_bar(subtests): + with subtests.test("bar subtest"): + pytest.xfail("xfail bar subtest") + pytest.xfail("xfail test_bar") + """ + ) + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "test_*.py .x * [[]100%[]]", + "*=== short test summary info ===*", + "* 1 passed, 1 xfailed in *", + ] + ) + + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo SUBXFAIL[[]foo subtest[]] (xfail foo subtest) * [[] 50%[]]", + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar SUBXFAIL[[]bar subtest[]] (xfail bar subtest) * [[]100%[]]", + "*.py::test_bar XFAIL (xfail test_bar) * [[]100%[]]", + "*=== short test summary info ===*", + "SUBXFAIL[[]foo subtest[]] *.py::test_foo - xfail foo subtest", + "SUBXFAIL[[]bar subtest[]] *.py::test_bar - xfail bar subtest", + "XFAIL *.py::test_bar - xfail test_bar", + "* 1 passed, 3 xfailed in *", + ] + ) + + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo PASSED * [[] 50%[]]", + "*.py::test_bar XFAIL (xfail test_bar) * [[]100%[]]", + "*=== short test summary info ===*", + "* 1 passed, 1 xfailed in *", + ] + ) + result.stdout.no_fnmatch_line( + "SUBXFAIL[[]foo subtest[]] *.py::test_foo - xfail foo subtest" + ) + result.stdout.no_fnmatch_line( + "SUBXFAIL[[]bar subtest[]] *.py::test_bar - xfail bar subtest" + ) + + +def test_typing_exported(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + from pytest import Subtests + + def test_typing_exported(subtests: Subtests) -> None: + assert isinstance(subtests, Subtests) + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines(["*1 passed*"]) + + +def test_subtests_and_parametrization( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + import pytest + + @pytest.mark.parametrize("x", [0, 1]) + def test_foo(subtests, x): + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + assert x == 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo[[]0[]] SUBFAILED[[]custom[]] (i=1) *[[] 50%[]]", + "*.py::test_foo[[]0[]] FAILED *[[] 50%[]]", + "*.py::test_foo[[]1[]] SUBFAILED[[]custom[]] (i=1) *[[]100%[]]", + "*.py::test_foo[[]1[]] FAILED *[[]100%[]]", + "contains 1 failed subtest", + "* 4 failed, 4 subtests passed in *", + ] + ) + + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*.py::test_foo[[]0[]] SUBFAILED[[]custom[]] (i=1) *[[] 50%[]]", + "*.py::test_foo[[]0[]] FAILED *[[] 50%[]]", + "*.py::test_foo[[]1[]] SUBFAILED[[]custom[]] (i=1) *[[]100%[]]", + "*.py::test_foo[[]1[]] FAILED *[[]100%[]]", + "contains 1 failed subtest", + "* 4 failed in *", + ] + ) + + +def test_subtests_fail_top_level_test(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def test_foo(subtests): + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "* 2 failed, 2 subtests passed in *", + ] + ) + + +def test_subtests_do_not_overwrite_top_level_failure(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import pytest + + def test_foo(subtests): + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + assert False, "top-level failure" + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "*AssertionError: top-level failure", + "* 2 failed, 2 subtests passed in *", + ] + ) + + +def test_msg_not_a_string( + pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch +) -> None: + """ + Using a non-string in subtests.test() should still show it in the terminal (#14195). + + Note: this was not a problem originally with the subtests fixture, only with TestCase.subTest; this test + was added for symmetry. + """ + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + def test_int_msg(subtests): + with subtests.test(42): + assert False, "subtest failure" + + def test_no_msg(subtests): + with subtests.test(): + assert False, "subtest failure" + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "SUBFAILED[[]42[]] test_msg_not_a_string.py::test_int_msg - AssertionError: subtest failure", + "SUBFAILED() test_msg_not_a_string.py::test_no_msg - AssertionError: subtest failure", + ] + ) + + +@pytest.mark.parametrize("flag", ["--last-failed", "--stepwise"]) +def test_subtests_last_failed_step_wise(pytester: pytest.Pytester, flag: str) -> None: + """Check that --last-failed and --step-wise correctly rerun tests with failed subtests.""" + pytester.makepyfile( + """ + import pytest + + def test_foo(subtests): + for i in range(3): + with subtests.test("custom", i=i): + assert i % 2 == 0 + """ + ) + result = pytester.runpytest("-v") + result.stdout.fnmatch_lines( + [ + "* 2 failed, 2 subtests passed in *", + ] + ) + + result = pytester.runpytest("-v", flag) + result.stdout.fnmatch_lines( + [ + "* 2 failed, 2 subtests passed in *", + ] + ) + + +class TestUnittestSubTest: + """Test unittest.TestCase.subTest functionality.""" + + def test_failures( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + from unittest import TestCase + + class T(TestCase): + def test_foo(self): + with self.subTest("foo subtest"): + assert False, "foo subtest failure" + + def test_bar(self): + with self.subTest("bar subtest"): + assert False, "bar subtest failure" + assert False, "test_bar also failed" + + def test_zaz(self): + with self.subTest("zaz subtest"): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "* 3 failed, 2 passed in *", + ] + ) + + def test_passes( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + from unittest import TestCase + + class T(TestCase): + def test_foo(self): + with self.subTest("foo subtest"): + pass + + def test_bar(self): + with self.subTest("bar subtest"): + pass + + def test_zaz(self): + with self.subTest("zaz subtest"): + pass + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "* 3 passed in *", + ] + ) + + def test_skip( + self, + pytester: pytest.Pytester, + ) -> None: + pytester.makepyfile( + """ + from unittest import TestCase, main + + class T(TestCase): + + def test_foo(self): + for i in range(5): + with self.subTest(msg="custom", i=i): + if i % 2 == 0: + self.skipTest('even number') + """ + ) + # This output might change #13756. + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 passed in *"]) + + def test_non_subtest_skip( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + from unittest import TestCase, main + + class T(TestCase): + + def test_foo(self): + with self.subTest(msg="subtest"): + assert False, "failed subtest" + self.skipTest('non-subtest skip') + """ + ) + # This output might change #13756. + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "SUBFAILED[[]subtest[]] test_non_subtest_skip.py::T::test_foo*", + "* 1 failed, 1 skipped in *", + ] + ) + + def test_xfail( + self, + pytester: pytest.Pytester, + ) -> None: + pytester.makepyfile( + """ + import pytest + from unittest import expectedFailure, TestCase + + class T(TestCase): + @expectedFailure + def test_foo(self): + for i in range(5): + with self.subTest(msg="custom", i=i): + if i % 2 == 0: + raise pytest.xfail('even number') + + if __name__ == '__main__': + main() + """ + ) + # This output might change #13756. + result = pytester.runpytest() + result.stdout.fnmatch_lines(["* 1 xfailed in *"]) + + def test_only_original_skip_is_called( + self, + pytester: pytest.Pytester, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Regression test for pytest-dev/pytest-subtests#173.""" + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + import unittest + from unittest import TestCase + + @unittest.skip("skip this test") + class T(unittest.TestCase): + def test_foo(self): + assert 1 == 2 + """ + ) + result = pytester.runpytest("-v", "-rsf") + result.stdout.fnmatch_lines( + ["SKIPPED [1] test_only_original_skip_is_called.py:6: skip this test"] + ) + + def test_skip_with_failure( + self, + pytester: pytest.Pytester, + monkeypatch: pytest.MonkeyPatch, + ) -> None: + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + import pytest + from unittest import TestCase + + class T(TestCase): + def test_foo(self): + with self.subTest("subtest 1"): + self.skipTest(f"skip subtest 1") + with self.subTest("subtest 2"): + assert False, "fail subtest 2" + """ + ) + + result = pytester.runpytest("-ra") + result.stdout.fnmatch_lines( + [ + "*.py u. * [[]100%[]]", + "*=== short test summary info ===*", + "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2", + "* 1 failed, 1 passed in *", + ] + ) + + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::T::test_foo SUBSKIPPED[[]subtest 1[]] (skip subtest 1) * [[]100%[]]", + "*.py::T::test_foo SUBFAILED[[]subtest 2[]] * [[]100%[]]", + "*.py::T::test_foo PASSED * [[]100%[]]", + "SUBSKIPPED[[]subtest 1[]] [[]1[]] *.py:*: skip subtest 1", + "SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2", + "* 1 failed, 1 passed, 1 skipped in *", + ] + ) + + pytester.makeini( + """ + [pytest] + verbosity_subtests = 0 + """ + ) + result = pytester.runpytest("-v", "-ra") + result.stdout.fnmatch_lines( + [ + "*.py::T::test_foo SUBFAILED[[]subtest 2[]] * [[]100%[]]", + "*.py::T::test_foo PASSED * [[]100%[]]", + "*=== short test summary info ===*", + r"SUBFAILED[[]subtest 2[]] *.py::T::test_foo - AssertionError: fail subtest 2", + r"* 1 failed, 1 passed in *", + ] + ) + result.stdout.no_fnmatch_line( + "*.py::T::test_foo SUBSKIPPED[[]subtest 1[]] (skip subtest 1) * [[]100%[]]" + ) + result.stdout.no_fnmatch_line( + "SUBSKIPPED[[]subtest 1[]] [[]1[]] *.py:*: skip subtest 1" + ) + + def test_msg_not_a_string( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Using a non-string in TestCase.subTest should still show it in the terminal (#14195).""" + monkeypatch.setenv("COLUMNS", "120") + pytester.makepyfile( + """ + from unittest import TestCase + + class T(TestCase): + def test_int_msg(self): + with self.subTest(42): + assert False, "subtest failure" + + def test_no_msg(self): + with self.subTest(): + assert False, "subtest failure" + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "SUBFAILED[[]42[]] test_msg_not_a_string.py::T::test_int_msg - AssertionError: subtest failure", + "SUBFAILED() test_msg_not_a_string.py::T::test_no_msg - AssertionError: subtest failure", + ] + ) + + +class TestCapture: + def create_file(self, pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import sys + def test(subtests): + print() + print('start test') + + with subtests.test(i='A'): + print("hello stdout A") + print("hello stderr A", file=sys.stderr) + assert 0 + + with subtests.test(i='B'): + print("hello stdout B") + print("hello stderr B", file=sys.stderr) + assert 0 + + print('end test') + assert 0 + """ + ) + + @pytest.mark.parametrize("mode", ["fd", "sys"]) + def test_capturing(self, pytester: pytest.Pytester, mode: str) -> None: + self.create_file(pytester) + result = pytester.runpytest(f"--capture={mode}") + result.stdout.fnmatch_lines( + [ + "*__ test (i='A') __*", + "*Captured stdout call*", + "hello stdout A", + "*Captured stderr call*", + "hello stderr A", + "*__ test (i='B') __*", + "*Captured stdout call*", + "hello stdout B", + "*Captured stderr call*", + "hello stderr B", + "*__ test __*", + "*Captured stdout call*", + "start test", + "end test", + ] + ) + + def test_no_capture(self, pytester: pytest.Pytester) -> None: + self.create_file(pytester) + result = pytester.runpytest("-s") + result.stdout.fnmatch_lines( + [ + "start test", + "hello stdout A", + "uhello stdout B", + "uend test", + "*__ test (i='A') __*", + "*__ test (i='B') __*", + "*__ test __*", + ] + ) + result.stderr.fnmatch_lines(["hello stderr A", "hello stderr B"]) + + @pytest.mark.parametrize("fixture", ["capsys", "capfd"]) + def test_capture_with_fixture( + self, pytester: pytest.Pytester, fixture: Literal["capsys", "capfd"] + ) -> None: + pytester.makepyfile( + rf""" + import sys + + def test(subtests, {fixture}): + print('start test') + + with subtests.test(i='A'): + print("hello stdout A") + print("hello stderr A", file=sys.stderr) + + out, err = {fixture}.readouterr() + assert out == 'start test\nhello stdout A\n' + assert err == 'hello stderr A\n' + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*1 passed*", + ] + ) + + +class TestLogging: + def create_file(self, pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import logging + + def test_foo(subtests): + logging.info("before") + + with subtests.test("sub1"): + print("sub1 stdout") + logging.info("sub1 logging") + logging.debug("sub1 logging debug") + + with subtests.test("sub2"): + print("sub2 stdout") + logging.info("sub2 logging") + logging.debug("sub2 logging debug") + assert False + """ + ) + + def test_capturing_info(self, pytester: pytest.Pytester) -> None: + self.create_file(pytester) + result = pytester.runpytest("--log-level=INFO") + result.stdout.fnmatch_lines( + [ + "*___ test_foo [[]sub2[]] __*", + "*-- Captured stdout call --*", + "sub2 stdout", + "*-- Captured log call ---*", + "INFO * before", + "INFO * sub1 logging", + "INFO * sub2 logging", + "*== short test summary info ==*", + ] + ) + result.stdout.no_fnmatch_line("sub1 logging debug") + result.stdout.no_fnmatch_line("sub2 logging debug") + + def test_capturing_debug(self, pytester: pytest.Pytester) -> None: + self.create_file(pytester) + result = pytester.runpytest("--log-level=DEBUG") + result.stdout.fnmatch_lines( + [ + "*___ test_foo [[]sub2[]] __*", + "*-- Captured stdout call --*", + "sub2 stdout", + "*-- Captured log call ---*", + "INFO * before", + "INFO * sub1 logging", + "DEBUG * sub1 logging debug", + "INFO * sub2 logging", + "DEBUG * sub2 logging debug", + "*== short test summary info ==*", + ] + ) + + def test_caplog(self, pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import logging + + def test(subtests, caplog): + caplog.set_level(logging.INFO) + logging.info("start test") + + with subtests.test("sub1"): + logging.info("inside %s", "subtest1") + + assert len(caplog.records) == 2 + assert caplog.records[0].getMessage() == "start test" + assert caplog.records[1].getMessage() == "inside subtest1" + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "*1 passed*", + ] + ) + + def test_no_logging(self, pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import logging + + def test(subtests): + logging.info("start log line") + + with subtests.test("sub passing"): + logging.info("inside %s", "passing log line") + + with subtests.test("sub failing"): + logging.info("inside %s", "failing log line") + assert False + + logging.info("end log line") + """ + ) + result = pytester.runpytest("-p no:logging") + result.stdout.fnmatch_lines( + [ + "*2 failed in*", + ] + ) + result.stdout.no_fnmatch_line("*root:test_no_logging.py*log line*") + + +class TestDebugging: + """Check --pdb support for subtests fixture and TestCase.subTest.""" + + class _FakePdb: + """Fake debugger class implementation that tracks which methods were called on it.""" + + quitting: bool = False + calls: list[str] = [] + + def __init__(self, *_: object, **__: object) -> None: + self.calls.append("init") + + def reset(self) -> None: + self.calls.append("reset") + + def interaction(self, *_: object) -> None: + self.calls.append("interaction") + + @pytest.fixture(autouse=True) + def cleanup_calls(self) -> None: + self._FakePdb.calls.clear() + + def test_pdb_fixture( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + pytester.makepyfile( + """ + def test(subtests): + with subtests.test(): + assert 0 + """ + ) + self.runpytest_and_check_pdb(pytester, monkeypatch) + + def test_pdb_unittest( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + pytester.makepyfile( + """ + from unittest import TestCase + class Test(TestCase): + def test(self): + with self.subTest(): + assert 0 + """ + ) + self.runpytest_and_check_pdb(pytester, monkeypatch) + + def runpytest_and_check_pdb( + self, pytester: pytest.Pytester, monkeypatch: pytest.MonkeyPatch + ) -> None: + # Install the fake pdb implementation in _pytest.subtests so we can reference + # it in the command line (any module would do). + import _pytest.subtests + + monkeypatch.setattr( + _pytest.subtests, "_CustomPdb", self._FakePdb, raising=False + ) + result = pytester.runpytest("--pdb", "--pdbcls=_pytest.subtests:_CustomPdb") + + # Ensure pytest entered in debugging mode when encountering the failing + # assert. + result.stdout.fnmatch_lines("*entering PDB*") + assert self._FakePdb.calls == ["init", "reset", "interaction"] + + +def test_exitfirst(pytester: pytest.Pytester) -> None: + """Validate that when passing --exitfirst the test exits after the first failed subtest.""" + pytester.makepyfile( + """ + def test_foo(subtests): + with subtests.test("sub1"): + assert False + + with subtests.test("sub2"): + assert False + """ + ) + result = pytester.runpytest("--exitfirst") + assert result.parseoutcomes()["failed"] == 2 + result.stdout.fnmatch_lines( + [ + "SUBFAILED*[[]sub1[]] *.py::test_foo - assert False*", + "FAILED *.py::test_foo - assert False", + "* stopping after 2 failures*", + ], + consecutive=True, + ) + result.stdout.no_fnmatch_line("*sub2*") # sub2 not executed. + + +def test_do_not_swallow_pytest_exit(pytester: pytest.Pytester) -> None: + pytester.makepyfile( + """ + import pytest + def test(subtests): + with subtests.test(): + pytest.exit() + + def test2(): pass + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "* _pytest.outcomes.Exit *", + "* 1 failed in *", + ] + ) + + +def test_nested(pytester: pytest.Pytester) -> None: + """ + Currently we do nothing special with nested subtests. + + This test only sediments how they work now, we might reconsider adding some kind of nesting support in the future. + """ + pytester.makepyfile( + """ + import pytest + def test(subtests): + with subtests.test("a"): + with subtests.test("b"): + assert False, "b failed" + assert False, "a failed" + """ + ) + result = pytester.runpytest_subprocess() + result.stdout.fnmatch_lines( + [ + "SUBFAILED[b] test_nested.py::test - AssertionError: b failed", + "SUBFAILED[a] test_nested.py::test - AssertionError: a failed", + "* 3 failed in *", + ] + ) + + +def test_serialization() -> None: + from _pytest.subtests import pytest_report_from_serializable + from _pytest.subtests import pytest_report_to_serializable + + report = SubtestReport( + "test_foo::test_foo", + ("test_foo.py", 12, ""), + keywords={}, + outcome="passed", + when="call", + longrepr=None, + context=SubtestContext(msg="custom message", kwargs=dict(i=10)), + ) + data = pytest_report_to_serializable(report) + assert data is not None + new_report = pytest_report_from_serializable(data) + assert new_report is not None + assert new_report.context == SubtestContext(msg="custom message", kwargs=dict(i=10)) diff --git a/testing/test_terminal.py b/testing/test_terminal.py index 11ad623fb6b..3053f5ef9a1 100644 --- a/testing/test_terminal.py +++ b/testing/test_terminal.py @@ -10,7 +10,9 @@ import textwrap from types import SimpleNamespace from typing import cast +from typing import Literal from typing import NamedTuple +from unittest import mock import pluggy @@ -30,6 +32,7 @@ from _pytest.terminal import _get_raw_skip_reason from _pytest.terminal import _plugin_nameversions from _pytest.terminal import getreportopt +from _pytest.terminal import TerminalProgressPlugin from _pytest.terminal import TerminalReporter import pytest @@ -49,7 +52,7 @@ def __init__(self, verbosity=0): @property def args(self): values = [] - values.append("--verbosity=%d" % self.verbosity) + values.append(f"--verbosity={self.verbosity}") return values @@ -112,6 +115,31 @@ def test_func(): [" def test_func():", "> assert 0", "E assert 0"] ) + def test_console_output_style_times_with_skipped_and_passed( + self, pytester: Pytester + ) -> None: + pytester.makepyfile( + test_repro=""" + def test_hello(): + pass + """, + test_repro_skip=""" + import pytest + pytest.importorskip("fakepackage_does_not_exist") + """, + ) + result = pytester.runpytest( + "test_repro.py", + "test_repro_skip.py", + "-o", + "console_output_style=times", + ) + + result.stdout.fnmatch_lines("* 1 passed, 1 skipped in *") + + combined = "\n".join(result.stdout.lines + result.stderr.lines) + assert "INTERNALERROR" not in combined + def test_internalerror(self, pytester: Pytester, linecomp) -> None: modcol = pytester.getmodulecol("def test_one(): pass") rep = TerminalReporter(modcol.config, file=linecomp.stringio) @@ -336,7 +364,7 @@ def test_report_teststatus_explicit_markup( pytester.makeconftest( f""" def pytest_report_teststatus(report): - return {category !r}, 'F', ('FOO', {{'red': True}}) + return {category!r}, 'F', ('FOO', {{'red': True}}) """ ) pytester.makepyfile( @@ -442,6 +470,16 @@ def test_long_xfail(): ] ) + @pytest.mark.parametrize("isatty", [True, False]) + def test_isatty(self, pytester: Pytester, monkeypatch, isatty: bool) -> None: + config = pytester.parseconfig() + f = StringIO() + monkeypatch.setattr(f, "isatty", lambda: isatty) + tr = TerminalReporter(config, f) + assert tr.isatty() == isatty + # It was incorrectly implemented as a boolean so we still support using it as one. + assert bool(tr.isatty) == isatty + class TestCollectonly: def test_collectonly_basic(self, pytester: Pytester) -> None: @@ -859,6 +897,7 @@ def test_header_trailer_info( self, monkeypatch: MonkeyPatch, pytester: Pytester, request ) -> None: monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD") + monkeypatch.delenv("PYTEST_PLUGINS", raising=False) pytester.makepyfile( """ def test_passes(): @@ -900,7 +939,7 @@ def test_header(self, pytester: Pytester) -> None: pytester.path.joinpath("tests").mkdir() pytester.path.joinpath("gui").mkdir() - # no ini file + # no configuration file result = pytester.runpytest() result.stdout.fnmatch_lines(["rootdir: *test_header0"]) @@ -1042,10 +1081,6 @@ def test_pass(): class TestClass(object): def test_skip(self): pytest.skip("hello") - def test_gen(): - def check(x): - assert x == 1 - yield check, 0 """ ) @@ -1058,7 +1093,6 @@ def test_verbose_reporting(self, verbose_testfile, pytester: Pytester) -> None: "*test_verbose_reporting.py::test_fail *FAIL*", "*test_verbose_reporting.py::test_pass *PASS*", "*test_verbose_reporting.py::TestClass::test_skip *SKIP*", - "*test_verbose_reporting.py::test_gen *XFAIL*", ] ) assert result.ret == 1 @@ -1192,7 +1226,7 @@ def test(param): @pytest.mark.parametrize( ("use_ci", "expected_message"), ( - (True, f"- AssertionError: {'this_failed'*100}"), + (True, f"- AssertionError: {'this_failed' * 100}"), (False, "- AssertionError: this_failedt..."), ), ids=("on CI", "not on CI"), @@ -1299,13 +1333,13 @@ def test_this(): "=*= FAILURES =*=", "{red}{bold}_*_ test_this _*_{reset}", "", - " {reset}{kw}def{hl-reset} {function}test_this{hl-reset}():{endline}", + " {reset}{kw}def{hl-reset}{kwspace}{function}test_this{hl-reset}():{endline}", "> fail(){endline}", "", "{bold}{red}test_color_yes.py{reset}:5: ", "_ _ * _ _*", "", - " {reset}{kw}def{hl-reset} {function}fail{hl-reset}():{endline}", + " {reset}{kw}def{hl-reset}{kwspace}{function}fail{hl-reset}():{endline}", "> {kw}assert{hl-reset} {number}0{hl-reset}{endline}", "{bold}{red}E assert 0{reset}", "", @@ -1552,6 +1586,19 @@ def test_func(): assert "--calling--" not in s assert "IndexError" not in s + def test_tb_line_show_capture(self, pytester: Pytester, option) -> None: + output_to_capture = "help! let me out!" + pytester.makepyfile( + f""" + import pytest + def test_fail(): + print('{output_to_capture}') + assert False + """ + ) + result = pytester.runpytest("--tb=line") + result.stdout.fnmatch_lines(["*- Captured stdout call -*", output_to_capture]) + def test_tb_crashline(self, pytester: Pytester, option) -> None: p = pytester.makepyfile( """ @@ -2078,6 +2125,21 @@ def test_foobar(i): pass """, ) + @pytest.fixture + def more_tests_files(self, pytester: Pytester) -> None: + pytester.makepyfile( + test_bar=""" + import pytest + @pytest.mark.parametrize('i', range(30)) + def test_bar(i): pass + """, + test_foo=""" + import pytest + @pytest.mark.parametrize('i', range(5)) + def test_foo(i): pass + """, + ) + def test_zero_tests_collected(self, pytester: Pytester) -> None: """Some plugins (testmon for example) might issue pytest_runtest_logreport without any tests being actually collected (#2971).""" @@ -2174,6 +2236,52 @@ def test_count(self, many_tests_files, pytester: Pytester) -> None: ] ) + def test_times(self, many_tests_files, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style = times + """ + ) + output = pytester.runpytest() + output.stdout.re_match_lines( + [ + r"test_bar.py \.{10} \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + r"test_foo.py \.{5} \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + r"test_foobar.py \.{5} \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + ] + ) + + def test_times_multiline( + self, more_tests_files, monkeypatch, pytester: Pytester + ) -> None: + monkeypatch.setenv("COLUMNS", "40") + pytester.makeini( + """ + [pytest] + console_output_style = times + """ + ) + output = pytester.runpytest() + output.stdout.re_match_lines( + [ + r"test_bar.py ...................", + r"........... \s+ \d{1,4}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + r"test_foo.py \.{5} \s+ \d{1,4}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + ], + consecutive=True, + ) + + def test_times_none_collected(self, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style = times + """ + ) + output = pytester.runpytest() + assert output.ret == ExitCode.NO_TESTS_COLLECTED + def test_verbose(self, many_tests_files, pytester: Pytester) -> None: output = pytester.runpytest("-v") output.stdout.re_match_lines( @@ -2200,6 +2308,22 @@ def test_verbose_count(self, many_tests_files, pytester: Pytester) -> None: ] ) + def test_verbose_times(self, many_tests_files, pytester: Pytester) -> None: + pytester.makeini( + """ + [pytest] + console_output_style = times + """ + ) + output = pytester.runpytest("-v") + output.stdout.re_match_lines( + [ + r"test_bar.py::test_bar\[0\] PASSED \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + r"test_foo.py::test_foo\[4\] PASSED \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + r"test_foobar.py::test_foobar\[4\] PASSED \s+ \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2}$", + ] + ) + def test_xdist_normal( self, many_tests_files, pytester: Pytester, monkeypatch ) -> None: @@ -2252,6 +2376,26 @@ def test_xdist_verbose( ] ) + def test_xdist_times( + self, many_tests_files, pytester: Pytester, monkeypatch + ) -> None: + pytest.importorskip("xdist") + monkeypatch.delenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", raising=False) + pytester.makeini( + """ + [pytest] + console_output_style = times + """ + ) + output = pytester.runpytest("-n2", "-v") + output.stdout.re_match_lines_random( + [ + r"\[gw\d\] \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2} PASSED test_bar.py::test_bar\[1\]", + r"\[gw\d\] \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2} PASSED test_foo.py::test_foo\[1\]", + r"\[gw\d\] \d{1,3}[\.[a-z\ ]{1,2}\d{0,3}\w{1,2} PASSED test_foobar.py::test_foobar\[1\]", + ] + ) + def test_capture_no(self, many_tests_files, pytester: Pytester) -> None: output = pytester.runpytest("-s") output.stdout.re_match_lines( @@ -2525,6 +2669,52 @@ def test(): ) +def test_full_sequence_print_with_vv( + monkeypatch: MonkeyPatch, pytester: Pytester +) -> None: + """Do not truncate sequences in summaries with -vv (#11777).""" + monkeypatch.setattr(_pytest.terminal, "running_on_ci", lambda: False) + + pytester.makepyfile( + """ + def test_len_list(): + l = list(range(10)) + assert len(l) == 9 + + def test_len_dict(): + d = dict(zip(range(10), range(10))) + assert len(d) == 9 + """ + ) + + result = pytester.runpytest("-vv") + assert result.ret == 1 + result.stdout.fnmatch_lines( + [ + "*short test summary info*", + f"*{list(range(10))}*", + f"*{dict(zip(range(10), range(10), strict=True))}*", + ] + ) + + +def test_force_short_summary(monkeypatch: MonkeyPatch, pytester: Pytester) -> None: + monkeypatch.setattr(_pytest.terminal, "running_on_ci", lambda: False) + + pytester.makepyfile( + """ + def test(): + assert "a\\n" * 10 == "" + """ + ) + + result = pytester.runpytest("-vv", "--force-short-summary") + assert result.ret == 1 + result.stdout.fnmatch_lines( + ["*short test summary info*", "*AssertionError: assert 'a\\na\\na\\na..."] + ) + + @pytest.mark.parametrize( "seconds, expected", [ @@ -2542,6 +2732,27 @@ def test_format_session_duration(seconds, expected): assert format_session_duration(seconds) == expected +@pytest.mark.parametrize( + "seconds, expected", + [ + (3600 * 100 - 60, " 99h 59m"), + (31 * 60 - 1, " 30m 59s"), + (10.1236, " 10.124s"), + (9.1236, " 9.124s"), + (0.1236, " 123.6ms"), + (0.01236, " 12.36ms"), + (0.001236, " 1.236ms"), + (0.0001236, " 123.6us"), + (0.00001236, " 12.36us"), + (0.000001236, " 1.236us"), + ], +) +def test_format_node_duration(seconds: float, expected: str) -> None: + from _pytest.terminal import format_node_duration + + assert format_node_duration(seconds) == expected + + def test_collecterror(pytester: Pytester) -> None: p1 = pytester.makepyfile("raise SyntaxError()") result = pytester.runpytest("-ra", str(p1)) @@ -2585,7 +2796,7 @@ def test_foo(): result.stdout.fnmatch_lines( color_mapping.format_for_fnmatch( [ - " {reset}{kw}def{hl-reset} {function}test_foo{hl-reset}():{endline}", + " {reset}{kw}def{hl-reset}{kwspace}{function}test_foo{hl-reset}():{endline}", "> {kw}assert{hl-reset} {number}1{hl-reset} == {number}10{hl-reset}{endline}", "{bold}{red}E assert 1 == 10{reset}", ] @@ -2607,7 +2818,7 @@ def test_foo(): result.stdout.fnmatch_lines( color_mapping.format_for_fnmatch( [ - " {reset}{kw}def{hl-reset} {function}test_foo{hl-reset}():{endline}", + " {reset}{kw}def{hl-reset}{kwspace}{function}test_foo{hl-reset}():{endline}", " {print}print{hl-reset}({str}'''{hl-reset}{str}{hl-reset}", "> {str} {hl-reset}{str}'''{hl-reset}); {kw}assert{hl-reset} {number}0{hl-reset}{endline}", "{bold}{red}E assert 0{reset}", @@ -2630,7 +2841,7 @@ def test_foo(): result.stdout.fnmatch_lines( color_mapping.format_for_fnmatch( [ - " {reset}{kw}def{hl-reset} {function}test_foo{hl-reset}():{endline}", + " {reset}{kw}def{hl-reset}{kwspace}{function}test_foo{hl-reset}():{endline}", "> {kw}assert{hl-reset} {number}1{hl-reset} == {number}10{hl-reset}{endline}", "{bold}{red}E assert 1 == 10{reset}", ] @@ -2694,6 +2905,100 @@ def test_format_trimmed() -> None: assert _format_trimmed(" ({}) ", msg, len(msg) + 3) == " (unconditional ...) " +def test_warning_when_init_trumps_pyproject_toml( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Regression test for #7814.""" + tests = pytester.path.joinpath("tests") + tests.mkdir() + pytester.makepyprojecttoml( + f""" + [tool.pytest.ini_options] + testpaths = ['{tests}'] + """ + ) + pytester.makefile(".ini", pytest="") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "configfile: pytest.ini (WARNING: ignoring pytest config in pyproject.toml!)", + ] + ) + + +def test_warning_when_init_trumps_multiple_files( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Regression test for #7814.""" + tests = pytester.path.joinpath("tests") + tests.mkdir() + pytester.makepyprojecttoml( + f""" + [tool.pytest.ini_options] + testpaths = ['{tests}'] + """ + ) + pytester.makefile(".ini", pytest="") + pytester.makeini( + """ + # tox.ini + [pytest] + minversion = 6.0 + addopts = -ra -q + testpaths = + tests + integration + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "configfile: pytest.ini (WARNING: ignoring pytest config in pyproject.toml, tox.ini!)", + ] + ) + + +def test_no_warning_when_init_but_pyproject_toml_has_no_entry( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Regression test for #7814.""" + tests = pytester.path.joinpath("tests") + tests.mkdir() + pytester.makepyprojecttoml( + f""" + [tool] + testpaths = ['{tests}'] + """ + ) + pytester.makefile(".ini", pytest="") + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "configfile: pytest.ini", + ] + ) + + +def test_no_warning_on_terminal_with_a_single_config_file( + pytester: Pytester, monkeypatch: MonkeyPatch +) -> None: + """Regression test for #7814.""" + tests = pytester.path.joinpath("tests") + tests.mkdir() + pytester.makepyprojecttoml( + f""" + [tool.pytest.ini_options] + testpaths = ['{tests}'] + """ + ) + result = pytester.runpytest() + result.stdout.fnmatch_lines( + [ + "configfile: pyproject.toml", + ] + ) + + class TestFineGrainedTestCase: DEFAULT_FILE_CONTENTS = """ import pytest @@ -3067,3 +3372,179 @@ def test_pass(): "*= 1 xpassed in * =*", ] ) + + +class TestNodeIDHandling: + def test_nodeid_handling_windows_paths(self, pytester: Pytester, tmp_path) -> None: + """Test the correct handling of Windows-style paths with backslashes.""" + pytester.makeini("[pytest]") # Change `config.rootpath` + + test_path = pytester.path / "tests" / "test_foo.py" + test_path.parent.mkdir() + os.chdir(test_path.parent) # Change `config.invocation_params.dir` + + test_path.write_text( + textwrap.dedent( + """ + import pytest + + @pytest.mark.parametrize("a", ["x/y", "C:/path", "\\\\", "C:\\\\path", "a::b/"]) + def test_x(a): + assert False + """ + ), + encoding="utf-8", + ) + + result = pytester.runpytest("-v") + + result.stdout.re_match_lines( + [ + r".*test_foo.py::test_x\[x/y\] .*FAILED.*", + r".*test_foo.py::test_x\[C:/path\] .*FAILED.*", + r".*test_foo.py::test_x\[\\\\\] .*FAILED.*", + r".*test_foo.py::test_x\[C:\\\\path\] .*FAILED.*", + r".*test_foo.py::test_x\[a::b/\] .*FAILED.*", + ] + ) + + +class TestTerminalProgressPlugin: + """Tests for the TerminalProgressPlugin.""" + + @pytest.fixture + def mock_file(self) -> StringIO: + return StringIO() + + @pytest.fixture + def mock_tr(self, mock_file: StringIO) -> pytest.TerminalReporter: + tr: pytest.TerminalReporter = mock.create_autospec(pytest.TerminalReporter) + + def write_raw(content: str, *, flush: bool = False) -> None: + mock_file.write(content) + + tr.write_raw = write_raw # type: ignore[method-assign] + tr._progress_nodeids_reported = set() + return tr + + @pytest.mark.skipif(sys.platform != "win32", reason="#13896") + def test_plugin_registration_enabled_by_default( + self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Test that the plugin registration is enabled by default. + + Currently only on Windows (#13896). + """ + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + # The plugin module should be registered as a default plugin. + config = pytester.parseconfigure() + plugin = config.pluginmanager.get_plugin("terminalprogress") + assert plugin is not None + + def test_plugin_registred_on_all_platforms_when_explicitly_requested( + self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Test that the plugin is registered on any platform if explicitly requested.""" + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + # The plugin module should be registered as a default plugin. + config = pytester.parseconfigure("-p", "terminalprogress") + plugin = config.pluginmanager.get_plugin("terminalprogress") + assert plugin is not None + + def test_disabled_for_non_tty( + self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Test that plugin is disabled for non-TTY output.""" + monkeypatch.setattr(sys.stdout, "isatty", lambda: False) + config = pytester.parseconfigure("-p", "terminalprogress") + plugin = config.pluginmanager.get_plugin("terminalprogress-plugin") + assert plugin is None + + def test_disabled_for_dumb_terminal( + self, pytester: pytest.Pytester, monkeypatch: MonkeyPatch + ) -> None: + """Test that plugin is disabled when TERM=dumb.""" + monkeypatch.setenv("TERM", "dumb") + monkeypatch.setattr(sys.stdout, "isatty", lambda: True) + config = pytester.parseconfigure("-p", "terminalprogress") + plugin = config.pluginmanager.get_plugin("terminalprogress-plugin") + assert plugin is None + + @pytest.mark.parametrize( + ["state", "progress", "expected"], + [ + ("indeterminate", None, "\x1b]9;4;3;\x1b\\"), + ("normal", 50, "\x1b]9;4;1;50\x1b\\"), + ("error", 75, "\x1b]9;4;2;75\x1b\\"), + ("paused", None, "\x1b]9;4;4;\x1b\\"), + ("paused", 80, "\x1b]9;4;4;80\x1b\\"), + ("remove", None, "\x1b]9;4;0;\x1b\\"), + ], + ) + def test_emit_progress_sequences( + self, + mock_file: StringIO, + mock_tr: pytest.TerminalReporter, + state: Literal["remove", "normal", "error", "indeterminate", "paused"], + progress: int | None, + expected: str, + ) -> None: + """Test that progress sequences are emitted correctly.""" + plugin = TerminalProgressPlugin(mock_tr) + plugin._emit_progress(state, progress) + assert expected in mock_file.getvalue() + + def test_session_lifecycle( + self, mock_file: StringIO, mock_tr: pytest.TerminalReporter + ) -> None: + """Test progress updates during session lifecycle.""" + plugin = TerminalProgressPlugin(mock_tr) + + session = mock.create_autospec(pytest.Session) + session.testscollected = 3 + + # Session start - should emit indeterminate progress. + plugin.pytest_sessionstart(session) + assert "\x1b]9;4;3;\x1b\\" in mock_file.getvalue() + mock_file.truncate(0) + mock_file.seek(0) + + # Collection finish - should emit 0% progress. + plugin.pytest_collection_finish() + assert "\x1b]9;4;1;0\x1b\\" in mock_file.getvalue() + mock_file.truncate(0) + mock_file.seek(0) + + # First test - 33% progress. + report1 = pytest.TestReport( + nodeid="test_1", + location=("test.py", 0, "test_1"), + when="call", + outcome="passed", + keywords={}, + longrepr=None, + ) + mock_tr.reported_progress = 1 # type: ignore[misc] + plugin.pytest_runtest_logreport(report1) + assert "\x1b]9;4;1;33\x1b\\" in mock_file.getvalue() + mock_file.truncate(0) + mock_file.seek(0) + + # Second test with failure - 66% in error state. + report2 = pytest.TestReport( + nodeid="test_2", + location=("test.py", 1, "test_2"), + when="call", + outcome="failed", + keywords={}, + longrepr=None, + ) + mock_tr.reported_progress = 2 # type: ignore[misc] + plugin.pytest_runtest_logreport(report2) + assert "\x1b]9;4;2;66\x1b\\" in mock_file.getvalue() + mock_file.truncate(0) + mock_file.seek(0) + + # Session finish - should remove progress. + plugin.pytest_sessionfinish() + assert "\x1b]9;4;0;\x1b\\" in mock_file.getvalue() diff --git a/testing/test_threadexception.py b/testing/test_threadexception.py index abd30144914..f4595ec435d 100644 --- a/testing/test_threadexception.py +++ b/testing/test_threadexception.py @@ -23,7 +23,7 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", @@ -59,7 +59,7 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", @@ -96,7 +96,7 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", @@ -130,4 +130,126 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == pytest.ExitCode.TESTS_FAILED - assert result.parseoutcomes() == {"passed": 1, "failed": 1} + result.assert_outcomes(passed=1, failed=1) + + +@pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning") +def test_threadexception_warning_multiple_errors(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + + def test_it(): + def oops(): + raise ValueError("Oops") + + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + t = threading.Thread(target=oops, name="MyThread2") + t.start() + t.join() + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.assert_outcomes(passed=1, failed=1) + result.stdout.fnmatch_lines( + [" | *ExceptionGroup: multiple thread exception warnings (2 sub-exceptions)"] + ) + + +def test_unraisable_collection_failure(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + + class Thread(threading.Thread): + @property + def name(self): + raise RuntimeError("oops!") + + def test_it(): + def oops(): + raise ValueError("Oops") + + t = Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_2(): pass + """ + ) + + result = pytester.runpytest() + assert result.ret == 1 + result.assert_outcomes(passed=1, failed=1) + result.stdout.fnmatch_lines( + ["E RuntimeError: Failed to process thread exception"] + ) + + +def test_unhandled_thread_exception_after_teardown(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + import pytest + + def thread(): + def oops(): + raise ValueError("Oops") + + t = threading.Thread(target=oops, name="MyThread") + t.start() + t.join() + + def test_it(request): + request.config.add_cleanup(thread) + """ + ) + + result = pytester.runpytest("-Werror") + + # TODO: should be a test failure or error + assert result.ret == pytest.ExitCode.INTERNAL_ERROR + + result.assert_outcomes(passed=1) + result.stderr.fnmatch_lines("ValueError: Oops") + + +@pytest.mark.filterwarnings("error::pytest.PytestUnhandledThreadExceptionWarning") +def test_possibly_none_excinfo(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import threading + import types + + def test_it(): + threading.excepthook( + types.SimpleNamespace( + exc_type=RuntimeError, + exc_value=None, + exc_traceback=None, + thread=None, + ) + ) + """ + ) + + result = pytester.runpytest() + + # TODO: should be a test failure or error + assert result.ret == pytest.ExitCode.TESTS_FAILED + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "E pytest.PytestUnhandledThreadExceptionWarning:" + " Exception in thread ", + "E ", + "E NoneType: None", + ] + ) diff --git a/testing/test_tmpdir.py b/testing/test_tmpdir.py index 865d8e0b05c..789e8005184 100644 --- a/testing/test_tmpdir.py +++ b/testing/test_tmpdir.py @@ -1,12 +1,13 @@ # mypy: allow-untyped-defs from __future__ import annotations +from collections.abc import Callable import dataclasses import os from pathlib import Path +import shutil import stat import sys -from typing import Callable from typing import cast import warnings @@ -386,7 +387,7 @@ def test_cleanup_lock_create(self, tmp_path): d = tmp_path.joinpath("test") d.mkdir() lockfile = create_cleanup_lock(d) - with pytest.raises(OSError, match="cannot create lockfile in .*"): + with pytest.raises(OSError, match=r"cannot create lockfile in .*"): create_cleanup_lock(d) lockfile.unlink() @@ -619,3 +620,33 @@ def test_tmp_path_factory_fixes_up_world_readable_permissions( # After - fixed. assert (basetemp.parent.stat().st_mode & 0o077) == 0 + + +@pytest.mark.skipif( + not hasattr(os, "getuid") or os.stat not in os.supports_follow_symlinks, + reason="checks unix permissions and symlinks", +) +def test_tmp_path_factory_doesnt_follow_symlinks( + tmp_path: Path, monkeypatch: MonkeyPatch +) -> None: + """Verify that if a /tmp/pytest-of-foo directory is a symbolic link, + it is rejected (#13669, CVE-2025-71176).""" + attacker_controlled = tmp_path / "attacker_controlled" + attacker_controlled.mkdir() + + # Use the test's tmp_path as the system temproot (/tmp). + monkeypatch.setenv("PYTEST_DEBUG_TEMPROOT", str(tmp_path)) + + # First just get the pytest-of-user path. + tmp_factory = TempPathFactory(None, 3, "all", lambda *args: None, _ispytest=True) + pytest_of_user = tmp_factory.getbasetemp().parent + # Just for safety in the test, before we nuke it. + assert "pytest-of-" in str(pytest_of_user) + shutil.rmtree(pytest_of_user) + + pytest_of_user.symlink_to(attacker_controlled) + + # This now tries to use the directory when it's a symlink. + tmp_factory = TempPathFactory(None, 3, "all", lambda *args: None, _ispytest=True) + with pytest.raises(OSError, match=r"temporary directory .* is a symbolic link"): + tmp_factory.getbasetemp() diff --git a/testing/test_unittest.py b/testing/test_unittest.py index 56224c08228..395c9fe647e 100644 --- a/testing/test_unittest.py +++ b/testing/test_unittest.py @@ -1094,6 +1094,49 @@ def test_two(self): result.assert_outcomes(passed=2) +def test_skip_setup_class(pytester: Pytester) -> None: + """Skipping tests in a class by raising unittest.SkipTest in `setUpClass` (#13985).""" + pytester.makepyfile( + """ + import unittest + + class Test(unittest.TestCase): + + @classmethod + def setUpClass(cls): + raise unittest.SkipTest('Skipping setupclass') + + def test_foo(self): + assert False + + def test_bar(self): + assert False + """ + ) + result = pytester.runpytest() + result.assert_outcomes(skipped=2) + + +def test_unittest_skip_function(pytester: Pytester) -> None: + """ + Ensure raising an explicit unittest.SkipTest skips standard pytest functions. + + Support for this is debatable -- technically we only support unittest.SkipTest in TestCase subclasses, + but stating this support here in this test because users currently expect this to work, + so if we ever break it we at least know we are breaking this use case (#13985). + """ + pytester.makepyfile( + """ + import unittest + + def test_foo(): + raise unittest.SkipTest('Skipping test_foo') + """ + ) + result = pytester.runpytest() + result.assert_outcomes(skipped=1) + + def test_testcase_handles_init_exceptions(pytester: Pytester) -> None: """ Regression test to make sure exceptions in the __init__ method are bubbled up correctly. @@ -1322,10 +1365,12 @@ def test_async_support(pytester: Pytester) -> None: reprec.assertoutcome(failed=1, passed=2) +@pytest.mark.skipif( + sys.version_info >= (3, 11), reason="asynctest is not compatible with Python 3.11+" +) def test_asynctest_support(pytester: Pytester) -> None: """Check asynctest support (#7110)""" pytest.importorskip("asynctest") - pytester.copy_example("unittest/test_unittest_asynctest.py") reprec = pytester.inline_run() reprec.assertoutcome(failed=1, passed=2) @@ -1374,7 +1419,7 @@ def test_cleanup_called_exactly_once(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert failed == 0 assert passed == 3 @@ -1398,7 +1443,7 @@ def test_cleanup_called_exactly_once(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert failed == 1 assert passed == 1 @@ -1426,7 +1471,7 @@ def test_cleanup_called_exactly_once(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, _failed = reprec.countoutcomes() assert passed == 3 @@ -1449,7 +1494,7 @@ def test_cleanup_called_the_right_number_of_times(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert failed == 0 assert passed == 3 @@ -1474,7 +1519,7 @@ def test_cleanup_called_the_right_number_of_times(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert failed == 2 assert passed == 1 @@ -1500,7 +1545,7 @@ def test_cleanup_called_the_right_number_of_times(): """ ) reprec = pytester.inline_run(testpath) - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert failed == 2 assert passed == 1 @@ -1614,7 +1659,7 @@ def test_it(self): """ ) reprec = pytester.inline_run() - passed, skipped, failed = reprec.countoutcomes() + passed, _skipped, failed = reprec.countoutcomes() assert passed == 1 assert failed == 1 assert reprec.ret == 1 diff --git a/testing/test_unraisableexception.py b/testing/test_unraisableexception.py index a15c754d067..a6a4d6f35e8 100644 --- a/testing/test_unraisableexception.py +++ b/testing/test_unraisableexception.py @@ -1,6 +1,8 @@ from __future__ import annotations +import gc import sys +from unittest import mock from _pytest.pytester import Pytester import pytest @@ -8,6 +10,24 @@ PYPY = hasattr(sys, "pypy_version_info") +UNRAISABLE_LINE = ( + ( + " * PytestUnraisableExceptionWarning: Exception ignored while calling " + "deallocator : None" + ) + if sys.version_info >= (3, 14) + else " * PytestUnraisableExceptionWarning: Exception ignored in: " +) + +TRACEMALLOC_LINES = ( + () + if sys.version_info >= (3, 14) + else ( + " Enable tracemalloc to get traceback where the object was allocated.", + " See https* for more info.", + ) +) + @pytest.mark.skipif(PYPY, reason="garbage-collection differences make this flaky") @pytest.mark.filterwarnings("default::pytest.PytestUnraisableExceptionWarning") @@ -27,16 +47,17 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", "test_it.py::test_it", - " * PytestUnraisableExceptionWarning: Exception ignored in: ", + UNRAISABLE_LINE, " ", " Traceback (most recent call last):", " ValueError: del is broken", " ", + *TRACEMALLOC_LINES, " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", ] ) @@ -64,16 +85,17 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", "test_it.py::test_it", - " * PytestUnraisableExceptionWarning: Exception ignored in: ", + UNRAISABLE_LINE, " ", " Traceback (most recent call last):", " ValueError: del is broken", " ", + *TRACEMALLOC_LINES, " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", ] ) @@ -102,16 +124,17 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == 0 - assert result.parseoutcomes() == {"passed": 2, "warnings": 1} + result.assert_outcomes(passed=2, warnings=1) result.stdout.fnmatch_lines( [ "*= warnings summary =*", "test_it.py::test_it", - " * PytestUnraisableExceptionWarning: Exception ignored in: ", + UNRAISABLE_LINE, " ", " Traceback (most recent call last):", " ValueError: del is broken", " ", + *TRACEMALLOC_LINES, " warnings.warn(pytest.PytestUnraisableExceptionWarning(msg))", ] ) @@ -135,4 +158,238 @@ def test_2(): pass ) result = pytester.runpytest() assert result.ret == pytest.ExitCode.TESTS_FAILED - assert result.parseoutcomes() == {"passed": 1, "failed": 1} + result.assert_outcomes(passed=1, failed=1) + + +@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") +def test_unraisable_warning_multiple_errors(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=f""" + class BrokenDel: + def __init__(self, msg: str): + self.msg = msg + + def __del__(self) -> None: + raise ValueError(self.msg) + + def test_it() -> None: + BrokenDel("del is broken 1") + BrokenDel("del is broken 2") + {"import gc; gc.collect()" * PYPY} + + def test_2(): pass + """ + ) + result = pytester.runpytest() + assert result.ret == pytest.ExitCode.TESTS_FAILED + result.assert_outcomes(passed=1, failed=1) + result.stdout.fnmatch_lines( + [ + " | *ExceptionGroup: multiple unraisable exception warnings (2 sub-exceptions)" + ] + ) + + +def test_unraisable_collection_failure(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=f""" + class BrokenDel: + def __del__(self): + raise ValueError("del is broken") + + def test_it(): + obj = BrokenDel() + del obj + {"import gc; gc.collect()" * PYPY} + + def test_2(): pass + """ + ) + + class MyError(BaseException): + pass + + with mock.patch("traceback.format_exception", side_effect=MyError): + result = pytester.runpytest() + assert result.ret == 1 + result.assert_outcomes(passed=1, failed=1) + result.stdout.fnmatch_lines( + ["E RuntimeError: Failed to process unraisable exception"] + ) + + +def _set_gc_state(enabled: bool) -> bool: + was_enabled = gc.isenabled() + if enabled: + gc.enable() + else: + gc.disable() + return was_enabled + + +def test_refcycle_unraisable(pytester: Pytester) -> None: + # see: https://github.com/pytest-dev/pytest/issues/10404 + pytester.makepyfile( + test_it=""" + # Should catch the unraisable exception even if gc is disabled. + import gc; gc.disable() + + import pytest + + class BrokenDel: + def __init__(self): + self.self = self # make a reference cycle + + def __del__(self): + raise ValueError("del is broken") + + def test_it(): + BrokenDel() + """ + ) + + result = pytester.runpytest_subprocess( + "-Wdefault::pytest.PytestUnraisableExceptionWarning" + ) + + assert result.ret == 0 + + result.assert_outcomes(passed=1) + result.stderr.fnmatch_lines("ValueError: del is broken") + + +def test_refcycle_unraisable_warning_filter(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + # Should catch the unraisable exception even if gc is disabled. + import gc; gc.disable() + + import pytest + + class BrokenDel: + def __init__(self): + self.self = self # make a reference cycle + + def __del__(self): + raise ValueError("del is broken") + + def test_it(): + BrokenDel() + """ + ) + + result = pytester.runpytest_subprocess( + "-Werror::pytest.PytestUnraisableExceptionWarning" + ) + + # TODO: Should be a test failure or error. Currently the exception + # propagates all the way to the top resulting in exit code 1. + assert result.ret == 1 + + result.assert_outcomes(passed=1) + result.stderr.fnmatch_lines("ValueError: del is broken") + + +def test_create_task_raises_unraisable_warning_filter(pytester: Pytester) -> None: + # note that the host pytest warning filter is disabled and the pytester + # warning filter applies during config teardown of unraisablehook. + # see: https://github.com/pytest-dev/pytest/issues/10404 + # This is a dupe of the above test, but using the exact reproducer from + # the issue + pytester.makepyfile( + test_it=""" + # Should catch the unraisable exception even if gc is disabled. + import gc; gc.disable() + + import asyncio + import pytest + + async def my_task(): + pass + + def test_scheduler_must_be_created_within_running_loop() -> None: + with pytest.raises(RuntimeError) as _: + asyncio.create_task(my_task()) + """ + ) + + result = pytester.runpytest_subprocess("-Werror") + + # TODO: Should be a test failure or error. Currently the exception + # propagates all the way to the top resulting in exit code 1. + assert result.ret == 1 + + result.assert_outcomes(passed=1) + result.stderr.fnmatch_lines("RuntimeWarning: coroutine 'my_task' was never awaited") + + +def test_refcycle_unraisable_warning_filter_default(pytester: Pytester) -> None: + # note this time we use a default warning filter for pytester + # and run it in a subprocess, because the warning can only go to the + # sys.stdout rather than the terminal reporter, which has already + # finished. + # see: https://github.com/pytest-dev/pytest/pull/13057#discussion_r1888396126 + pytester.makepyfile( + test_it=""" + import gc + gc.disable() + + import pytest + + class BrokenDel: + def __init__(self): + self.self = self # make a reference cycle + + def __del__(self): + raise ValueError("del is broken") + + def test_it(): + BrokenDel() + """ + ) + + # since we use subprocess we need to disable gc inside test_it + result = pytester.runpytest_subprocess("-Wdefault") + + assert result.ret == pytest.ExitCode.OK + + # TODO: should be warnings=1, but the outcome has already come out + # by the time the warning triggers + result.assert_outcomes(passed=1) + result.stderr.fnmatch_lines("ValueError: del is broken") + + +@pytest.mark.filterwarnings("error::pytest.PytestUnraisableExceptionWarning") +def test_possibly_none_excinfo(pytester: Pytester) -> None: + pytester.makepyfile( + test_it=""" + import sys + import types + + def test_it(): + sys.unraisablehook( + types.SimpleNamespace( + exc_type=RuntimeError, + exc_value=None, + exc_traceback=None, + err_msg=None, + object=None, + ) + ) + """ + ) + + result = pytester.runpytest() + + # TODO: should be a test failure or error + assert result.ret == pytest.ExitCode.TESTS_FAILED + + result.assert_outcomes(failed=1) + result.stdout.fnmatch_lines( + [ + "E pytest.PytestUnraisableExceptionWarning:" + " Exception ignored in: None", + "E ", + "E NoneType: None", + ] + ) diff --git a/testing/test_warning_types.py b/testing/test_warning_types.py index 19fe0f8a272..81d8785733c 100644 --- a/testing/test_warning_types.py +++ b/testing/test_warning_types.py @@ -43,7 +43,7 @@ def test(): @pytest.mark.filterwarnings("error") def test_warn_explicit_for_annotates_errors_with_location(): - with pytest.raises(Warning, match="(?m)test\n at .*python_api.py:\\d+"): + with pytest.raises(Warning, match=r"(?m)test\n at .*raises.py:\d+"): warning_types.warn_explicit_for( pytest.raises, # type: ignore[arg-type] warning_types.PytestWarning("test"), diff --git a/testing/test_warnings.py b/testing/test_warnings.py index d4d0e0b7f93..e3221da7569 100644 --- a/testing/test_warnings.py +++ b/testing/test_warnings.py @@ -149,6 +149,7 @@ def test_func(fix): ) +@pytest.mark.skip("issue #13485") def test_works_with_filterwarnings(pytester: Pytester) -> None: """Ensure our warnings capture does not mess with pre-installed filters (#2430).""" pytester.makepyfile( @@ -279,8 +280,7 @@ def pytest_warning_recorded(self, warning_message, when, nodeid, location): ("call warning", "runtest", "test_warning_recorded_hook.py::test_func"), ("teardown warning", "runtest", "test_warning_recorded_hook.py::test_func"), ] - assert len(collected) == len(expected) # python < 3.10 zip(strict=True) - for collected_result, expected_result in zip(collected, expected): + for collected_result, expected_result in zip(collected, expected, strict=True): assert collected_result[0] == expected_result[0], str(collected) assert collected_result[1] == expected_result[1], str(collected) assert collected_result[2] == expected_result[2], str(collected) @@ -382,7 +382,7 @@ def test_bar(): def test_option_precedence_cmdline_over_ini( pytester: Pytester, ignore_on_cmdline ) -> None: - """Filters defined in the command-line should take precedence over filters in ini files (#3946).""" + """Filters defined in the command-line should take precedence over filters in config files (#3946).""" pytester.makeini( """ [pytest] @@ -424,6 +424,33 @@ def test(): result.stdout.fnmatch_lines(["* 1 failed in*"]) +def test_accept_unknown_category(pytester: Pytester) -> None: + """Category types that can't be imported don't cause failure (#13732).""" + pytester.makeini( + """ + [pytest] + filterwarnings = + always:Failed to import filter module.*:pytest.PytestConfigWarning + ignore::foobar.Foobar + """ + ) + pytester.makepyfile( + """ + def test(): + pass + """ + ) + result = pytester.runpytest_subprocess("-W", "ignore::bizbaz.Bizbaz") + result.stdout.fnmatch_lines( + [ + f"*== {WARNINGS_SUMMARY_HEADER} ==*", + "*PytestConfigWarning: Failed to import filter module 'foobar': ignore::foobar.Foobar", + "*PytestConfigWarning: Failed to import filter module 'bizbaz': ignore::bizbaz.Bizbaz", + "* 1 passed, * warning*", + ] + ) + + class TestDeprecationWarningsByDefault: """ Note: all pytest runs are executed in a subprocess so we don't inherit warning filters @@ -511,8 +538,32 @@ def test_hidden_by_system(self, pytester: Pytester, monkeypatch) -> None: result = pytester.runpytest_subprocess() assert WARNINGS_SUMMARY_HEADER not in result.stdout.str() + def test_invalid_regex_in_filterwarning(self, pytester: Pytester) -> None: + self.create_file(pytester) + pytester.makeini( + """ + [pytest] + filterwarnings = + ignore::DeprecationWarning:* + """ + ) + result = pytester.runpytest_subprocess() + assert result.ret == pytest.ExitCode.USAGE_ERROR + result.stderr.fnmatch_lines( + [ + "ERROR: while parsing the following warning configuration:", + "", + " ignore::DeprecationWarning:[*]", + "", + "This error occurred:", + "", + "Invalid regex '[*]': nothing to repeat at position 0", + ] + ) + -@pytest.mark.skip("not relevant until pytest 9.0") +# In 9.1, uncomment below and change RemovedIn9 -> RemovedIn10. +# @pytest.mark.skip("not relevant until pytest 10.0") @pytest.mark.parametrize("change_default", [None, "ini", "cmdline"]) def test_removed_in_x_warning_as_error(pytester: Pytester, change_default) -> None: """This ensures that PytestRemovedInXWarnings raised by pytest are turned into errors. @@ -691,10 +742,8 @@ def test_issue4445_rewrite(self, pytester: Pytester, capwarn) -> None: assert func == "" # the above conftest.py assert lineno == 4 - def test_issue4445_preparse(self, pytester: Pytester, capwarn) -> None: - """#4445: Make sure the warning points to a reasonable location - See origin of _issue_warning_captured at: _pytest.config.__init__.py:910 - """ + def test_issue4445_initial_conftest(self, pytester: Pytester, capwarn) -> None: + """#4445: Make sure the warning points to a reasonable location.""" pytester.makeconftest( """ import nothing @@ -710,7 +759,7 @@ def test_issue4445_preparse(self, pytester: Pytester, capwarn) -> None: assert "could not load initial conftests" in str(warning.message) assert f"config{os.sep}__init__.py" in file - assert func == "_preparse" + assert func == "parse" @pytest.mark.filterwarnings("default") def test_conftest_warning_captured(self, pytester: Pytester) -> None: diff --git a/testing/typing_checks.py b/testing/typing_checks.py index d4d6a97aea6..3ee2dfb3019 100644 --- a/testing/typing_checks.py +++ b/testing/typing_checks.py @@ -8,12 +8,13 @@ from __future__ import annotations import contextlib -from typing import Optional +from typing import Literal from typing_extensions import assert_type import pytest from pytest import MonkeyPatch +from pytest import TestReport # Issue #7488. @@ -50,4 +51,10 @@ class Foo(TypedDict): def check_raises_is_a_context_manager(val: bool) -> None: with pytest.raises(RuntimeError) if val else contextlib.nullcontext() as excinfo: pass - assert_type(excinfo, Optional[pytest.ExceptionInfo[RuntimeError]]) + assert_type(excinfo, pytest.ExceptionInfo[RuntimeError] | None) + + +# Issue #12941. +def check_testreport_attributes(report: TestReport) -> None: + assert_type(report.when, Literal["setup", "call", "teardown"]) + assert_type(report.location, tuple[str, int | None, str]) diff --git a/testing/typing_raises_group.py b/testing/typing_raises_group.py new file mode 100644 index 00000000000..081ffd59bca --- /dev/null +++ b/testing/typing_raises_group.py @@ -0,0 +1,240 @@ +from __future__ import annotations + +from collections.abc import Callable +import sys + +from typing_extensions import assert_type + +from _pytest.main import Failed as main_Failed +from _pytest.outcomes import Failed +from pytest import raises +from pytest import RaisesExc +from pytest import RaisesGroup + + +# does not work +assert_type(raises.Exception, Failed) # type: ignore[assert-type, attr-defined] + +# FIXME: these are different for some reason(?) +assert Failed is not main_Failed # type: ignore[comparison-overlap] + +if sys.version_info < (3, 11): + from exceptiongroup import BaseExceptionGroup + from exceptiongroup import ExceptionGroup + +# split into functions to isolate the different scopes + + +def check_raisesexc_typevar_default(e: RaisesExc) -> None: + assert e.expected_exceptions is not None + _exc: type[BaseException] | tuple[type[BaseException], ...] = e.expected_exceptions + # this would previously pass, as the type would be `Any` + e.exception_type().blah() # type: ignore + + +def check_basic_contextmanager() -> None: + with RaisesGroup(ValueError) as e: + raise ExceptionGroup("foo", (ValueError(),)) + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_basic_matches() -> None: + # check that matches gets rid of the naked ValueError in the union + exc: ExceptionGroup[ValueError] | ValueError = ExceptionGroup("", (ValueError(),)) + if RaisesGroup(ValueError).matches(exc): + assert_type(exc, ExceptionGroup[ValueError]) + + # also check that BaseExceptionGroup shows up for BaseExceptions + if RaisesGroup(KeyboardInterrupt).matches(exc): + assert_type(exc, BaseExceptionGroup[KeyboardInterrupt]) + + +def check_matches_with_different_exception_type() -> None: + e: BaseExceptionGroup[KeyboardInterrupt] = BaseExceptionGroup( + "", + (KeyboardInterrupt(),), + ) + + # note: it might be tempting to have this warn. + # however, that isn't possible with current typing + if RaisesGroup(ValueError).matches(e): + assert_type(e, ExceptionGroup[ValueError]) + + +def check_raisesexc_init() -> None: + def check_exc(exc: BaseException) -> bool: + return isinstance(exc, ValueError) + + # Check various combinations of constructor signatures. + # At least 1 arg must be provided. + RaisesExc() # type: ignore + RaisesExc(ValueError) + RaisesExc(ValueError, match="regex") + RaisesExc(ValueError, match="regex", check=check_exc) + RaisesExc(match="regex") + RaisesExc(check=check_exc) + RaisesExc(ValueError, match="regex") + RaisesExc(match="regex", check=check_exc) + + def check_filenotfound(exc: FileNotFoundError) -> bool: + return not exc.filename.endswith(".tmp") + + # If exception_type is provided, that narrows the `check` method's argument. + RaisesExc(FileNotFoundError, check=check_filenotfound) + RaisesExc(ValueError, check=check_filenotfound) # type: ignore + RaisesExc(check=check_filenotfound) # type: ignore + RaisesExc(FileNotFoundError, match="regex", check=check_filenotfound) + + # exceptions are pos-only + RaisesExc(expected_exception=ValueError) # type: ignore + # match and check are kw-only + RaisesExc(ValueError, "regex") # type: ignore + + +def raisesgroup_check_type_narrowing() -> None: + """Check type narrowing on the `check` argument to `RaisesGroup`. + All `type: ignore`s are correctly pointing out type errors. + """ + + def handle_exc(e: BaseExceptionGroup[BaseException]) -> bool: + return True + + def handle_kbi(e: BaseExceptionGroup[KeyboardInterrupt]) -> bool: + return True + + def handle_value(e: BaseExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(BaseException, check=handle_exc) + RaisesGroup(BaseException, check=handle_kbi) # type: ignore + + RaisesGroup(Exception, check=handle_exc) + RaisesGroup(Exception, check=handle_value) # type: ignore + + RaisesGroup(KeyboardInterrupt, check=handle_exc) + RaisesGroup(KeyboardInterrupt, check=handle_kbi) + RaisesGroup(KeyboardInterrupt, check=handle_value) # type: ignore + + RaisesGroup(ValueError, check=handle_exc) + RaisesGroup(ValueError, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, check=handle_value) + + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_exc) + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_kbi) # type: ignore + RaisesGroup(ValueError, KeyboardInterrupt, check=handle_value) # type: ignore + + +def raisesgroup_narrow_baseexceptiongroup() -> None: + """Check type narrowing specifically for the container exceptiongroup.""" + + def handle_group(e: ExceptionGroup[Exception]) -> bool: + return True + + def handle_group_value(e: ExceptionGroup[ValueError]) -> bool: + return True + + RaisesGroup(ValueError, check=handle_group_value) + + RaisesGroup(Exception, check=handle_group) + + +def check_raisesexc_transparent() -> None: + with RaisesGroup(RaisesExc(ValueError)) as e: + ... + _: BaseExceptionGroup[ValueError] = e.value + assert_type(e.value, ExceptionGroup[ValueError]) + + +def check_nested_raisesgroups_contextmanager() -> None: + with RaisesGroup(RaisesGroup(ValueError)) as excinfo: + raise ExceptionGroup("foo", (ValueError(),)) + + _: BaseExceptionGroup[BaseExceptionGroup[ValueError]] = excinfo.value + + assert_type( + excinfo.value, + ExceptionGroup[ExceptionGroup[ValueError]], + ) + + assert_type( + excinfo.value.exceptions[0], + # this union is because of how typeshed defines .exceptions + ExceptionGroup[ValueError] | ExceptionGroup[ExceptionGroup[ValueError]], + ) + + +def check_nested_raisesgroups_matches() -> None: + """Check nested RaisesGroup with .matches""" + exc: ExceptionGroup[ExceptionGroup[ValueError]] = ExceptionGroup( + "", + (ExceptionGroup("", (ValueError(),)),), + ) + + if RaisesGroup(RaisesGroup(ValueError)).matches(exc): + assert_type(exc, ExceptionGroup[ExceptionGroup[ValueError]]) + + +def check_multiple_exceptions_1() -> None: + a = RaisesGroup(ValueError, ValueError) + b = RaisesGroup(RaisesExc(ValueError), RaisesExc(ValueError)) + c = RaisesGroup(ValueError, RaisesExc(ValueError)) + + d: RaisesGroup[ValueError] + d = a + d = b + d = c + assert d + + +def check_multiple_exceptions_2() -> None: + # This previously failed due to lack of covariance in the TypeVar + a = RaisesGroup(RaisesExc(ValueError), RaisesExc(TypeError)) + b = RaisesGroup(RaisesExc(ValueError), TypeError) + c = RaisesGroup(ValueError, TypeError) + + d: RaisesGroup[Exception] + d = a + d = b + d = c + assert d + + +def check_raisesgroup_overloads() -> None: + # allow_unwrapped=True does not allow: + # multiple exceptions + RaisesGroup(ValueError, TypeError, allow_unwrapped=True) # type: ignore + # nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), allow_unwrapped=True) # type: ignore + # specifying match + RaisesGroup(ValueError, match="foo", allow_unwrapped=True) # type: ignore + # specifying check + RaisesGroup(ValueError, check=bool, allow_unwrapped=True) # type: ignore + # allowed variants + RaisesGroup(ValueError, allow_unwrapped=True) + RaisesGroup(ValueError, allow_unwrapped=True, flatten_subgroups=True) + RaisesGroup(RaisesExc(ValueError), allow_unwrapped=True) + + # flatten_subgroups=True does not allow nested RaisesGroup + RaisesGroup(RaisesGroup(ValueError), flatten_subgroups=True) # type: ignore + # but rest is plenty fine + RaisesGroup(ValueError, TypeError, flatten_subgroups=True) + RaisesGroup(ValueError, match="foo", flatten_subgroups=True) + RaisesGroup(ValueError, check=bool, flatten_subgroups=True) + RaisesGroup(ValueError, flatten_subgroups=True) + RaisesGroup(RaisesExc(ValueError), flatten_subgroups=True) + + # if they're both false we can of course specify nested raisesgroup + RaisesGroup(RaisesGroup(ValueError)) + + +def check_triple_nested_raisesgroup() -> None: + with RaisesGroup(RaisesGroup(RaisesGroup(ValueError))) as e: + assert_type(e.value, ExceptionGroup[ExceptionGroup[ExceptionGroup[ValueError]]]) + + +def check_check_typing() -> None: + # `BaseExceptiongroup` should perhaps be `ExceptionGroup`, but close enough + assert_type( + RaisesGroup(ValueError).check, + Callable[[BaseExceptionGroup[ValueError]], bool] | None, + ) diff --git a/tox.ini b/tox.ini index 61563ca2c5f..e2e09fa8b7f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,21 +1,19 @@ [tox] -isolated_build = True -minversion = 3.20.0 -distshare = {homedir}/.tox/distshare +requires = + tox >= 4 envlist = linting - py38 - py39 py310 py311 py312 py313 + py314 pypy3 - py38-{pexpect,xdist,unittestextras,numpy,pluggymain,pylib} + py310-{pexpect,xdist,twisted24,twisted25,asynctest,numpy,pluggymain,pylib} doctesting doctesting-coverage plugins - py38-freeze + py310-freeze docs docs-checklinks @@ -25,6 +23,26 @@ envlist = +[pkgenv] +# NOTE: This section tweaks how Tox manages the PEP 517 build +# NOTE: environment where it assembles wheels (editable and regular) +# NOTE: for further installing them into regular testenvs. +# +# NOTE: `[testenv:.pkg]` does not work due to a regression in tox v4.14.1 +# NOTE: so `[pkgenv]` is being used in place of it. +# Refs: +# * https://github.com/tox-dev/tox/pull/3237 +# * https://github.com/tox-dev/tox/issues/3238 +# * https://github.com/tox-dev/tox/issues/3292 +# * https://hynek.me/articles/turbo-charge-tox/ +# +# NOTE: The `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST` environment +# NOTE: variable allows enforcing a pre-determined version for use in +# NOTE: the wheel being installed into usual testenvs. +pass_env = + SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST + + [testenv] description = run the tests @@ -36,29 +54,32 @@ description = pexpect: against `pexpect` pluggymain: against the bleeding edge `pluggy` from Git pylib: against `py` lib - unittestextras: against the unit test extras + twisted24: against the unit test extras with twisted prior to 24.0 + twisted25: against the unit test extras with twisted 25.0 or later + asynctest: against the unit test extras with asynctest xdist: with pytest in parallel mode under `{basepython}` doctesting: including doctests commands = {env:_PYTEST_TOX_COVERAGE_RUN:} pytest {posargs:{env:_PYTEST_TOX_DEFAULT_POSARGS:}} - doctesting: {env:_PYTEST_TOX_COVERAGE_RUN:} pytest --doctest-modules --pyargs _pytest + doctesting: {env:_PYTEST_TOX_COVERAGE_RUN:} pytest --doctest-modules {env:_PYTEST_TOX_POSARGS_JUNIT:} --pyargs _pytest coverage: coverage combine coverage: coverage report -m passenv = COVERAGE_* PYTEST_ADDOPTS TERM - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST + CI setenv = - _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} + _PYTEST_TOX_DEFAULT_POSARGS={env:_PYTEST_TOX_POSARGS_DOCTESTING:} {env:_PYTEST_TOX_POSARGS_JUNIT:} {env:_PYTEST_TOX_POSARGS_LSOF:} {env:_PYTEST_TOX_POSARGS_XDIST:} {env:_PYTEST_FILES:} # See https://docs.python.org/3/library/io.html#io-encoding-warning # If we don't enable this, neither can any of our downstream users! - PYTHONWARNDEFAULTENCODING=1 + # pylib is not PYTHONWARNDEFAULTENCODING clean, so don't set for it. + !pylib: PYTHONWARNDEFAULTENCODING=1 # Configuration to run with coverage similar to CI, e.g. - # "tox -e py38-coverage". + # "tox -e py313-coverage". coverage: _PYTEST_TOX_COVERAGE_RUN=coverage run -m coverage: _PYTEST_TOX_EXTRA_DEP=coverage-enable-subprocess coverage: COVERAGE_FILE={toxinidir}/.coverage @@ -66,6 +87,12 @@ setenv = doctesting: _PYTEST_TOX_POSARGS_DOCTESTING=doc/en + # The configurations below are related only to standard unittest support. + # Run only tests from test_unittest.py. + asynctest: _PYTEST_FILES=testing/test_unittest.py + twisted24: _PYTEST_FILES=testing/test_unittest.py + twisted25: _PYTEST_FILES=testing/test_unittest.py + nobyte: PYTHONDONTWRITEBYTECODE=1 lsof: _PYTEST_TOX_POSARGS_LSOF=--lsof @@ -79,17 +106,20 @@ deps = pexpect: pexpect>=4.8.0 pluggymain: pluggy @ git+https://github.com/pytest-dev/pluggy.git pylib: py>=1.8.2 - unittestextras: twisted - unittestextras: asynctest + twisted24: twisted<25 + twisted25: twisted>=25 + asynctest: asynctest xdist: pytest-xdist>=2.1.0 xdist: -e . {env:_PYTEST_TOX_EXTRA_DEP:} +# Can use the same wheel for all environments. +package = wheel +wheel_build_env = .pkg [testenv:linting] description = run pre-commit-defined linters under `{basepython}` skip_install = True -basepython = python3 deps = pre-commit>=2.9.3 commands = pre-commit run --all-files --show-diff-on-failure {posargs:} setenv = @@ -100,17 +130,11 @@ setenv = description = build the documentation site under \ `{toxinidir}{/}doc{/}en{/}_build{/}html` with `{basepython}` -basepython = python3.12 # sync with rtd to get errors +basepython = python3.13 # Sync with .readthedocs.yaml to get errors. usedevelop = True deps = -r{toxinidir}/doc/en/requirements.txt -allowlist_externals = - git commands = - # Retrieve possibly missing commits: - -git fetch --unshallow - -git fetch --tags - sphinx-build \ -j auto \ -W --keep-going \ @@ -123,7 +147,6 @@ setenv = [testenv:docs-checklinks] description = check the links in the documentation with `{basepython}` -basepython = python3 usedevelop = True changedir = doc/en deps = -r{toxinidir}/doc/en/requirements.txt @@ -137,9 +160,6 @@ setenv = description = regenerate documentation examples under `{basepython}` changedir = doc/en -basepython = python3 -passenv = - SETUPTOOLS_SCM_PRETEND_VERSION_FOR_PYTEST deps = PyYAML regendoc>=0.8.1 @@ -151,6 +171,10 @@ commands = setenv = # We don't want this warning to reach regen output. PYTHONWARNDEFAULTENCODING= + # Remove CI markers: pytest auto-detects those and uses more verbose output, which is undesirable + # for the example documentation. + CI= + BUILD_NUMBER= [testenv:plugins] description = @@ -178,7 +202,7 @@ commands = pytest pytest_twisted_integration.py pytest simple_integration.py --force-sugar --flakes -[testenv:py38-freeze] +[testenv:py310-freeze] description = test pytest frozen with `pyinstaller` under `{basepython}` changedir = testing/freeze @@ -190,16 +214,12 @@ commands = [testenv:release] description = do a release, required posarg of the version number -basepython = python3 usedevelop = True passenv = * deps = colorama - github3.py pre-commit>=2.9.3 - wheel - # https://github.com/twisted/towncrier/issues/340 - towncrier<21.3.0 + towncrier commands = python scripts/release.py {posargs} [testenv:prepare-release-pr] @@ -211,8 +231,19 @@ commands = python scripts/prepare-release-pr.py {posargs} [testenv:generate-gh-release-notes] description = generate release notes that can be published as GitHub Release -basepython = python3 usedevelop = True deps = - pypandoc + pypandoc_binary commands = python scripts/generate-gh-release-notes.py {posargs} + +[testenv:update-plugin-list] +description = update the plugin list +skip_install = True +deps = + packaging + requests + tabulate[widechars] + tqdm + requests-cache + platformdirs +commands = python scripts/update-plugin-list.py {posargs}