diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ecd85064f..0b7c7b289 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,9 @@ source/guides/github-actions-ci-cd-sample/* @webknjaz source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst @webknjaz +# Sphinx extension +pug_sphinx_extensions/ @FFY00 + # build-details.json source/specifications/build-details/ @FFY00 source/specifications/specs/build-details-*.json @FFY00 diff --git a/.github/workflows/pr-preview-links.yml b/.github/workflows/pr-preview-links.yml index 90ea9cc73..291ec3ad2 100644 --- a/.github/workflows/pr-preview-links.yml +++ b/.github/workflows/pr-preview-links.yml @@ -17,6 +17,6 @@ jobs: documentation-links: runs-on: ubuntu-latest steps: - - uses: readthedocs/actions/preview@v1 + - uses: readthedocs/actions/preview@b8bba1484329bda1a3abe986df7ebc80a8950333 # v1.5 with: project-slug: "python-packaging-user-guide" diff --git a/.github/workflows/test-translations.yml b/.github/workflows/test-translations.yml index 45dc60aa3..537a8df72 100644 --- a/.github/workflows/test-translations.yml +++ b/.github/workflows/test-translations.yml @@ -31,9 +31,10 @@ jobs: steps: - name: Grab the repo src - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ env.I18N_BRANCH }} + persist-credentials: false - name: List languages id: languages @@ -53,12 +54,13 @@ jobs: steps: - name: Grab the repo src - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: ref: ${{ env.I18N_BRANCH }} + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: >- 3.10 @@ -67,10 +69,12 @@ jobs: run: python -m pip install --upgrade nox virtualenv sphinx-lint - name: Set Sphinx problem matcher - uses: sphinx-doc/github-problem-matcher@v1.0 + uses: sphinx-doc/github-problem-matcher@1f74d6599f4a5e89a20d3c99aab4e6a70f7bda0f # v1.1 - name: Build translated docs in ${{ matrix.language }} - run: nox -s build -- -q -D language=${{ matrix.language }} + run: nox -s build -- -q -D language=${LANGUAGE} + env: + LANGUAGE: ${{ matrix.language }} - name: Set Sphinx Lint problem matcher if: always() @@ -78,4 +82,6 @@ jobs: - name: Lint translation file if: always() - run: sphinx-lint locales/${{ matrix.language }}/LC_MESSAGES/messages.po + run: sphinx-lint locales/${LANGUAGE}/LC_MESSAGES/messages.po + env: + LANGUAGE: ${{ matrix.language }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8503ca720..81ea4f054 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,28 +6,45 @@ on: branches-ignore: - gh-readonly-queue/** # Temporary merge queue-related GH-made branches pull_request: + types: + - opened # default + - synchronize # default + - reopened # default + - ready_for_review # used in PRs created from GitHub Actions workflows workflow_call: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: true +permissions: {} + jobs: build: name: ${{ matrix.noxenv }} if: ${{ github.repository_owner == 'pypa' || github.event_name != 'schedule' }} runs-on: ubuntu-latest + timeout-minutes: 20 + continue-on-error: >- + ${{ fromJSON(matrix.continue-on-error) }} strategy: matrix: noxenv: - build - - linkcheck + continue-on-error: + - false + include: + - noxenv: linkcheck + continue-on-error: >- # Don't block PRs on linkcheck unrelated failures + ${{ toJSON(github.event_name == 'pull_request') }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: "3.11" cache: 'pip' @@ -38,6 +55,9 @@ jobs: python -m pip install --upgrade nox virtualenv - name: Nox ${{ matrix.noxenv }} + env: + # Authenticate github.com requests during linkcheck to avoid rate limits. + GITHUB_TOKEN: ${{ matrix.noxenv == 'linkcheck' && github.token || '' }} run: | python -m nox -s ${{ matrix.noxenv }} @@ -55,6 +75,6 @@ jobs: steps: - name: Decide whether the needed jobs succeeded or failed - uses: re-actors/alls-green@release/v1 + uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2 with: jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/translation.yml b/.github/workflows/translation.yml index 7cfae2991..67fcb5edf 100644 --- a/.github/workflows/translation.yml +++ b/.github/workflows/translation.yml @@ -17,16 +17,20 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'pypa' + permissions: + contents: write # to push to I18N_BRANCH + steps: - name: Grab the repo src - uses: actions/checkout@v3 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: fetch-depth: 0 # To reach the common commit + persist-credentials: true # For `git push` - name: Set up git user as [bot] # Refs: # * https://github.community/t/github-actions-bot-email-address/17204/6 # * https://github.com/actions/checkout/issues/13#issuecomment-724415212 - uses: fregante/setup-git-user@v1.1.0 + uses: fregante/setup-git-user@024bc0b8e177d7e77203b48dab6fb45666854b35 # v2.0.2 - name: Switch to the translation source branch run: | @@ -48,10 +52,12 @@ jobs: run: | sh -x - git merge '${{ github.event.repository.default_branch }}' + git merge "${DEFAULT_BRANCH}" + env: + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 with: python-version: >- 3.10 diff --git a/.github/workflows/update-uv-build-version.yml b/.github/workflows/update-uv-build-version.yml new file mode 100644 index 000000000..d204bd391 --- /dev/null +++ b/.github/workflows/update-uv-build-version.yml @@ -0,0 +1,43 @@ +--- + +name: Update uv build version + +on: + schedule: + - cron: "0 6 * * 1" # mondays at 6am + workflow_dispatch: + +jobs: + update-uv-build-version: + name: Update uv_build version + if: github.repository_owner == 'pypa' # suppress noise in forks + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + steps: + - name: Checkout repository + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + with: + persist-credentials: false + - name: Set up uv + uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0 + - name: Update uv_build version + id: update_script + run: uv run scripts/update_uv_build_version.py + - # If there are no changes, no pull request will be created and the action exits silently. + name: Create Pull Request + uses: peter-evans/create-pull-request@271a8d0340265f705b14b6d32b9829c1cb33d45e # v7.0.8 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update uv_build version to ${{ steps.update_script.outputs.version }} + title: Update uv_build version to ${{ steps.update_script.outputs.version }} + draft: true # Trigger CI by un-drafting the PR, otherwise `GITHUB_TOKEN` PRs don't trigger CI. + body: | + Automated update of uv_build version bounds for uv ${{ steps.update_script.outputs.version }}. + + This PR was created automatically by the cron workflow, ping `@konstin` for problems. + branch: bot/update-uv-build-version + delete-branch: true + +... diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml index d99b6473c..6c8c62f7d 100644 --- a/.github/workflows/zizmor.yml +++ b/.github/workflows/zizmor.yml @@ -19,12 +19,12 @@ jobs: actions: read steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 with: persist-credentials: false - name: Install the latest version of uv - uses: astral-sh/setup-uv@v5 + uses: astral-sh/setup-uv@3259c6206f993105e3a61b142c2d97bf4b9ef83d # v7.1.0 - name: Run zizmor 🌈 run: uvx zizmor --format sarif source/guides/github-actions-ci-cd-sample/* > results.sarif @@ -32,7 +32,7 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload SARIF file - uses: github/codeql-action/upload-sarif@v3 + uses: github/codeql-action/upload-sarif@f443b600d91635bebf5b0d9ebc620189c0d6fba5 # v4.30.8 with: sarif_file: results.sarif category: zizmor diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e092c419c..6d8d4e78b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,7 @@ ci: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v5.0.0 + rev: v6.0.0 hooks: - id: check-added-large-files - id: check-case-conflict @@ -37,7 +37,7 @@ repos: - id: rst-inline-touching-normal - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.12.2 + rev: v0.14.10 hooks: - - id: ruff + - id: ruff-check - id: ruff-format diff --git a/source/specifications/schemas/build-details-v1.0.schema.json b/extra/specifications/schemas/build-details-v1.0.schema.json similarity index 100% rename from source/specifications/schemas/build-details-v1.0.schema.json rename to extra/specifications/schemas/build-details-v1.0.schema.json diff --git a/source/specifications/schemas/direct-url.schema.json b/extra/specifications/schemas/direct-url.schema.json similarity index 100% rename from source/specifications/schemas/direct-url.schema.json rename to extra/specifications/schemas/direct-url.schema.json diff --git a/source/specifications/schemas/pylock.schema.json b/extra/specifications/schemas/pylock.schema.json similarity index 100% rename from source/specifications/schemas/pylock.schema.json rename to extra/specifications/schemas/pylock.schema.json diff --git a/noxfile.py b/noxfile.py index 698e82f9d..484a8d39a 100644 --- a/noxfile.py +++ b/noxfile.py @@ -89,6 +89,7 @@ def linkcheck(session): "--keep-going", # be strict "source", # where the rst files are located "build", # where to put the check output + *session.posargs, ) diff --git a/pug_sphinx_extensions/__init__.py b/pug_sphinx_extensions/__init__.py new file mode 100644 index 000000000..00d91da3c --- /dev/null +++ b/pug_sphinx_extensions/__init__.py @@ -0,0 +1,86 @@ +import os +import pathlib +import urllib + +import sphinx.application +import sphinx.util.logging + + +DOMAIN = "packaging.python.org" + + +logger = sphinx.util.logging.getLogger(__name__) + + +def resolve_local_html_link(app: sphinx.application.Sphinx, url_path: str) -> str: + """Takes path of a link pointing an HTML render of the current project, + and returns local path of the referenced document. + + Support links to renders from both the `html` and `dirhtml` builders. + + Example: + + .. code-block:: python + + >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow/') + '{srcdir}/flow.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/flow.html') + '{srcdir}/flow.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/') + '{srcdir}/specifications/schemas/index.rst' + >>> resolve_local_html_link('https://packaging.python.org/en/latest/specifications/schemas/build-details-v1.0.schema.json') + '{html_extra_path0}/specifications/schemas/build-details-v1.0.schema.json' + + """ + # Search for document in html_extra_path + for entry in app.config.html_extra_path: + candidate = (app.confdir / entry / url_path).resolve() + if candidate.is_dir(): + candidate = candidate / "index.html" + if candidate.exists(): + return os.fspath(candidate) + # Convert html path to source path + url_path = url_path.removesuffix("/") # Normalize + if url_path.endswith(".html"): + document = url_path.removesuffix(".html") + elif (candidate := f"{url_path}/index") in app.project.docnames: + document = candidate + else: + document = url_path + return app.env.doc2path(document) + + +def rewrite_local_uri(app: sphinx.application.Sphinx, uri: str) -> str: + """Replace remote URIs targeting https://packaging.python.org/en/latest/... + with local ones, so that local changes are taken into account by linkcheck. + + Additionally, resolve local relative links to html_extra_path. + """ + local_uri = uri + parsed = urllib.parse.urlparse(uri) + # Links to https://packaging.python.org/en/latest/... + if parsed.hostname == DOMAIN and parsed.path.startswith("/en/latest/"): + document = parsed.path.removeprefix("/en/latest/") + local_uri = resolve_local_html_link(app, document) + logger.verbose( + f"{uri!s} is a remote URL that points to local sources, " + "replacing it with a local URL in linkcheck to take new changes " + "into account (pass -vv for more info)" + ) + logger.debug(f"Replacing linkcheck URL {uri!r} with {local_uri!r}") + # Local relative links + if not parsed.scheme and not parsed.netloc and parsed.path: + full_path = pathlib.Path(app.env.docname).parent / parsed.path + local_uri = resolve_local_html_link(app, os.fspath(full_path)) + if local_uri != uri: + logger.verbose(f"Local linkcheck URL {uri!r} resolved as {local_uri!r}") + return local_uri + + +def setup(app: sphinx.application.Sphinx) -> dict[str, bool]: + app.connect("linkcheck-process-uri", rewrite_local_uri) + + return { + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/scripts/update_uv_build_version.py b/scripts/update_uv_build_version.py new file mode 100644 index 000000000..69fefba27 --- /dev/null +++ b/scripts/update_uv_build_version.py @@ -0,0 +1,64 @@ +# /// script +# requires-python = ">= 3.12" +# dependencies = [ +# "httpx>=0.28.1,<0.29", +# "packaging>=25.0", +# ] +# /// +import os +import re +from pathlib import Path + +import httpx +from packaging.utils import parse_wheel_filename +from packaging.version import Version + + +def main(): + response = httpx.get( + "https://pypi.org/simple/uv-build/", + headers={"Accept": "application/vnd.pypi.simple.v1+json"}, + ) + response.raise_for_status() + data = response.json() + current_release = None + for file in data["files"]: + if not file["filename"].endswith(".whl"): + continue + _name, version, _build, _tags = parse_wheel_filename(file["filename"]) + if version.is_prerelease: + continue + if current_release is None or version > current_release: + current_release = version + + [major, minor, _patch] = current_release.release + if major != 0: + raise NotImplementedError("The script needs to be updated for uv 1.x") + upper_bound = Version(f"{major}.{minor + 1}.0") + + repository_root = Path(__file__).parent.parent + existing = repository_root.joinpath( + "source/shared/build-backend-tabs.rst" + ).read_text() + replacement = f'requires = ["uv_build >= {current_release}, <{upper_bound}"]' + searcher = re.compile(re.escape('requires = ["uv_build') + ".*" + re.escape('"]')) + if not searcher.search(existing): + raise RuntimeError("Could not `uv-build` entry") + updated = searcher.sub(replacement, existing) + + if existing != updated: + print("Updating source/shared/build-backend-tabs.rst") + Path("source/shared/build-backend-tabs.rst").write_text(updated) + if github_output := os.environ.get("GITHUB_OUTPUT"): + with open(github_output, "a") as f: + f.write(f"version={current_release}\n") + f.write("updated=true\n") + else: + print("Already up-to-date source/shared/build-backend-tabs.rst") + if github_output := os.environ.get("GITHUB_OUTPUT"): + with open(github_output, "a") as f: + f.write("updated=false\n") + + +if __name__ == "__main__": + main() diff --git a/source/conf.py b/source/conf.py index a8a040d6c..4516880ec 100644 --- a/source/conf.py +++ b/source/conf.py @@ -2,6 +2,11 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information import os +import pathlib +import sys + +_ROOT = pathlib.Path(__file__).resolve().parent.parent +sys.path.append(os.fspath(_ROOT)) # Some options are only enabled for the main packaging.python.org deployment builds RTD_BUILD = bool(os.getenv("READTHEDOCS")) @@ -22,6 +27,7 @@ root_doc = "index" extensions = [ + "pug_sphinx_extensions", "sphinx.ext.extlinks", "sphinx.ext.intersphinx", "sphinx.ext.todo", @@ -83,6 +89,10 @@ # https://plausible.io/packaging.python.org html_js_files.extend(_metrics_js_files) +html_extra_path = [ + "../extra", +] + # -- Options for HTML help output ------------------------------------------------------ # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-help-output @@ -129,7 +139,6 @@ linkcheck_ignore = [ r"http://localhost:\d+", - r"https://packaging\.python\.org/en/latest/specifications/schemas/.*", r"https://test\.pypi\.org/project/example-package-YOUR-USERNAME-HERE", r"https://pypi\.org/manage/.*", r"https://test\.pypi\.org/manage/.*", @@ -139,6 +148,8 @@ # Ignore while StackOverflow is blocking GitHub CI. Ref: # https://github.com/pypa/packaging.python.org/pull/1474 r"https://stackoverflow\.com/.*", + # Cloudflare challenge blocks automated link checking. + r"https://clickpy\.clickhouse\.com/$", r"https://pyscaffold\.org/.*", r"https://anaconda\.org", r"https://www\.cisa\.gov/sbom", @@ -146,8 +157,16 @@ r"https://math-atlas\.sourceforge\.net/?", r"https://click\.palletsprojects\.com/.*", r"https://typer\.tiangolo\.com/.*", + r"https://www.npmjs.com/.*", + r"https://docutils\.sourceforge\.io/.*", + # Temporarily ignored due to expired TLS cert. + # Ref: https://github.com/pypa/packaging.python.org/issues/1998 + r"https://blog\.ganssle\.io/.*", + # Temporarily ignored due to expired TLS cert. + r"https://kivy.org/.*", ] -linkcheck_retries = 5 +linkcheck_retries = 2 +linkcheck_timeout = 30 # Ignore anchors for common targets when we know they likely won't be found linkcheck_anchors_ignore_for_url = [ # GitHub synthesises anchors in JavaScript, so Sphinx can't find them in the HTML @@ -157,6 +176,13 @@ # https://github.com/pypa/packaging.python.org/issues/1744 r"https://pypi\.org/", ] +# Authenticate requests to github.com (when a token is available) to avoid +# unauthenticated rate limits that can stall linkcheck for hours on CI. +if _gh_token := os.getenv("GITHUB_TOKEN"): + linkcheck_request_headers = { + "https://github.com/": {"Authorization": f"Bearer {_gh_token}"}, + "https://api.github.com/": {"Authorization": f"Bearer {_gh_token}"}, + } # -- Options for extlinks ---------------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/extensions/extlinks.html#configuration @@ -209,7 +235,6 @@ "tox": ("https://tox.wiki/en/latest/", None), "twine": ("https://twine.readthedocs.io/en/stable/", None), "virtualenv": ("https://virtualenv.pypa.io/en/stable/", None), - "warehouse": ("https://warehouse.pypa.io/", None), } # -- Options for todo extension -------------------------------------------------------- diff --git a/source/discussions/deploying-python-applications.rst b/source/discussions/deploying-python-applications.rst index e10f36f9c..59856c4a4 100644 --- a/source/discussions/deploying-python-applications.rst +++ b/source/discussions/deploying-python-applications.rst @@ -97,7 +97,7 @@ services, and DLL/EXE COM servers might work but it is not actively supported. The distutils extension is released under the MIT-licence and Mozilla Public License 2.0. -.. __: https://devguide.python.org/#status-of-python-branches +.. __: https://devguide.python.org/versions/ macOS ----- diff --git a/source/discussions/versioning.rst b/source/discussions/versioning.rst index eeea3578c..aee3d25ea 100644 --- a/source/discussions/versioning.rst +++ b/source/discussions/versioning.rst @@ -27,7 +27,7 @@ examples of version numbers [#version-examples]_: - A post-release of an alpha release (possible, but discouraged): ``1.2.0a1.post1`` - A simple version with only two components: ``23.12`` - A simple version with just one component: ``42`` -- A version with an epoch: ``1!1.0`` +- A version with an epoch (discouraged): ``1!1.0`` Projects can use a cycle of pre-releases to support testing by their users before a final release. In order, the steps are: alpha releases, beta releases, @@ -46,13 +46,14 @@ notes. They should not be used for bug fixes; these should be done with a new final release (e.g., incrementing the third component when using semantic versioning). -Finally, epochs, a rarely used feature, serve to fix the sorting order when -changing the versioning scheme. For example, if a project is using calendar -versioning, with versions like 23.12, and switches to semantic versioning, with -versions like 1.0, the comparison between 1.0 and 23.12 will go the wrong way. -To correct this, the new version numbers should have an explicit epoch, as in -"1!1.0", in order to be treated as more recent than the old version numbers. - +Finally, epochs were intended to fix the sorting order when changing the +versioning scheme. For example, if a project was using calendar versioning, with +versions like ``23.12``, and switched to semantic versioning, with versions like +``1.0``, the comparison between ``1.0`` and ``23.12`` would go the wrong way. To +correct this, the new version numbers would have an explicit epoch, as in +``1!1.0``, in order to be treated as more recent than the old version numbers. +However, this is discouraged, and it is preferable to use a higher version +number that is unlikely to cause user confusion, such as ``100.0``. Semantic versioning vs. calendar versioning diff --git a/source/glossary.rst b/source/glossary.rst index 6a592125f..ab32f077b 100644 --- a/source/glossary.rst +++ b/source/glossary.rst @@ -14,7 +14,7 @@ Glossary Build Backend - A library that takes a source tree + A library that takes a :term:`source tree ` and builds a :term:`source distribution ` or :term:`built distribution ` from it. The build is delegated to the backend by a @@ -160,7 +160,7 @@ Glossary A string with valid SPDX license expression syntax, including one or more SPDX :term:`License Identifier`\(s), - which describes a :term:`Project`'s license(s) + which describes a :term:`Distribution Archive`'s license(s) and how they inter-relate. Examples: ``GPL-3.0-or-later``, @@ -287,8 +287,7 @@ Glossary PyPA is a working group that maintains many of the relevant projects in Python packaging. They maintain a site at :doc:`pypa.io `, host projects on `GitHub - `_ and `Bitbucket - `_, and discuss issues on the + `_, and discuss issues on the `distutils-sig mailing list `_ and `the Python Discourse forum `__. diff --git a/source/guides/analyzing-pypi-package-downloads.rst b/source/guides/analyzing-pypi-package-downloads.rst index 2ad02fed5..2e2fdb583 100644 --- a/source/guides/analyzing-pypi-package-downloads.rst +++ b/source/guides/analyzing-pypi-package-downloads.rst @@ -333,6 +333,14 @@ Usage: The `pandas-gbq`_ project allows for accessing query results via `Pandas`_. +``ClickPy`` +----------- + +The `ClickPy`_ project provides a public application to visualize download +statistics, with free direct SQL access to the underlying open-source +`ClickHouse`_ database, updated daily. + + References ========== @@ -346,3 +354,5 @@ References .. _google-cloud-bigquery: https://cloud.google.com/bigquery/docs/reference/libraries .. _pandas-gbq: https://pandas-gbq.readthedocs.io/en/latest/ .. _Pandas: https://pandas.pydata.org/ +.. _ClickHouse: https://github.com/ClickHouse/ClickHouse +.. _Clickpy: https://clickpy.clickhouse.com/ diff --git a/source/guides/creating-command-line-tools.rst b/source/guides/creating-command-line-tools.rst index 8266fffdb..cbe8b3bb0 100644 --- a/source/guides/creating-command-line-tools.rst +++ b/source/guides/creating-command-line-tools.rst @@ -40,33 +40,22 @@ named after the main module: def greet( - name: Annotated[str, typer.Argument(help="The (last, if --gender is given) name of the person to greet")] = "", - gender: Annotated[str, typer.Option(help="The gender of the person to greet")] = "", - knight: Annotated[bool, typer.Option(help="Whether the person is a knight")] = False, + name: Annotated[str, typer.Argument(help="The (last, if --title is given) name of the person to greet")] = "", + title: Annotated[str, typer.Option(help="The preferred title of the person to greet")] = "", + doctor: Annotated[bool, typer.Option(help="Whether the person is a doctor (MD or PhD)")] = False, count: Annotated[int, typer.Option(help="Number of times to greet the person")] = 1 ): - greeting = "Greetings, dear " - masculine = gender == "masculine" - feminine = gender == "feminine" - if gender or knight: - salutation = "" - if knight: - salutation = "Sir " - elif masculine: - salutation = "Mr. " - elif feminine: - salutation = "Ms. " - greeting += salutation - if name: - greeting += f"{name}!" + greeting = "Greetings, " + if doctor and not title: + title = "Dr." + if not name: + if title: + name = title.lower().rstrip(".") else: - pronoun = "her" if feminine else "his" if masculine or knight else "its" - greeting += f"what's-{pronoun}-name" - else: - if name: - greeting += f"{name}!" - elif not gender: - greeting += "friend!" + name = "friend" + if title: + greeting += f"{title} " + greeting += f"{name}!" for i in range(0, count): print(greeting) @@ -145,12 +134,14 @@ Let's test it: .. code-block:: console - $ greet --knight Lancelot - Greetings, dear Sir Lancelot! - $ greet --gender feminine Parks - Greetings, dear Ms. Parks! - $ greet --gender masculine - Greetings, dear Mr. what's-his-name! + $ greet + Greetings, friend! + $ greet --doctor Brennan + Greetings, Dr. Brennan! + $ greet --title Ms. Parks + Greetings, Ms. Parks! + $ greet --title Mr. + Greetings, Mr. mr! Since this example uses ``typer``, you could now also get an overview of the program's usage by calling it with the ``--help`` option, or configure completions via the ``--install-completion`` option. @@ -160,10 +151,10 @@ To just run the program without installing it permanently, use ``pipx run``, whi .. code-block:: console - $ pipx run --spec . greet --knight + $ pipx run --spec . greet --doctor This syntax is a bit impractical, however; as the name of the entry point we defined above does not match the package name, -we need to state explicitly which executable script to run (even though there is only on in existence). +we need to state explicitly which executable script to run (even though there is only one in existence). There is, however, a more practical solution to this problem, in the form of an entry point specific to ``pipx run``. The same can be defined as follows in :file:`pyproject.toml`: @@ -179,7 +170,7 @@ default one and run it, which makes this command possible: .. code-block:: console - $ pipx run . --knight + $ pipx run . --doctor Conclusion ========== diff --git a/source/guides/dropping-older-python-versions.rst b/source/guides/dropping-older-python-versions.rst index 267d7b923..223b65cd0 100644 --- a/source/guides/dropping-older-python-versions.rst +++ b/source/guides/dropping-older-python-versions.rst @@ -89,7 +89,7 @@ such as at least Python 3.9. Or, at least Python 3.7 and beyond, skipping the 3. If using the :ref:`setuptools` build backend, consult the `dependency-management`_ documentation for more options. .. caution:: - Avoid adding upper bounds to the version ranges, e. g. ``">= 3.8, < 3.10"``. Doing so can cause different errors + Avoid adding upper bounds to the version ranges, e.g. ``">= 3.8, < 3.10"``. Doing so can cause different errors and version conflicts. See the `discourse-discussion`_ for more information. 3. Validating the Metadata before publishing diff --git a/source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml b/source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml similarity index 90% rename from source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml rename to source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml index 8813a0392..155f82555 100644 --- a/source/guides/github-actions-ci-cd-sample/publish-to-test-pypi.yml +++ b/source/guides/github-actions-ci-cd-sample/publish-to-pypi.yml @@ -8,11 +8,11 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.x" - name: Install pypa/build @@ -24,7 +24,7 @@ jobs: - name: Build a binary wheel and a source tarball run: python3 -m build - name: Store the distribution packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: python-package-distributions path: dist/ @@ -44,7 +44,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: python-package-distributions path: dist/ @@ -66,7 +66,7 @@ jobs: steps: - name: Download all the dists - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v6 with: name: python-package-distributions path: dist/ diff --git a/source/guides/licensing-examples-and-user-scenarios.rst b/source/guides/licensing-examples-and-user-scenarios.rst index 2c25ddfb0..b6cdfe327 100644 --- a/source/guides/licensing-examples-and-user-scenarios.rst +++ b/source/guides/licensing-examples-and-user-scenarios.rst @@ -6,8 +6,8 @@ Licensing examples and user scenarios ===================================== -:pep:`639` has specified the way to declare a project's license and paths to -license files and other legally required information. +:pep:`639` has specified the way to declare a :term:`Distribution Archive`'s +license and paths to license files and other legally required information. This document aims to provide clear guidance how to migrate from the legacy to the standardized way of declaring licenses. Make sure your preferred build backend supports :pep:`639` before @@ -53,7 +53,7 @@ Or, if the project used :file:`setup.cfg`, in its ``[metadata]`` table: [metadata] license = MIT -The output Core Metadata for the distribution packages would then be: +The output Core Metadata for the :term:`Distribution Package` would then be: .. code-block:: email @@ -63,8 +63,9 @@ The output Core Metadata for the distribution packages would then be: The :file:`LICENSE` file would be stored at :file:`/setuptools-{VERSION}/LICENSE` in the sdist and :file:`/setuptools-{VERSION}.dist-info/licenses/LICENSE` in the wheel, and unpacked from there into the site directory (e.g. -:file:`site-packages/`) on installation; :file:`/` is the root of the respective archive -and ``{VERSION}`` the version of the Setuptools release in the Core Metadata. +:file:`site-packages/`) on installation; :file:`/` is the root of the respective +archive and ``{VERSION}`` the version of the Setuptools release in the Core +Metadata. .. _licensing-example-advanced: @@ -83,7 +84,7 @@ directories; specifically: ordered-set==3.1.1 more_itertools==8.8.0 -The license expressions for these projects are: +The appropriate license expressions are: .. code-block:: text @@ -287,7 +288,7 @@ and make sure to remove any legacy ``license`` table subkeys or ``License ::`` classifiers. Your existing ``license`` value may already be valid as one (e.g. ``MIT``, ``Apache-2.0 OR BSD-2-Clause``, etc); otherwise, check the `SPDX license list `__ for the identifier -that matches the license used in your project. +that matches the license used. Make sure to list your license files under ``license-files`` under ``[project]`` in :file:`pyproject.toml` @@ -312,12 +313,11 @@ to describe the licenses involved and the relationship between them. In short, ``License-1 AND License-2`` mean that *both* licenses apply -to your project, or parts of it (for example, you included a file -under another license), and ``License-1 OR License-2`` means that -*either* of the licenses can be used, at the user's option (for example, -you want to allow users a choice of multiple licenses). You can use -parenthesis (``()``) for grouping to form expressions that cover even the most -complex situations. +(for example, you included a file under another license), and +``License-1 OR License-2`` means that *either* of the licenses can be used, at +the user's option (for example, you want to allow users a choice of multiple +licenses). You can use parenthesis (``()``) for grouping to form expressions +that cover even the most complex situations. In your project config file, enter your license expression under ``license`` (``[project]`` table of :file:`pyproject.toml`), diff --git a/source/guides/modernize-setup-py-project.rst b/source/guides/modernize-setup-py-project.rst index 5b6ab3c26..1f71d1973 100644 --- a/source/guides/modernize-setup-py-project.rst +++ b/source/guides/modernize-setup-py-project.rst @@ -67,7 +67,7 @@ For more details: * :ref:`distributing-packages` * :ref:`pyproject-build-system-table` -* :doc:`pip:reference/build-system/pyproject-toml` +* :doc:`pip:reference/build-system` How to handle additional build-time dependencies? @@ -128,7 +128,7 @@ For some projects this isolation is unwanted and it can be deactivated as follow For more details: -* :doc:`pip:reference/build-system/pyproject-toml` +* :doc:`pip:reference/build-system` How to handle packaging metadata? @@ -244,5 +244,5 @@ Where to read more about this? ============================== * :ref:`pyproject-toml-spec` -* :doc:`pip:reference/build-system/pyproject-toml` +* :doc:`pip:reference/build-system` * :doc:`setuptools:build_meta` diff --git a/source/guides/multi-version-installs.rst b/source/guides/multi-version-installs.rst index a09bc900a..456ee22e2 100644 --- a/source/guides/multi-version-installs.rst +++ b/source/guides/multi-version-installs.rst @@ -37,7 +37,3 @@ time, but that approach does mean that standard command line invocations of the affected tools can't be used - it's necessary to write a custom wrapper script or use ``python3 -c ''`` to invoke the application's main entry point directly. - -Refer to the `pkg_resources documentation -`__ -for more details. diff --git a/source/guides/packaging-namespace-packages.rst b/source/guides/packaging-namespace-packages.rst index 3d929d527..6ff93c4c9 100644 --- a/source/guides/packaging-namespace-packages.rst +++ b/source/guides/packaging-namespace-packages.rst @@ -159,8 +159,7 @@ Legacy namespace packages These two methods, that were used to create namespace packages prior to :pep:`420`, are now considered to be obsolete and should not be used unless you need compatibility -with packages already using this method. Also, :doc:`pkg_resources ` -has been deprecated. +with packages already using one of these methods. To migrate an existing package, all packages sharing the namespace must be migrated simultaneously. @@ -176,7 +175,7 @@ pkgutil-style namespace packages Python 2.3 introduced the :doc:`pkgutil ` module and the :py:func:`python:pkgutil.extend_path` function. This can be used to declare namespace packages that need to be compatible with both Python 2.3+ and Python 3. This -is the recommended approach for the highest level of compatibility. +was the recommended approach for the highest level of compatibility. To create a pkgutil-style namespace package, you need to provide an :file:`__init__.py` file for the namespace package: @@ -216,10 +215,18 @@ in the `pkgutil namespace example project`_. pkg_resources-style namespace packages ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -:doc:`Setuptools ` provides the `pkg_resources.declare_namespace`_ function and +.. warning:: + + The information in this section is obsolete and is no longer functional + (as of Setuptools 82.0.0). It is only retained for historical reference. + + ``pkg_resources`` has been deprecated and was fully removed in Setuptools 82.0.0. + +:doc:`Setuptools ` previously provided the ``pkg_resources.declare_namespace`` function and the ``namespace_packages`` argument to :func:`~setuptools.setup`. Together -these can be used to declare namespace packages. While this approach is no -longer recommended, it is widely present in most existing namespace packages. +these could be used to declare namespace packages. While this approach is no +longer supported, it may still be encountered in environments using older +``setuptools`` versions. If you are creating a new distribution within an existing namespace package that uses this method then it's recommended to continue using this as the different methods are not cross-compatible and it's not advisable to try to migrate an @@ -281,11 +288,3 @@ to :func:`~setuptools.setup` in :file:`setup.py`. For example: packages=find_packages() namespace_packages=['mynamespace'] ) - -A complete working example of two pkg_resources-style namespace packages can be found -in the `pkg_resources namespace example project`_. - -.. _pkg_resources.declare_namespace: - https://setuptools.readthedocs.io/en/latest/pkg_resources.html#namespace-package-support -.. _pkg_resources namespace example project: - https://github.com/pypa/sample-namespace-packages/tree/master/pkg_resources diff --git a/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst b/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst index e9f601e03..3b5e6ed28 100644 --- a/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst +++ b/source/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows.rst @@ -87,13 +87,13 @@ Creating a workflow definition GitHub CI/CD workflows are declared in YAML files stored in the ``.github/workflows/`` directory of your repository. -Let's create a ``.github/workflows/publish-to-test-pypi.yml`` +Let's create a ``.github/workflows/publish-to-pypi.yml`` file. Start it with a meaningful name and define the event that should make GitHub run this workflow: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :end-before: jobs: @@ -107,7 +107,14 @@ build the distribution packages. First, we'll define the job for building the dist packages of your project and storing them for later use: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. tip:: + + If you adapt this workflow to build multiple platform-specific wheels, use + uniquely named artifacts for each build job and adjust the download step + accordingly. The `cibuildwheel GitHub Actions examples`_ show a fuller + wheel matrix layout. + +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :start-at: jobs: :end-before: Install pypa/build @@ -119,7 +126,7 @@ And now we can build the dists from source and store them. In this example, we'll use the ``build`` package. So add this to the steps list: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :start-at: Install pypa/build :end-before: publish-to-pypi @@ -136,7 +143,7 @@ UI nicely. Additionally, it allows acquiring an OpenID Connect token that the ``pypi-publish`` actions needs to implement secretless Trusted Publishing to PyPI. -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :start-after: path: dist/ :end-before: steps: @@ -149,7 +156,7 @@ Publishing the distribution to PyPI Finally, add the following steps at the end: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :start-after: id-token: write :end-before: publish-to-testpypi: @@ -175,7 +182,7 @@ Now, repeat these steps and create another job for publishing to the TestPyPI package index under the ``jobs`` section: -.. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml +.. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml :start-at: publish-to-testpypi @@ -191,7 +198,7 @@ This paragraph showcases the whole workflow after following the above guide. .. collapse:: Click here to display the entire GitHub Actions CI/CD workflow definition - .. literalinclude:: github-actions-ci-cd-sample/publish-to-test-pypi.yml + .. literalinclude:: github-actions-ci-cd-sample/publish-to-pypi.yml :language: yaml That's all, folks! @@ -231,3 +238,5 @@ sure that your release pipeline remains healthy! .. _Secrets: https://docs.github.com/en/actions/reference/encrypted-secrets .. _Trusted Publishing: https://docs.pypi.org/trusted-publishers/ +.. _`cibuildwheel GitHub Actions examples`: + https://cibuildwheel.pypa.io/en/latest/ci-services/#github-actions diff --git a/source/guides/supporting-multiple-python-versions.rst b/source/guides/supporting-multiple-python-versions.rst index 8c128ed91..2e6d87f12 100644 --- a/source/guides/supporting-multiple-python-versions.rst +++ b/source/guides/supporting-multiple-python-versions.rst @@ -62,9 +62,8 @@ of many continuous-integration systems. There are two hosted services which when used in conjunction provide automated testing across Linux, Mac and Windows: - - `Travis CI `_ provides both a Linux and a macOS - environment. The Linux environment is Ubuntu 12.04 LTS Server Edition 64 bit - while the macOS is 10.9.2 at the time of writing. + - `GitHub Actions `_ provides Windows, + Linux and a macOS environments. - `Appveyor `_ provides a Windows environment (Windows Server 2012). @@ -76,7 +75,7 @@ Windows: TODO How do we keep the Travis Linux and macOS versions up-to-date in this document? -Both `Travis CI`_ and Appveyor_ require a `YAML +Both `GitHub Actions`_ and Appveyor_ require a `YAML `_-formatted file as specification for the instructions for testing. If any tests fail, the output log for that specific configuration can be inspected. diff --git a/source/guides/supporting-windows-using-appveyor.rst b/source/guides/supporting-windows-using-appveyor.rst index 0044d8c5e..b661024f7 100644 --- a/source/guides/supporting-windows-using-appveyor.rst +++ b/source/guides/supporting-windows-using-appveyor.rst @@ -20,12 +20,6 @@ Many projects are developed on Unix by default, and providing Windows support can be a challenge, because setting up a suitable Windows test environment is non-trivial, and may require buying software licenses. -The Appveyor service is a continuous integration service, much like the -better-known `Travis`_ service that is commonly used for testing by projects -hosted on `GitHub`_. However, unlike Travis, the build workers on Appveyor are -Windows hosts and have the necessary compilers installed to build Python -extensions. - Windows users typically do not have access to a C compiler, and therefore are reliant on projects that use C extensions distributing binary wheels on PyPI in order for the distribution to be installable via ``python -m pip install ``. By @@ -46,8 +40,7 @@ your project is hosted on one of those two services, setting up Appveyor integration is straightforward. Once you have set up your Appveyor account and added your project, Appveyor will -automatically build your project each time a commit occurs. This behaviour will -be familiar to users of Travis. +automatically build your project each time a commit occurs. Adding Appveyor support to your project ======================================= @@ -179,7 +172,7 @@ other CI systems). 2. When used interactively, ``tox`` allows you to run your tests against multiple environments (often, this means multiple Python versions). This feature is not as - useful in a CI environment like Travis or Appveyor, where all tests are run in + useful in a CI environment like Appveyor, where all tests are run in isolated environments for each configuration. As a result, projects often supply an argument ``-e ENVNAME`` to ``tox`` to specify which environment to use (there are default environments for most versions of Python). @@ -237,6 +230,6 @@ For reference, the SDK setup support script is listed here: :linenos: .. _Appveyor: https://www.appveyor.com/ -.. _Travis: https://travis-ci.org/ .. _GitHub: https://github.com +.. _GitHub Actions: https://docs.github.com/en/actions .. _Bitbucket: https://bitbucket.org/ diff --git a/source/guides/tool-recommendations.rst b/source/guides/tool-recommendations.rst index 1ba36ed61..bf8d93d5a 100644 --- a/source/guides/tool-recommendations.rst +++ b/source/guides/tool-recommendations.rst @@ -109,6 +109,11 @@ Do **not** use :ref:`distutils`, which is deprecated, and has been removed from the standard library in Python 3.12, although it still remains available from setuptools. +.. _extension-module-tool-recommendations: + +Build backends for extension modules +------------------------------------ + For packages with :term:`extension modules `, it is best to use a build system with dedicated support for the language the extension is written in, for example: diff --git a/source/guides/writing-pyproject-toml.rst b/source/guides/writing-pyproject-toml.rst index 1d035a384..92a7f25bf 100644 --- a/source/guides/writing-pyproject-toml.rst +++ b/source/guides/writing-pyproject-toml.rst @@ -296,10 +296,10 @@ You can also specify the format explicitly, like this: ``license`` and ``license-files`` --------------------------------- -As per :pep:`639` licenses should be declared with two fields: +As per :pep:`639`, licenses should be declared with two fields: -- ``license`` is an :term:`SPDX license expression ` consisting - of one or more :term:`license identifiers `. +- ``license`` is an :term:`SPDX license expression ` + consisting of one or more :term:`license identifiers `. - ``license-files`` is a list of license file glob patterns. A previous PEP had specified ``license`` to be a table with a ``file`` or a @@ -319,7 +319,7 @@ backend>` now support the new format as shown in the following table. - 77.0.3 - 3.12 - 2.4.0 - - `not yet `_ + - 2.2.0 - 0.7.19 @@ -350,10 +350,11 @@ As a general rule, it is a good idea to use a standard, well-known license, both to avoid confusion and because some organizations avoid software whose license is unapproved. -If your project is licensed with a license that doesn't have an existing SPDX -identifier, you can create a custom one in format ``LicenseRef-[idstring]``. -The custom identifiers must follow the SPDX specification, -`clause 10.1 `_ of the version 2.2 or any later compatible one. +If your :term:`Distribution Archive` is licensed with a license that doesn't +have an existing SPDX identifier, you can create a custom one in format +``LicenseRef-[idstring]``. The custom identifiers must follow the SPDX +specification, `clause 10.1 `_ of the version 2.2 or any later +compatible one. .. code-block:: toml @@ -412,6 +413,7 @@ A list of PyPI classifiers that apply to your project. Check the .. code-block:: toml + [project] classifiers = [ # How mature is this project? Common values are # 3 - Alpha @@ -587,7 +589,6 @@ A full example .. _pypi-search-pip: https://pypi.org/search?q=pip .. _classifier-list: https://pypi.org/classifiers .. _requires-python-blog-post: https://iscinumpy.dev/post/bound-version-constraints/#pinning-the-python-version-is-special -.. _poetry-pep639-issue: https://github.com/python-poetry/poetry/issues/9670 .. _pytest: https://pytest.org .. _pygments: https://pygments.org .. _rest: https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html diff --git a/source/key_projects.rst b/source/key_projects.rst index e4501fe0e..dc7544a0b 100644 --- a/source/key_projects.rst +++ b/source/key_projects.rst @@ -827,6 +827,19 @@ scientific applications on clusters and supercomputers. Spack is not in PyPI (yet), but it requires no installation and can be used immediately after cloning from GitHub. +.. _uv: + +uv +== + +`Docs `__ | +`GitHub `__ | +`PyPI `__ + +A Python package and project manager, written in Rust for high performance. It +supports creating and managing virtual environments, installing packages, +locking dependencies, and managing Python versions and projects. + .. _zestreleaser: zest.releaser diff --git a/source/overview.rst b/source/overview.rst index 8c68036a7..d7b3efdaf 100644 --- a/source/overview.rst +++ b/source/overview.rst @@ -279,7 +279,7 @@ A similar model involves installing an alternative Python distribution, but does not support arbitrary operating system-level packages: -* `ActiveState ActivePython `_ +* `ActiveState ActivePython `_ * `WinPython `_ .. _bringing-your-own-python: @@ -339,7 +339,7 @@ originated and where the technologies below work best: Bringing your own kernel ^^^^^^^^^^^^^^^^^^^^^^^^ -Most operating systems support some form of classical virtualization, +Most desktop operating systems support some form of classical virtualization, running applications packaged as images containing a full operating system of their own. Running these virtual machines, or VMs, is a mature approach, widespread in data center environments. @@ -348,9 +348,13 @@ These techniques are mostly reserved for larger scale deployments in data centers, though certain complex applications can benefit from this packaging. The technologies are Python agnostic, and include: -* `Vagrant `_ -* `VHD `_, `AMI `_, and :doc:`other formats ` -* `OpenStack `_ - A cloud management system in Python, with extensive VM support +* KVM on Linux +* Hyper-V on Windows +* `VHD `_, + `AMI `_, + and :doc:`other formats ` +* `OpenStack `_ - + A cloud management system written in Python, with extensive VM support Bringing your own hardware ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/source/shared/build-backend-tabs.rst b/source/shared/build-backend-tabs.rst index 608fcaddd..5f3e0bf4c 100644 --- a/source/shared/build-backend-tabs.rst +++ b/source/shared/build-backend-tabs.rst @@ -22,7 +22,7 @@ .. code-block:: toml [build-system] - requires = ["flit_core >= 3.12.0, <4"] + requires = ["flit_core >= 3.12.0, <5"] build-backend = "flit_core.buildapi" .. tab:: PDM @@ -38,5 +38,5 @@ .. code-block:: toml [build-system] - requires = ["uv_build >= 0.7.19, <0.9.0"] + requires = ["uv_build >= 0.12.1, <0.13.0"] build-backend = "uv_build" diff --git a/source/specifications/binary-distribution-format.rst b/source/specifications/binary-distribution-format.rst index 8bb41ab40..a6f141851 100644 --- a/source/specifications/binary-distribution-format.rst +++ b/source/specifications/binary-distribution-format.rst @@ -240,18 +240,17 @@ The .dist-info directory secure hashes. Unlike PEP 376, every file except RECORD, which cannot contain a hash of itself, must include its hash. The hash algorithm must be sha256 or better; specifically, md5 and sha1 are - not permitted, as signed wheel files rely on the strong hashes in - RECORD to validate the integrity of the archive. + not permitted. #. PEP 376's INSTALLER and REQUESTED are not included in the archive. -#. RECORD.jws is used for digital signatures. It is not mentioned in - RECORD. -#. RECORD.p7s is allowed as a courtesy to anyone who would prefer to - use S/MIME signatures to secure their wheel files. It is not - mentioned in RECORD. +#. RECORD.jws and RECORD.p7s are deprecated. Where they are still + used, neither RECORD.jws nor RECORD.p7s are mentioned in RECORD. + Build backends and other tools must not add them to wheels anymore, + installers should be aware that these files may still be part of + some wheels. #. During extraction, wheel installers verify all the hashes in RECORD - against the file contents. Apart from RECORD and its signatures, - installation will fail if any file in the archive is not both - mentioned and correctly hashed in RECORD. + against the file contents. Apart from RECORD, RECORD.jws and + RECORD.p7s, installation will fail if any file in the archive is not + both mentioned and correctly hashed in RECORD. Subdirectories in :file:`.dist-info/` ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -277,6 +276,8 @@ fields is specified, the :file:`.dist-info/` directory MUST contain a ``License-File`` fields in the :file:`METADATA` file at their respective paths relative to the :file:`licenses/` directory. +.. _dist-info-sbom-directory: + The :file:`.dist-info/sboms/` directory ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -300,52 +301,6 @@ documentation and so forth from the distribution. During installation the contents of these subdirectories are moved onto their destination paths. -Signed wheel files ------------------- - -Wheel files include an extended RECORD that enables digital -signatures. PEP 376's RECORD is altered to include a secure hash -``digestname=urlsafe_b64encode_nopad(digest)`` (urlsafe base64 -encoding with no trailing = characters) as the second column instead -of an md5sum. All possible entries are hashed, including any -generated files such as .pyc files, but not RECORD which cannot contain its -own hash. For example:: - - file.py,sha256=AVTFPZpEKzuHr7OvQZmhaU3LvwKz06AJw8mT\_pNh2yI,3144 - distribution-1.0.dist-info/RECORD,, - -The signature file(s) RECORD.jws and RECORD.p7s are not mentioned in -RECORD at all since they can only be added after RECORD is generated. -Every other file in the archive must have a correct hash in RECORD -or the installation will fail. - -If JSON web signatures are used, one or more JSON Web Signature JSON -Serialization (JWS-JS) signatures is stored in a file RECORD.jws adjacent -to RECORD. JWS is used to sign RECORD by including the SHA-256 hash of -RECORD as the signature's JSON payload: - -.. code-block:: json - - { "hash": "sha256=ADD-r2urObZHcxBW3Cr-vDCu5RJwT4CaRTHiFmbcIYY" } - -(The hash value is the same format used in RECORD.) - -If RECORD.p7s is used, it must contain a detached S/MIME format signature -of RECORD. - -A wheel installer is not required to understand digital signatures but -MUST verify the hashes in RECORD against the extracted file contents. -When the installer checks file hashes against RECORD, a separate signature -checker only needs to establish that RECORD matches the signature. - -See - -- https://datatracker.ietf.org/doc/html/rfc7515 -- https://datatracker.ietf.org/doc/html/draft-jones-json-web-signature-json-serialization-01 -- https://datatracker.ietf.org/doc/html/rfc7517 -- https://datatracker.ietf.org/doc/html/draft-jones-jose-json-private-key-01 - - FAQ === @@ -361,34 +316,6 @@ Wheel defines a .data directory. Should I put all my data there? in *wheel's* ``.data`` directory. -Why does wheel include attached signatures? -------------------------------------------- - - Attached signatures are more convenient than detached signatures - because they travel with the archive. Since only the individual files - are signed, the archive can be recompressed without invalidating - the signature or individual files can be verified without having - to download the whole archive. - - -Why does wheel allow JWS signatures? ------------------------------------- - - The JOSE specifications of which JWS is a part are designed to be easy - to implement, a feature that is also one of wheel's primary design - goals. JWS yields a useful, concise pure-Python implementation. - - -Why does wheel also allow S/MIME signatures? --------------------------------------------- - - S/MIME signatures are allowed for users who need or want to use - existing public key infrastructure with wheel. - - Signed packages are only a basic building block in a secure package - update system. Wheel only provides the building block. - - What's the deal with "purelib" vs. "platlib"? --------------------------------------------- @@ -465,6 +392,7 @@ History :pep:`639`. - January 2025: Clarified that name and version needs to be normalized for ``.dist-info`` and ``.data`` directories. +- January 2026: Deprecate RECORD.jws and RECORD.p7s :pep:`815`. Appendix diff --git a/source/specifications/build-details/v1.0.rst b/source/specifications/build-details/v1.0.rst index cfe902e1e..3a8cfe277 100644 --- a/source/specifications/build-details/v1.0.rst +++ b/source/specifications/build-details/v1.0.rst @@ -8,7 +8,7 @@ Specification ------------- -.. jsonschema:: ../schemas/build-details-v1.0.schema.json +.. jsonschema:: ../../../extra/specifications/schemas/build-details-v1.0.schema.json :lift_title: false diff --git a/source/specifications/core-metadata.rst b/source/specifications/core-metadata.rst index 550c6e55a..b6fd009e2 100644 --- a/source/specifications/core-metadata.rst +++ b/source/specifications/core-metadata.rst @@ -6,7 +6,7 @@ Core metadata specifications ============================ -This page describes version 2.4, approved in August 2024. +This page describes version 2.6, approved in May 2026. Fields defined in the following specification should be considered valid, complete and not subject to change. The required fields are: @@ -50,11 +50,11 @@ Metadata-Version .. versionadded:: 1.0 Version of the file format; legal values are "1.0", "1.1", "1.2", "2.1", -"2.2", "2.3", and "2.4". +"2.2", "2.3", "2.4", "2.5", and "2.6". -Automated tools consuming metadata SHOULD warn if ``metadata_version`` is +Automated tools consuming metadata SHOULD warn if ``metadata-version`` is greater than the highest version they support, and MUST fail if -``metadata_version`` has a greater major version than the highest +``metadata-version`` has a greater major version than the highest version they support (as described in the :ref:`Version specifier specification `, the major version is the value before the first dot). @@ -109,6 +109,10 @@ Dynamic (multiple use) ====================== .. versionadded:: 2.2 +.. versionchanged:: 2.6 + A multiple use field that is present in the sdist and also marked + ``Dynamic`` may only be appended to in a wheel built from the sdist. + Previously any field listed in Dynamic was ignored in an sdist. A string containing the name of another core metadata field. The field names ``Name``, ``Version``, and ``Metadata-Version`` may not be specified @@ -121,8 +125,12 @@ rules apply: in any wheel built from the sdist MUST match the value in the sdist. If the field is not in the sdist, and not marked as ``Dynamic``, then it MUST NOT be present in the wheel. -2. If a field is marked as ``Dynamic``, it may contain any valid value in - a wheel built from the sdist (including not being present at all). +2. If a single-use field is marked as ``Dynamic``, it may contain any valid + value in a wheel built from the sdist (including not being present at all). +3. If a multiple use field is present in the sdist and also marked ``Dynamic``, + then a wheel built from the sdist MUST include the value(s) present in the + sdist. The wheel MAY add further values, but it MUST NOT remove, reorder, or + modify the values present in the sdist. If the sdist metadata version is older than version 2.2, then all fields should be treated as if they were specified with ``Dynamic`` (i.e. there are no special @@ -133,6 +141,16 @@ only, and indicates that the field value was calculated at wheel build time, and may not be the same as the value in the sdist or in other wheels for the project. +Note in particular that if you have obtained a prebuilt wheel, you cannot +assume that a field which is not marked as ``Dynamic`` will have the same value +in other wheels, as some wheels are not built directly from the sdist, but are +modified from existing wheels (the ``auditwheel`` tool does this, for example, +and it's commonly used when building wheels for PyPI). Such modifications +*could* include changing metadata (even non-dynamic metadata). Similarly, if +you have a sdist and a wheel which you didn't build from that sdist, you cannot +assume that the wheel's metadata matches that of the sdist, even if the field +is not marked as ``Dynamic``. + Full details of the semantics of ``Dynamic`` are described in :pep:`643`. .. _core-metadata-platform: @@ -473,6 +491,12 @@ Text string that is a valid SPDX :term:`license expression `, as specified in :doc:`/specifications/license-expression`. +Note that the expression in this field only applies to the +:term:`Distribution Archive` containing the metadata with this field (e.g., +:term:`Source Distribution ` or :term:`Wheel`), +not the project overall or other files related to the project (including other +distribution archives). + Examples:: License-Expression: MIT @@ -558,14 +582,14 @@ The format of a requirement string contains from one to four parts: * An environment marker after a semicolon. This means that the requirement is only needed in the specified conditions. -See :pep:`508` for full details of the allowed format. - The project names should correspond to names as found on the `Python Package Index`_. Version specifiers must follow the rules described in :doc:`version-specifiers`. +See :ref:`dependency-specifiers` for full details of the allowed format. + Examples:: Requires-Dist: pkginfo @@ -708,6 +732,101 @@ user SHOULD be warned and the value ignored to avoid ambiguity. Tools MAY choose to raise an error when reading an invalid name for older metadata versions. +.. _core-metadata-import-name: + +Import-Name (multiple use) +========================== + +.. versionadded:: 2.5 + +A string containing an import name that the project exclusively provides when +installed. The specified import name MUST be a valid Python identifier or can +be empty. The import names listed in this field MUST be importable when the +project is installed on *some* platform for the same version of the project. +This implies that the metadata MUST be consistent across all sdists and wheels +for a project release. + +An import name MAY be followed by a semicolon and the term "private" +(e.g. ``; private``) with any amount of whitespace surrounding the semicolon. +This signals to tools that the import name is not part of the public API for +the project. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name SHOULD also be listed appropriately +in ``Import-Name`` and/or ``Import-Namespace``. + +If a project lists the same name in both ``Import-Name`` and +``Import-Namespace``, tools MUST raise an error due to ambiguity. + +Tools SHOULD raise an error when two projects that are about to be installed +list names that overlap in each other's ``Import-Name`` entries, or when a +project has an entry in ``Import-Name`` that overlaps with another project's +``Import-Namespace`` entries. This is to avoid projects unexpectedly shadowing +another project's code. Tools MAY warn or raise an error when installing a +project into a preexisting environment where there is import name overlap with +a project that is already installed. + +Projects MAY have an empty ``Import-Name`` field in their metadata to represent +a project with no import names (i.e. there are no Python modules of any kind in +the distribution file). + +Since projects MAY have no ``Import-Name`` metadata (either because the +project uses an older metadata version, or because it didn't specify any), then +tools have no information about what names the project provides. However, in +practice the majority of projects have their project name match what their +import name would be. As such, it is a reasonable assumption to make that a +project name that is normalized in some way to an import name +(e.g. ``packaging.utils.canonicalize_name(name, validate=True).replace("-", "_")``) +can be used if some answer is needed. + +Examples:: + + Import-Name: PIL + Import-Name: _private_module ; private + Import-Name: zope.interface + Import-Name: + + +.. _core-metadata-import-namespace: + +Import-Namespace (multiple use) +=============================== + +.. versionadded:: 2.5 + +A string containing an import name that the project provides when installed, but +not exclusively. The specified import name MUST be a valid Python identifier. +This field is used for namespace packages where multiple projects can contribute +to the same import namespace. Projects all listing the same import name in +``Import-Namespace`` can be installed together without shadowing each other. + +An import name MAY be followed by a semicolon and the term "private" (e.g. +``; private``) with any amount of whitespace surrounding the semicolon. This +signals to tools that the import name is not part of the public API for the +project. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name SHOULD also be listed appropriately +in ``Import-Name`` and/or ``Import-Namespace``. + +The import names listed in this field MUST be importable when the project is +installed on *some* platform for the same version of the project. This implies +that the metadata MUST be consistent across all sdists and wheels for a project +release. + +If a project lists the same name in both ``Import-Name`` and +``Import-Namespace``, tools MUST raise an error due to ambiguity. + +Note that ``Import-Namespace`` CANNOT be empty like ``Import-Name``. + +Examples:: + + Import-Namespace: zope + Import-Name: _private_module ; private + + Rarely Used Fields ================== @@ -923,30 +1042,51 @@ Example:: History ======= -- August 2024: Core metadata 2.4 was approved through :pep:`639`. +- March 2001: Core metadata 1.0 was approved through :pep:`241`. - - Added the ``License-Expression`` field. - - Added the ``License-File`` field. +- April 2003: Core metadata 1.1 was approved through :pep:`314`. -- March 2022: Core metadata 2.3 was approved through :pep:`685`. +- February 2010: Core metadata 1.2 was approved through :pep:`345`. - - Restricted extra names to be normalized. +- February 2018: Core metadata 2.1 was approved through :pep:`566`. + + - Added ``Description-Content-Type`` and ``Provides-Extra``. + - Added canonical method for transforming metadata to JSON. + - Restricted the grammar of the ``Name`` field. - October 2020: Core metadata 2.2 was approved through :pep:`643`. - Added the ``Dynamic`` field. -- February 2018: Core metadata 2.1 was approved through :pep:`566`. +- March 2022: Core metadata 2.3 was approved through :pep:`685`. - - Added ``Description-Content-Type`` and ``Provides-Extra``. - - Added canonical method for transforming metadata to JSON. - - Restricted the grammar of the ``Name`` field. + - Restricted extra names to be normalized. -- February 2010: Core metadata 1.2 was approved through :pep:`345`. +- August 2024: Core metadata 2.4 was approved through :pep:`639`. -- April 2003: Core metadata 1.1 was approved through :pep:`314`: + - Added the ``License-Expression`` field. + - Added the ``License-File`` field. -- March 2001: Core metadata 1.0 was approved through :pep:`241`. +- August 2025: Clarified that ``Dynamic`` only affects how fields + must be treated when building a wheel from a sdist, not when modifying + a wheel. + +- September 2025: Core metadata 2.5 was approved through :pep:`794`. + + - Added the ``Import-Name`` field. + - Added the ``Import-Namespace`` field. + +- October 2025: Clarified that ``License-Expression`` applies to the containing + distribution file and not the project itself. + +- January 2026: Replaced outdated direct reference to :pep:`508` with a + reference to :ref:`dependency-specifiers`. + +- May 2026: Core metadata 2.6 was approved through :pep:`808`. + + - Allowed a multiple use field marked ``Dynamic`` to be appended to in a + wheel built from a sdist, requiring the wheel to preserve the value(s) + present in the sdist. ---- diff --git a/source/specifications/dependency-groups.rst b/source/specifications/dependency-groups.rst index 22e4cba0d..2fa758f7e 100644 --- a/source/specifications/dependency-groups.rst +++ b/source/specifications/dependency-groups.rst @@ -4,15 +4,15 @@ Dependency Groups ================= -This specification defines Dependency Groups, a mechanism for storing package +This specification defines dependency groups, a mechanism for storing package requirements in ``pyproject.toml`` files such that they are not included in project metadata when it is built. -Dependency Groups are suitable for internal development use-cases like linting +Dependency groups are suitable for internal development use-cases like linting and testing, as well as for projects which are not built for distribution, like collections of related scripts. -Fundamentally, Dependency Groups should be thought of as being a standardized +Fundamentally, dependency groups should be thought of as being a standardized subset of the capabilities of ``requirements.txt`` files (which are ``pip``-specific). @@ -38,7 +38,7 @@ and a similar table which defines ``docs``, ``test``, and ``coverage`` groups:: The ``[dependency-groups]`` Table --------------------------------- -Dependency Groups are defined as a table in ``pyproject.toml`` named +Dependency groups are defined as a table in ``pyproject.toml`` named ``dependency-groups``. The ``dependency-groups`` table contains an arbitrary number of user-defined keys, each of which has, as its value, a list of requirements. @@ -103,9 +103,9 @@ Package Building Build backends MUST NOT include Dependency Group data in built distributions as package metadata. This means that sdist ``PKG-INFO`` and wheel ``METADATA`` -files should not include referenceable fields containing Dependency Groups. +files should not include referenceable fields containing dependency groups. -It is, however, valid to use Dependency Groups in the evaluation of dynamic +It is, however, valid to use dependency groups in the evaluation of dynamic metadata, and ``pyproject.toml`` files included in sdists will still contain ``[dependency-groups]``. However, the table's contents are not part of a built package's interfaces. @@ -114,28 +114,28 @@ Installing Dependency Groups & Extras ------------------------------------- There is no syntax or specification-defined interface for installing or -referring to Dependency Groups. Tools are expected to provide dedicated +referring to dependency groups. Tools are expected to provide dedicated interfaces for this purpose. Tools MAY choose to provide the same or similar interfaces for interacting -with Dependency Groups as they do for managing extras. Tools authors are +with dependency groups as they do for managing extras. Tools authors are advised that the specification does not forbid having an extra whose name matches a Dependency Group. Separately, users are advised to avoid creating -Dependency Groups whose names match extras, and tools MAY treat such matching +dependency groups whose names match extras, and tools MAY treat such matching as an error. Validation and Compatibility ---------------------------- -Tools supporting Dependency Groups may want to validate data before using it. +Tools supporting dependency groups may want to validate data before using it. When implementing such validation, authors should be aware of the possibility of future extensions to the specification, so that they do not unnecessarily emit errors or warnings. Tools SHOULD error when evaluating or processing unrecognized data in -Dependency Groups. +dependency groups. -Tools SHOULD NOT eagerly validate the contents of *all* Dependency Groups +Tools SHOULD NOT eagerly validate the contents of *all* dependency groups unless they have a need to do so. This means that in the presence of the following data, most tools should allow @@ -151,7 +151,7 @@ the ``foo`` group to be used and only error if the ``bar`` group is used: There are several known cases of tools which have good cause to be stricter. Linters and validators are an example, as their purpose is to - validate the contents of all Dependency Groups. + validate the contents of all dependency groups. Reference Implementation ======================== @@ -209,8 +209,8 @@ The output is therefore valid ``requirements.txt`` data. realized_group = [] for item in raw_group: if isinstance(item, str): - # packaging.requirements.Requirement parsing ensures that this is a valid - # PEP 508 Dependency Specifier + # packaging.requirements.Requirement parsing ensures that this + # is a valid dependency specifier # raises InvalidRequirement on failure Requirement(item) realized_group.append(item) @@ -232,7 +232,7 @@ The output is therefore valid ``requirements.txt`` data. def resolve(dependency_groups: dict, group: str) -> list[str]: if not isinstance(dependency_groups, dict): - raise TypeError("Dependency Groups table is not a dict") + raise TypeError("Dependency groups table is not a dict") if not isinstance(group, str): raise TypeError("Dependency group name is not a str") return _resolve_dependency_group(dependency_groups, group) @@ -244,7 +244,7 @@ The output is therefore valid ``requirements.txt`` data. dependency_groups_raw = pyproject["dependency-groups"] dependency_groups = _normalize_group_names(dependency_groups_raw) - print("\n".join(resolve(pyproject["dependency-groups"], sys.argv[1]))) + print("\n".join(resolve(dependency_groups, sys.argv[1]))) History ======= diff --git a/source/specifications/dependency-specifiers.rst b/source/specifications/dependency-specifiers.rst index d9466c26e..d66f77503 100644 --- a/source/specifications/dependency-specifiers.rst +++ b/source/specifications/dependency-specifiers.rst @@ -6,21 +6,25 @@ Dependency specifiers ===================== -This document describes the dependency specifiers format as originally specified -in :pep:`508`. +This document defines the format used to specify dependencies on other projects. +The language defined is a compact line based format which was adapted from the +format originally used in ``pip`` requirements files. The job of a dependency is to enable tools like pip [#pip]_ to find the right package to install. Sometimes this is very loose - just specifying a name, and sometimes very specific - referring to a specific file to install. Sometimes -dependencies are only relevant in one platform, or only some versions are +dependencies are only relevant on one platform, or only some versions are acceptable, so the language permits describing all these cases. -The language defined is a compact line based format which is already in -widespread use in pip requirements files, though we do not specify the command -line option handling that those files permit. There is one caveat - the -URL reference form, specified in :ref:`Versioning specifier specification ` -is not actually implemented in pip, but we use that format rather -than pip's current native format. +Whether tools should be strict or permissive in their processing of dependency +specifiers is largely dependent on the role of the tool in the wider ecosystem: + +* publishing tools and index servers SHOULD be strict in their processing for + new releases, encouraging the consistency of published specifiers to improve + over time +* locking and installation tools MAY be permissive in their processing, allowing + consumption of older packages which may contain dependency specifiers that are + arguably nonsensical Specification ============= @@ -30,7 +34,7 @@ Examples All features of the language shown with a name based lookup:: - requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "2.7" + requests [security,tests] >= 2.8.1, == 2.8.* ; python_version < "3.7" A minimal URL based lookup:: @@ -63,7 +67,7 @@ Versions may be specified according to the rules of the :ref:`Version specifier specification `. (Note: URI is defined in :rfc:`std-66 <3986>`):: - version_cmp = wsp* '<' | '<=' | '!=' | '==' | '>=' | '>' | '~=' | '===' + version_cmp = wsp* '<=' | '<' | '!=' | '===' | '==' | '>=' | '>' | '~=' version = wsp* ( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+ version_one = version_cmp version wsp* version_many = version_one (',' version_one)* (',' wsp*)? @@ -108,8 +112,6 @@ field:: extras_list = identifier (wsp* ',' wsp* identifier)* extras = '[' wsp* extras_list? wsp* ']' -Restrictions on names for extras is defined in :pep:`685`. - Giving us a rule for name based requirements:: name_req = name wsp* extras? wsp* versionspec? wsp* quoted_marker? @@ -126,23 +128,22 @@ Whitespace ---------- Non line-breaking whitespace is mostly optional with no semantic meaning. The -sole exception is detecting the end of a URL requirement. +sole exceptions are detecting the end of a URL requirement and inside user +supplied constants in environment markers. .. _dependency-specifiers-names: Names ----- -Python distribution names are currently defined in :pep:`345`. Names -act as the primary identifier for distributions. They are present in all +Distribution names are defined in the :ref:`Core metadata `. +Names act as the primary identifier for distributions. They are present in all dependency specifications, and are sufficient to be a specification on their -own. However, PyPI places strict restrictions on names - they must match a -case insensitive regex or they won't be accepted. Accordingly, in this -document we limit the acceptable values for identifiers to that regex. A full -redefinition of name may take place in a future metadata PEP. The regex (run -with re.IGNORECASE) is:: +own. + +Valid distribution names are defined in the :ref:`name format specification +`. - ^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\Z .. _dependency-specifiers-extras: @@ -163,6 +164,12 @@ are listed in the "security" extra of requests. If multiple extras are listed, all the dependencies are unioned together. +Restrictions on names for extras are defined in the +:ref:`Core metadata specification `. Publication +tools SHOULD enforce these restrictions in dependency specifiers, while locking +and installation tools MAY normalize invalid extra names in order to accept +published metadata using core metadata versions prior to 2.3. + .. _dependency-specifiers-versions: Versions @@ -172,7 +179,7 @@ See the :ref:`Version specifier specification ` for more detail on both version numbers and version comparisons. Version specifications limit the versions of a distribution that can be used. They only apply to distributions looked up by name, rather than -via a URL. Version comparison are also used in the markers feature. The +via a URL. Version comparisons are also used in environment markers. The optional brackets around a version are present for compatibility with :pep:`345` but should not be generated, only accepted. @@ -183,63 +190,172 @@ Environment Markers Environment markers allow a dependency specification to provide a rule that describes when the dependency should be used. For instance, consider a package -that needs argparse. In Python 2.7 argparse is always present. On older Python -versions it has to be installed as a dependency. This can be expressed as so:: +that needs ``pywin32`` when running on Windows. This can be expressed as:: - argparse;python_version<"2.7" + pywin32; sys_platform == "win32" -A marker expression evaluates to either True or False. When it evaluates to -False, the dependency specification should be ignored. +A marker expression evaluates to either True or False for a given deployment +environment. When it evaluates to False, the dependency should be ignored. The marker language is inspired by Python itself, chosen for the ability to safely evaluate it without running arbitrary code that could become a security -vulnerability. Markers were first standardised in :pep:`345`. This document -fixes some issues that were observed in the design described in :pep:`426`. - -Comparisons in marker expressions are typed by the comparison operator and the -type of the marker value. The operators that are not in - perform the same as they do for strings or sets in Python based on -whether the marker value is a string or set itself. The operators -use the version comparison rules of the -:ref:`Version specifier specification ` when those are -defined (that is when both sides have a valid version specifier). If there is no -defined behaviour of this specification and the operator exists in Python, then -the operator falls back to the Python behaviour for the types involved. -Otherwise an error should be raised. e.g. the following will result in errors:: - - "dog" ~= "fred" - python_version ~= "surprise" - -User supplied constants are always encoded as strings with either ``'`` or -``"`` quote marks. Note that backslash escapes are not defined, but existing -implementations do support them. They are not included in this -specification because they add complexity and there is no observable need for -them today. Similarly we do not define non-ASCII character support: all the -runtime variables we are referencing are expected to be ASCII-only. - -The variables in the marker grammar such as "os_name" resolve to values looked -up in the Python runtime. With the exception of "extra" all values are defined -on all Python versions today - it is an error in the implementation of markers -if a value is not defined. - -Unknown variables must raise an error rather than resulting in a comparison -that evaluates to True or False. +vulnerability. + +Markers were first defined in :pep:`345`, formally specified in :pep:`508`, +then subsequently amended over time (amendments since :pep:`508` are recorded +:ref:`at the end of this specification `). + +Marker field types +'''''''''''''''''' + +Environment marker fields are each defined as one of the following types: + +* ``String``: the contents of the field are always treated as an opaque string. +* ``Set of strings``: the contents of the field are always treated as a set + containing opaque strings. In comparisons, the user supplied constant MUST + still be a single string (as set literals are not part of the marker syntax). +* ``Version``: the contents of the field are always expected to be a valid + :ref:`version specifier `. Publishing tools SHOULD emit + an error if that is not the case, but installation tools MAY fall back to + treating the field as a string field. +* ``Version | String``: the contents of the field are expected to be a valid + :ref:`version specifier ` on some platforms, but an + opaque string on others. The specifics of this distinction are field dependent + and whether or not tools actually make the distinction will be tool dependent. + +Marker comparisons +'''''''''''''''''' + +All marker comparison expressions are expected to compare a named marker field +against a given user supplied constant. The type of the comparison is determined +by the comparison operator used and the type of the named field as given +in :ref:`the table below `. Tools MAY emit an +error if no marker field is referenced in a comparison (that is, both operands +are given as constants). + +The follow comparison operations are defined in the marker expression grammar: + +* ``==`` (for example, ``sys_platform == "win32"``) +* ``!=`` (for example, ``sys_platform != "win32"``) +* ``>`` (for example, ``python_version > "3.10"``) +* ``>=`` (for example, ``python_version >= "3.10"``) +* ``<`` (for example, ``python_version < "3.10"``) +* ``<=`` (for example, ``python_version <= "3.10"``) +* ``~=`` (for example, ``python_version ~= "3"``) +* ``===`` (for example, ``implementation_version === "not.a.valid.version"``) +* ``in`` (for example, ``"gui" in extras``, ``"SMP" in platform_version``) +* ``not in`` (for example, ``"dev" not in dependency_groups``) + +For ``String`` fields, ``==``, ``!=``, ``in``, and ``not in`` are defined as +they are for Python strings (case sensitive, with no value normalization of any +kind). The use of ``~=`` or ``===`` with string fields is explicitly +discouraged and publishing tools SHOULD emit an error, index servers MAY +disallow uploads containing such environment markers, while locking and +installation tools MAY instead interpret them as equivalent to ``==``. The use +of ordered comparisons (``<``, ``<=``, ``>``, ``>=``) with string fields is +explicitly discouraged (as it makes no semantic sense in the packaging context) +and publishing tools SHOULD emit an error, index servers MAY disallow uploads +containing such environment markers, while locking and installation tools +SHOULD implement the following behavior: + +* treat ``>=`` and ``<=`` as equivalent to ``==`` +* treat ``>`` and ``<`` as always being False + +For ``Set of String`` fields, as there is no marker syntax for set literals, +the only valid operations are ``in`` and ``not in`` comparisons with a user +supplied string literal as the left operand. Publishing tools SHOULD emit an +error if environment markers attempt to use any other comparison operations on +these fields and index servers MAY disallow uploads containing such environment +markers, while locking and installation tools SHOULD treat such operations as +always being False. + +For ``Version`` fields, the comparison operations are defined by the +:ref:`Version specifier specification ` when either both +the marker field value and the user supplied constant can be parsed as valid +version specifiers or the ``===`` arbitrary equivalence comparison operator +is used. When an operator other than ``===`` is used, publishing tools SHOULD +emit an error if the user supplied constant cannot be parsed as a valid version +specifier, index servers MAY disallow uploads containing such environment +markers, while locking and installation tools MAY either emit an error or else +fall back to ``String`` field comparison logic if either the marker field value +or the user supplied constant cannot be parsed as a valid version specifier. +Note that ``in`` and ``not in`` containment checks are NOT valid for ``Version`` +fields and publishing tools SHOULD emit an error, index servers MAY disallow +uploads containing such environment markers, while locking and installation +tools MAY treat them as always being False. + +For ``Version | String`` fields, comparison operations are defined as they are +for ``Version`` fields, while ``in`` and ``not in`` containment checks are +defined as they are for ``String`` fields. However, there is no consistent +cross-platform expectation that the parsing of the marker field value or the +user supplied constant as a valid version will succeed, so tools SHOULD fall +back to processing the field as a ``String`` field if parsing either value as a +version fails. Tools MAY emit a warning if the field is expected to contain a +valid version on a given platform but does not in fact do so. Tools SHOULD NOT +unconditionally treat such fields as ``String`` fields, as doing so may give +incorrect answers for environment markers that are appropriately scoped +to the relevant platforms before performing a version based comparison. + + +Composing marker expressions +'''''''''''''''''''''''''''' + +More complex marker expressions may be composed using the ``and`` and ``or`` +logical operators. Parentheses may be used as necessary to control operand +precedence (with all comparison operations having a higher precedence). + +For example:: + + sys_platform == "ios" or sys_platform == "darwin" + sys_platform == "linux" and "SMP" in platform_version + sys_platform == "darwin" and platform_release >= "12" + +Python's comparison chaining (such as ``3.4 < python_version < 3.9``) is NOT +supported in environment markers (such expressions must instead be written out +as two separate comparisons joined by ``and``). + +User supplied constants +''''''''''''''''''''''' + +User supplied constants are always given as strings within either ``'`` or +``"`` quote marks. Triple-quoted multi-line strings are NOT permitted. + +Backslash escapes are not specified, although tools MAY support them. +They are not included in the specification because they add complexity and +there is currently no known need for treating user supplied constants as +anything other than either opaque strings or valid version specifiers. + +Similarly, non-ASCII character support is not specified, but tools MAY accept +them (usually based on the text encoding of the file or stream containing the +dependency specifier). This may be revisited in the future if it becomes more +common for the runtime variables typically referenced in environment markers to +contain non-ASCII text that users wish to perform comparisons against. + +Unknown marker fields +''''''''''''''''''''' + +References to unknown marker fields SHOULD render a package version ineligible +for installation or inclusion in a locked dependency tree rather than resulting +in a comparison that evaluates to True or False. This is so that published +package versions with unknown marker fields are either ignored when resolving +dependencies or emit a descriptive installation failure, rather than producing +an apparently successful installation that then fails at runtime due to missing +dependencies (if the unknown marker is treated as False) or a potentially +cryptic installation failure of a dependency that is not valid for the +current platform (if the unknown marker is treated as True). Variables whose value cannot be calculated on a given Python implementation -should evaluate to ``0`` for versions, and an empty string for all other -variables. +should evaluate to ``0`` for ``Version`` fields, and an empty string for all +other variables (including ``Version | String`` fields). + +.. _dependency-specifiers-environment-marker-fields: +.. _environment-marker-fields: -The "extra" variable is special. It is used by wheels to signal which -specifications apply to a given extra in the wheel ``METADATA`` file, but -since the ``METADATA`` file is based on a draft version of :pep:`426`, there is -no current specification for this. Regardless, outside of a context where this -special handling is taking place, the "extra" variable should result in an -error like all other unknown variables. +Defined environment marker fields +''''''''''''''''''''''''''''''''' -The "extras" and "dependency_groups" variables are also special. They are used -to specify any requested extras or dependency groups when installing from a lock -file. Outside of the context of lock files, these two variables should result in -an error like all other unknown variables. +Unless otherwise noted below, marker evaluation environments MUST support all +of the following marker fields: .. list-table:: :header-rows: 1 @@ -247,7 +363,7 @@ an error like all other unknown variables. * - Marker - Python equivalent - Type - - Sample values + - Sample values & notes * - ``os_name`` - :py:data:`os.name` - String @@ -255,61 +371,110 @@ an error like all other unknown variables. * - ``sys_platform`` - :py:data:`sys.platform` - String - - ``linux``, ``linux2``, ``darwin``, ``java1.8.0_51`` (note that "linux" - is from Python3 and "linux2" from Python2) + - ``linux``, ``win32``, ``darwin``, ``java1.8.0_51`` + (note that this is the most well defined field for use when declaring + platform specific dependencies) * - ``platform_machine`` - :py:func:`platform.machine()` - String - - ``x86_64`` + - ``x86_64``, ``aarch64``, ``AMD64``, ``arm64`` + (note that this value is provided by the operating system, so the same + CPU architecture may use different strings on different platforms) * - ``platform_python_implementation`` - :py:func:`platform.python_implementation()` - String - - ``CPython``, ``Jython`` + - ``CPython``, ``PyPy`` * - ``platform_release`` - :py:func:`platform.release()` - - String + - Version | String - ``3.14.1-x86_64-linode39``, ``14.5.0``, ``1.8.0_51`` + (may be a valid version field, for example on macOS/darwin) * - ``platform_system`` - :py:func:`platform.system()` - String - ``Linux``, ``Windows``, ``Java`` * - ``platform_version`` - :py:func:`platform.version()` - - String + - Version | String - ``#1 SMP Fri Apr 25 13:07:35 EDT 2014`` ``Java HotSpot(TM) 64-Bit Server VM, 25.51-b03, Oracle Corporation`` ``Darwin Kernel Version 14.5.0: Wed Jul 29 02:18:53 PDT 2015; root:xnu-2782.40.9~2/RELEASE_X86_64`` + ``13`` + (may be a valid version field, for example on iOS or Android) * - ``python_version`` - ``'.'.join(platform.python_version_tuple()[:2])`` - :ref:`Version ` - - ``3.4``, ``2.7`` + - ``3.9``, ``3.15`` * - ``python_full_version`` - :py:func:`platform.python_version()` - :ref:`Version ` - - ``3.4.0``, ``3.5.0b1`` + - ``3.10.12``, ``3.15.0a1`` * - ``implementation_name`` - :py:data:`sys.implementation.name ` - String - - ``cpython`` + - ``cpython``, ``pypy`` * - ``implementation_version`` - see definition below - :ref:`Version ` - - ``3.4.0``, ``3.5.0b1`` + - ``3.10.12``, ``7.3.17`` + (examples are for CPython and PyPy respectively) * - ``extra`` - - An error except when defined by the context interpreting the - specification. - - String + - Used to indicate optional dependencies in project dependency metadata. + An error except when defined by the context interpreting the + specifier. + - Special (see below) - ``toml`` + (publishing tools SHOULD permit use of this field) * - ``extras`` - - An error except when defined by the context interpreting the - specification. + - Used to indicate optional public dependencies in lock files. An error + except when defined by the context interpreting the specifier. - Set of strings - ``{"toml"}`` + (publishing tools SHOULD NOT permit use of this field and index servers + SHOULD NOT accept uploads containing such environment markers) * - ``dependency_groups`` - - An error except when defined by the context interpreting the - specification. + - Used to indicate optional project internal dependencies in lock files. + An error except when defined by the context interpreting the + specifier. - Set of strings - ``{"test"}`` + (publishing tools SHOULD NOT permit use of this field and index servers + SHOULD NOT accept uploads containing such environment markers) + +For backwards compatibility with older locking and installation tools, the +``extras`` and ``dependency_groups`` fields are currently only valid for use in +``packages.marker`` fields in :ref:`lock files `. For these +comparisons, the ``extras`` and ``dependency_groups`` sets used for the marker +evaluation refer to the *currently selected* extras and dependency groups when +installing from the lock file, not the full set of defined extras and dependency +groups listed in the corresponding top level lock file fields. The interface +for selecting which extras and dependency groups to install is tool dependent. +Publishing tools SHOULD emit an error if projects attempt to reference the +``extras`` or ``dependency_groups`` fields in their published dependency +declaration metadata, and index servers SHOULD NOT accept uploads referencing +these fields. Outside lock file processing, marker evaluation environments +DO NOT need to define these fields. + +The ``extra`` field is also special, as it expects set-like behaviour, but +predates the addition of ``Set of strings`` as a defined marker field type. +Accordingly, ``extra == "name"`` in a dependency declaration is similar to +``"name" in extras``, while ``extra != "name"`` is similar to +``"name" not in extras``. For dependency marker evaluations, the set of extra +names used for these comparisons is the full set of requested extras for *that +particular package*, whether requested directly in a top level dependency +declaration, or indirectly in a transitive dependency declaration. Other +comparison operations on ``extra`` are not defined and publishing tools SHOULD +emit an error, index servers MAY disallow uploads containing such environment +markers, while locking and installation tools SHOULD evaluate them as False. + +Unlike the newer ``extras`` field, environment markers using this field SHOULD +be accepted by both publishing tools and index servers. Marker evaluation +environments intended for project dependency declarations will typically need +to handle evaluation of ``extra`` field comparisons, while other evaluations +of environment markers will not generally need to do so. The legacy ``extra`` +comparison syntax is NOT permitted in lock file ``packages.marker`` fields, +and installation tools SHOULD reject lock files containing such comparisons as +invalid. The ``implementation_version`` marker variable is derived from :py:data:`sys.implementation.version `: @@ -328,9 +493,6 @@ The ``implementation_version`` marker variable is derived from else: implementation_version = "0" -This environment markers section, initially defined through :pep:`508`, supersedes the environment markers -section in :pep:`345`. - .. _dependency-specifiers-grammar: Complete Grammar @@ -339,7 +501,7 @@ Complete Grammar The complete parsley grammar:: wsp = ' ' | '\t' - version_cmp = wsp* <'<=' | '<' | '!=' | '==' | '>=' | '>' | '~=' | '==='> + version_cmp = wsp* <'<=' | '<' | '!=' | '===' | '==' | '>=' | '>' | '~='> version = wsp* <( letterOrDigit | '-' | '_' | '.' | '*' | '+' | '!' )+> version_one = version_cmp:op version:v wsp* -> (op, v) version_many = version_one:v1 (',' version_one)*:v2 (',' wsp*)? -> [v1] + v2 @@ -512,6 +674,8 @@ A test program - if the grammar is in a string ``grammar``: print("%s -> %s" % (test, parsed)) +.. _dependency-specifier-history: + History ======= @@ -521,14 +685,28 @@ History ``'.'.join(platform.python_version_tuple()[:2])``, to accommodate potential future versions of Python with 2-digit major and minor versions (e.g. 3.10). [#future_versions]_ +- March 2022: Standardised the normalization of extra names at publication time + (for core metadata 2.3 and later) through :pep:`685` - June 2024: The definition of ``version_many`` was changed to allow trailing commas, matching with the behavior of the Python implementation that has been in use since late 2022. -- April 2025: Added ``extras`` and ``dependency_groups`` for +- April 2025: Added ``extras`` and ``dependency_groups`` marker field for :ref:`lock-file-spec` as approved through :pep:`751`. - August 2025: The suggested name validation regex was fixed to match the field specification (it previously finished with ``$`` instead of ``\Z``, incorrectly permitting trailing newlines) +- December 2025: Ensure ``===`` is before ``==`` in grammar, to allow arbitrary + equality comparisons to be parsed. +- January 2026: Amend the definition of environment marker comparison operations + to restrict version comparison semantics to fields where they make sense, + make extra name restrictions more explicit, adjust the way ordered comparisons + are defined for strings, and make the fallback from version comparisons to + string comparisons when version parsing fails optional. Also provide different + tool behaviour recommendations for publishing tools vs installation tools. + This brought the nominal specification into line with the way tools actually + work. [#marker_comparison_logic]_ +- January 2026: fix outdated references to other documents that were + inadvertently retained from :pep:`508` References @@ -544,6 +722,9 @@ References definition of Environment Marker Variable ``python_version`` (https://github.com/python/peps/issues/560) +.. [#marker_comparison_logic] Resolving inconsistencies between actual tool + behavior and the nominal definitions of environment marker field comparisons + (https://discuss.python.org/t/spec-change-bugfix-dependency-specifiers-simplification-pep-508/105203) .. _python-version-change: https://mail.python.org/pipermail/distutils-sig/2018-January/031920.html diff --git a/source/specifications/direct-url-data-structure.rst b/source/specifications/direct-url-data-structure.rst index 5f3af0fae..a82537f0a 100644 --- a/source/specifications/direct-url-data-structure.rst +++ b/source/specifications/direct-url-data-structure.rst @@ -236,7 +236,7 @@ JSON Schema The following JSON Schema can be used to validate the contents of ``direct_url.json``: -.. literalinclude:: schemas/direct-url.schema.json +.. literalinclude:: ../../extra/specifications/schemas/direct-url.schema.json Examples ======== diff --git a/source/specifications/entry-points.rst b/source/specifications/entry-points.rst index dea039492..9e59862aa 100644 --- a/source/specifications/entry-points.rst +++ b/source/specifications/entry-points.rst @@ -106,8 +106,7 @@ Within a value, readers must accept and ignore spaces (including multiple consecutive spaces) before or after the colon, between the object reference and the left square bracket, between the extra names and the square brackets and colons delimiting them, and after the right square bracket. The syntax for -extras is formally specified as part of :pep:`508` (as ``extras``) and -restrictions on values specified in :pep:`685`. +extras is formally specified in :ref:`dependency-specifiers`. For tools writing the file, it is recommended only to insert a space between the object reference and the left square bracket. @@ -166,6 +165,8 @@ History - October 2017: This specification was written to formalize the existing entry points feature of setuptools (discussion_). +- January 2026: Replaced outdated direct references to :pep:`508` and + :pep:`685` with a reference to :ref:`dependency-specifiers`. .. _discussion: https://mail.python.org/pipermail/distutils-sig/2017-October/031585.html diff --git a/source/specifications/glob-patterns.rst b/source/specifications/glob-patterns.rst index abdb15b0f..8ff3f09fb 100644 --- a/source/specifications/glob-patterns.rst +++ b/source/specifications/glob-patterns.rst @@ -15,8 +15,8 @@ Valid glob patterns For PyPA purposes, a *valid glob pattern* MUST be a string matched against filesystem entries as specified below: -- Alphanumeric characters, underscores (``_``), hyphens (``-``) and dots (``.``) - MUST be matched verbatim. +- Alphanumeric characters, spaces (`` ``), underscores (``_``), hyphens (``-``), + and dots (``.``) MUST be matched verbatim. - Special glob characters: ``*``, ``?``, ``**`` and character ranges: ``[]`` containing only the verbatim matched characters MUST be supported. @@ -107,9 +107,15 @@ The code below is as a simple reference implementation: raise ValueError( f"Pattern {pattern!r} should be relative and must not start with '/'" ) - if re.match(r'^[\w\-\.\/\*\?\[\]]+$', pattern) is None: + if re.match(r'^[\w \-\.\/\*\?\[\]]+$', pattern) is None: raise ValueError(f"Pattern '{pattern}' contains invalid characters.") found = glob(pattern, recursive=True) if not found: raise ValueError(f"Pattern '{pattern}' did not match any files.") return found + +History +======= + +- January 2025: Initial version +- March 2026: Treat spaces as a verbatim character diff --git a/source/specifications/inline-script-metadata.rst b/source/specifications/inline-script-metadata.rst index 6fa832a3e..f9df2f0f5 100644 --- a/source/specifications/inline-script-metadata.rst +++ b/source/specifications/inline-script-metadata.rst @@ -70,6 +70,17 @@ and the regular expression, the text specification takes precedence. Tools MUST NOT read from metadata blocks with types that have not been standardized by this specification. +Note that the specification only requires that *top-level* comment blocks are +recognised as containing metadata. However, parsing Python code is non-trivial, +and therefore: + +* Tools MAY choose to do a simple textual scan, rather than a full Python parse. +* As a result of the previous point, the behaviour of scripts that contain data + that looks like metadata within another Python construct such as a multi-line + string is tool-dependent and should not be relied on. +* The canonical regular expression provided above is an example of an + implementation that does a simple textual scan. + script type ----------- diff --git a/source/specifications/platform-compatibility-tags.rst b/source/specifications/platform-compatibility-tags.rst index 0502c8c03..b4c14a4c0 100644 --- a/source/specifications/platform-compatibility-tags.rst +++ b/source/specifications/platform-compatibility-tags.rst @@ -82,6 +82,11 @@ decide how to best use the ABI tag. Platform Tag ============ +.. important:: + Platform tags are dependent on the versioning of the operating system or + platform they represent and may change over time as the underlying platform + changes its versioning. + Basic platform tags ------------------- @@ -199,12 +204,13 @@ artefact of Apple's official macOS naming scheme). The schema for compatibility tags is :file:`macosx_{x}_{y}_{arch}`, indicating that the wheel is compatible with macOS ``x.y`` or later on the architecture ``arch``. -The values of ``x`` and ``y`` correspond to the major and minor version number of -the macOS release, respectively. They must both be positive integers, with the -``x`` value being ``>= 10``. The version number always includes a major *and* -minor version, even if Apple's official version numbering only refers to -the major value. For example, ``macosx_11_0_arm64`` indicates compatibility -with macOS 11 or later. +For macOS 10, the tag is :file:`macosx_10_{y}_{arch}`, where ``y`` corresponds +to the minor version number of the macOS release. For macOS 11 and higher, the +tag is :file:`macosx_{x}_0_{arch}`, where ``x`` corresponds to the major +version number of the macOS release. Following the published macOS major +versions, the ``x`` value is either ``10 <= x <= 15``, or ``>=26`` and +corresponding to the year of the macOS release. For example, +``macosx_11_0_arm64`` indicates compatibility with macOS 11 or later. macOS binaries can be compiled for a single architecture, or can include support for multiple architectures in the same binary (sometimes called "fat" binaries). diff --git a/source/specifications/pylock-toml.rst b/source/specifications/pylock-toml.rst index 342e608c5..394f9a206 100644 --- a/source/specifications/pylock-toml.rst +++ b/source/specifications/pylock-toml.rst @@ -449,7 +449,10 @@ See :ref:`pylock-packages-vcs-subdirectory`. ``packages.archive.url`` '''''''''''''''''''''''' -See :ref:`pylock-packages-vcs-url`. +- **Type**: string +- **Required?**: if :ref:`pylock-packages-archive-path` is not specified +- **Inspiration**: :ref:`direct-url-data-structure-archive` +- The URL_ to the archive. .. _pylock-packages-archive-path: @@ -457,7 +460,15 @@ See :ref:`pylock-packages-vcs-url`. ``packages.archive.path`` ''''''''''''''''''''''''' -See :ref:`pylock-packages-vcs-path`. +- **Type**: string +- **Required?**: if :ref:`pylock-packages-archive-url` is not specified +- **Inspiration**: :ref:`direct-url-data-structure-archive` +- The path to the archive. +- If a relative path is used it MUST be relative to the location of this file. +- If the path is relative it MAY use POSIX-style path separators explicitly + for portability. +- If :ref:`pylock-packages-archive-url` is also specified, the filename as + specified by this key takes precedence. .. _pylock-packages-archive-size: @@ -554,6 +565,8 @@ See :ref:`pylock-packages-vcs-subdirectory`. the same value - **Inspiration**: PDM_, Poetry_, uv_ - The file name of the :ref:`source-distribution-format-sdist` file. +- If specified, this key's value takes precedence over the file name found in + either :ref:`pylock-packages-sdist-url` or :ref:`pylock-packages-sdist-path`. .. _pylock-packages-sdist-upload-time: @@ -623,6 +636,8 @@ See :ref:`pylock-packages-archive-hashes`. the same value - **Inspiration**: PDM_, Poetry_, uv_ - The file name of the :ref:`binary-distribution-format` file. +- If specified, this key's value takes precedence over the file name found in + either :ref:`pylock-packages-wheels-url` or :ref:`pylock-packages-wheels-path`. .. _pylock-packages-wheels-upload-time: @@ -826,6 +841,7 @@ History ------- - April 2025: Initial version, approved via :pep:`751`. +- March 2026: Clarify file name precedence for archives, sdists, and wheels. .. _Content-Length: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Length diff --git a/source/specifications/pylock-toml/pylock.example.toml b/source/specifications/pylock-toml/pylock.example.toml index 45e8731b2..8a439cd7a 100644 --- a/source/specifications/pylock-toml/pylock.example.toml +++ b/source/specifications/pylock-toml/pylock.example.toml @@ -1,6 +1,6 @@ lock-version = '1.0' environments = ["sys_platform == 'win32'", "sys_platform == 'linux'"] -requires-python = '== 3.12' +requires-python = '== 3.12.*' created-by = 'mousebender' [[packages]] diff --git a/source/specifications/pypirc.rst b/source/specifications/pypirc.rst index aeba72b0d..b937a1d62 100644 --- a/source/specifications/pypirc.rst +++ b/source/specifications/pypirc.rst @@ -10,6 +10,11 @@ indexes ` (referred to here as "repositories"), so that you don't have to enter the URL, username, or password whenever you upload a package with :ref:`twine` or :ref:`flit`. +The :file:`.pypirc` file **SHOULD** be UTF-8 encoded. + +Tools that read or write :file:`.pypirc` files may not function correctly +if another character encoding is used. + The format (originally defined by the :ref:`distutils` package) is: .. code-block:: ini diff --git a/source/specifications/pyproject-toml.rst b/source/specifications/pyproject-toml.rst index 4ce9b7484..695b6e7f7 100644 --- a/source/specifications/pyproject-toml.rst +++ b/source/specifications/pyproject-toml.rst @@ -14,8 +14,6 @@ The ``pyproject.toml`` file acts as a configuration file for packaging-related tools (as well as other tools). -.. note:: This specification was originally defined in :pep:`518` and :pep:`621`. - The ``pyproject.toml`` file is written in `TOML `_. Three tables are currently specified, namely :ref:`[build-system] `, @@ -61,6 +59,10 @@ table then the default values as specified above should be used. If the table is specified but is missing required fields then the tool should consider it an error. +Tools may choose to present an error to the user if the file exists, +``[build-system]`` table is missing, and there is no clear indication +that the project should be built (e.g., no setup.py/setup.cfg or other +build configuration files, and no ``[project]`` table). To provide a type-specific representation of the resulting data from the TOML file for illustrative purposes only, the following @@ -112,6 +114,13 @@ by the metadata). Dynamic metadata is listed via the ``dynamic`` key (defined later in this specification) and represents metadata that a tool will later provide. +A key whose value is a list or a table of arbitrary entries MAY be +specified statically *and* listed in ``dynamic`` at the same time. In +that case the entries given statically are fixed and a build back-end +MAY only *append* further entries to them; the back-end MUST NOT +remove, reorder, or modify any statically-specified entries. See the +:ref:`dynamic ` key for details. + The lack of a ``[project]`` table implicitly means the :term:`build backend ` will dynamically provide all keys. @@ -136,6 +145,8 @@ The complete list of keys allowed in the ``[project]`` table are: - ``dynamic`` - ``entry-points`` - ``gui-scripts`` +- ``import-names`` +- ``import-namespaces`` - ``keywords`` - ``license`` - ``license-files`` @@ -211,7 +222,7 @@ If the file path ends in a case-insensitive ``.md`` suffix, then tools MUST assume the content-type is ``text/markdown``. If the file path ends in a case-insensitive ``.rst``, then tools MUST assume the content-type is ``text/x-rst``. If a tool recognizes more extensions -than this PEP, they MAY infer the content-type for the user without +than this specification, it MAY infer the content-type for the user without specifying this key as ``dynamic``. For all unrecognized suffixes when a content-type is not provided, tools MUST raise an error. @@ -259,6 +270,12 @@ Text string that is a valid SPDX as specified in :doc:`/specifications/license-expression`. Tools SHOULD validate and perform case normalization of the expression. +This key should **only** be specified if the license expression for any +and all distribution files created by a build backend using the +:file:`pyproject.toml` is the same as the one specified. If the license +expression will differ then it should either be specified as dynamic or +not set at all. + Legacy specification '''''''''''''''''''' @@ -437,34 +454,148 @@ be ambiguous in the face of ``[project.scripts]`` and .. _pyproject-toml-dependencies: -.. _pyproject-toml-optional-dependencies: -``dependencies``/``optional-dependencies`` ------------------------------------------- +``dependencies`` +---------------- -- TOML_ type: Array of :pep:`508` strings (``dependencies``), and a - table with values of arrays of :pep:`508` strings - (``optional-dependencies``) +- TOML_ type: Array of :ref:`dependency specifier ` + strings (``dependencies``) - Corresponding :ref:`core metadata ` field: - :ref:`Requires-Dist ` and - :ref:`Provides-Extra ` + :ref:`Requires-Dist ` -The (optional) dependencies of the project. +``dependencies`` lists the expected dependencies of the project as an +array of strings. -For ``dependencies``, it is a key whose value is an array of strings. Each string represents a dependency of the project and MUST be -formatted as a valid :pep:`508` string. Each string maps directly to -a :ref:`Requires-Dist ` entry. +formatted as a valid :ref:`dependency specifier `. + +Each string maps directly to a +:ref:`Requires-Dist ` entry. + +Dependencies listed in this array are always considered +for installation, but may still contain environment markers that cause them +to be skipped in some environments. + -For ``optional-dependencies``, it is a table where each key specifies -an extra and whose value is an array of strings. The strings of the -arrays must be valid :pep:`508` strings. The keys MUST be valid values +.. _pyproject-toml-optional-dependencies: + +``optional-dependencies`` +------------------------- + +- TOML_ type: table with string keys mapping to arrays of + :ref:`dependency specifier ` strings (``optional-dependencies``) +- Corresponding :ref:`core metadata ` fields: + :ref:`Requires-Dist ` and + :ref:`Provides-Extra ` + +``optional-dependencies`` is a table where each key specifies +an extra and whose value is an array of strings using the same format as the +``dependencies`` array (the strings in the +arrays must be valid :ref:`dependency specifiers `). + +The keys MUST be valid values for :ref:`Provides-Extra `. Each value in the array thus becomes a corresponding :ref:`Requires-Dist ` entry for the matching :ref:`Provides-Extra ` metadata. +The optionality of these dependencies is recorded by modifying the environment +marker clause on the related ``Requires-Dist`` entries to check the extra name. +Optional dependencies are thus only considered for installation if installation +if the associated extra name is requested. + + +.. _pyproject-toml-import-names: + +``import-names`` +---------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Import-Name ` + +An array of strings specifying the import names that the project exclusively +provides when installed. Each string MUST be a valid Python identifier or can +be empty. An import name MAY be followed by a semicolon and the term "private" +(e.g. ``"; private"``) with any amount of whitespace surrounding the semicolon. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name should also be listed appropriately +in ``import-names`` and/or ``import-namespaces``. For instance, a project which +is a single package named spam with multiple submodules would only list +``project.import-names = ["spam"]``. A project that lists ``spam.bacon.eggs`` +would also need to account for ``spam`` and ``spam.bacon`` appropriately in +``import-names`` and ``import-namespaces``. Listing all names acts as a check +that the intent of the import names is as expected. As well, projects SHOULD +list all import names, public or private, using the ``; private`` modifier as +appropriate. + +If a project lists the same name in both ``import-names`` and +``import-namespaces``, then tools MUST raise an error due to ambiguity. + +Projects MAY set ``import-names`` to an empty array to represent a project with +no import names (i.e. there are no Python modules of any kind in the +distribution file). + +Build back-ends MAY support dynamically calculating the value if the user +declares the key in ``project.dynamic``. + +Examples: + +.. code-block:: toml + + [project] + name = "pillow" + import-names = ["PIL"] + +.. code-block:: toml + + [project] + name = "myproject" + import-names = ["mypackage", "_private_module ; private"] + + +.. _pyproject-toml-import-namespaces: + +``import-namespaces`` +--------------------- + +- TOML_ type: array of strings +- Corresponding :ref:`core metadata ` field: + :ref:`Import-Namespace ` + +An array of strings specifying the import names that the project provides when +installed, but not exclusively. Each string MUST be a valid Python identifier. +An import name MAY be followed by a semicolon and the term "private" (e.g. +``"; private"``) with any amount of whitespace surrounding the semicolon. Note +that unlike ``import-names``, ``import-namespaces`` CANNOT be an empty array. + +Projects SHOULD list all the shortest import names that are exclusively provided +by the project. If any of the shortest names are dotted names, all intervening +names from that name to the top-level name should also be listed appropriately +in ``import-names`` and/or ``import-namespaces``. + +This field is used for namespace packages where multiple projects can contribute +to the same import namespace. Projects all listing the same import name in +``import-namespaces`` can be installed together without shadowing each other. + +If a project lists the same name in both ``import-names`` and +``import-namespaces``, then tools MUST raise an error due to ambiguity. + +Build back-ends MAY support dynamically calculating the value if the user +declares the key in ``project.dynamic``. + +Example: + +.. code-block:: toml + + [project] + name = "zope-interface" + import-namespaces = ["zope"] + import-names = ["zope.interface"] + .. _pyproject-toml-dynamic: .. _declaring-project-metadata-dynamic: @@ -495,8 +626,9 @@ provided via tooling later on. field as "Optional", the metadata MAY list it in ``dynamic`` if the expectation is a build back-end will provide the data for the key later. -- Build back-ends MUST raise an error if the metadata specifies a - key statically as well as being listed in ``dynamic``. +- Build back-ends MUST raise an error if the metadata specifies a key + statically as well as being listed in ``dynamic``, *unless* the key + represents a list or arbitrary table that can be extended, listed below. - If the metadata does not list a key in ``dynamic``, then a build back-end CANNOT fill in the requisite metadata on behalf of the user (i.e. ``dynamic`` is the only way to allow a tool to fill in @@ -506,6 +638,35 @@ provided via tooling later on. the data for it (omitting the data, if determined to be the accurate value, is acceptable). +A key whose value is a list or a table of arbitrary entries MAY be +specified statically and listed in ``dynamic`` simultaneously. The +keys fitting that description are: + +- ``authors`` +- ``classifiers`` +- ``dependencies`` +- ``entry-points`` +- ``gui-scripts`` +- ``import-names`` +- ``import-namespaces`` +- ``keywords`` +- ``license-files`` +- ``maintainers`` +- ``optional-dependencies`` +- ``scripts`` +- ``urls`` + +When such a key is specified both statically and listed in +``dynamic``: + +- A build back-end MAY only *append* entries to the value; it MUST NOT + remove, reorder, or modify any statically-specified entries. For + tables (such as ``optional-dependencies`` or ``entry-points``) this + means a back-end MAY add new keys and MAY append to the values of + existing keys (in the case of a list), but MUST NOT change or remove the + entries given statically. +- A build back-end SHOULD raise an error if a key is listed in + ``dynamic`` and it does not support extending that key. .. _pyproject-tool-table: @@ -540,5 +701,17 @@ History - December 2024: The ``license`` key was redefined, the ``license-files`` key was added and ``License::`` classifiers were deprecated through :pep:`639`. +- September 2025: Clarity that the ``license`` key applies to all distribution + files generated from the :file:`pyproject.toml` file. + +- October 2025: The ``import-names`` and ``import-namespaces`` keys were added + through :pep:`794`. + +- January 2026: Replaced outdated direct reference to :pep:`508` with a + reference to :ref:`dependency-specifiers`. + +- May 2026: Allowed list and table keys to be specified statically as well + as listed in ``dynamic``, with build back-ends only able to append + entries, through :pep:`808`. .. _TOML: https://toml.io diff --git a/source/specifications/simple-repository-api.rst b/source/specifications/simple-repository-api.rst index 4f5bb0043..d7e14a4d3 100644 --- a/source/specifications/simple-repository-api.rst +++ b/source/specifications/simple-repository-api.rst @@ -122,6 +122,15 @@ HTML Serialization .. _simple-repository-html-project-list: +.. important:: + + The HTML representation is considered "frozen" and is not expected + to be updated. Producers and consumers of the simple API + should prefer the :ref:`JSON representation `. + + See :pep:`833` for additional information about the HTML representation's + status. + The following constraints apply to all HTML serialized responses described in this spec: @@ -477,16 +486,12 @@ The format of this URL is ``//`` where the ```` is replaced by name for that project, so a project named "Silly_Walk" would have a URL like ``/silly-walk/``. -This URL must respond with a JSON encoded dictionary that has four keys: +This URL must respond with a JSON encoded dictionary that has five keys: - ``name``: The normalized name of the project. -- ``files``: A list of dictionaries, each one representing an individual file. -- ``meta``: The general response metadata as `described earlier `__. +- ``project-status``: An optional dictionary, containing the following: - In addition to the general response metadata, the project detail ``meta`` - dictionary **MAY** also include the following: - - - ``project-status``: If present, this **MUST** be a valid project status marker. + - ``status``: If present, this **MUST** be a valid project status marker. .. note:: @@ -495,15 +500,21 @@ This URL must respond with a JSON encoded dictionary that has four keys: .. note:: - The ``project-status`` key was added with API version 1.4. + The ``status`` key was added with API version 1.4. - - ``project-status-reason``: If present, this **MUST** be an arbitrary string - description of the project status. + - ``reason``: If present, this **MUST** be an arbitrary string description + of the project status. .. note:: - The ``project-status-reason`` key was added with API version 1.4. + The ``reason`` key was added with API version 1.4. + + .. note:: + The ``project-status`` key was added with API version 1.4. + +- ``files``: A list of dictionaries, each one representing an individual file. +- ``meta``: The general response metadata as `described earlier `__. - ``versions``: A list of version strings specifying all of the project versions uploaded for this project. The value of ``versions`` is logically a set, and as such may not contain duplicates, and the order of the versions is @@ -910,46 +921,6 @@ which version+format a specific repository URL was configured for, and when maki a request to that server, emit an ``Accept`` header that *only* includes the correct content type. - -TUF Support - PEP 458 ---------------------- - -:pep:`458` requires that all API responses are hashable and that they can be uniquely -identified by a path relative to the repository root. For a Simple API repository, the -target path is the Root of our API (e.g. ``/simple/`` on PyPI). This creates -challenges when accessing the API using a TUF client instead of directly using a -standard HTTP client, as the TUF client cannot handle the fact that a target could -have multiple different representations that all hash differently. - -:pep:`458` does not specify what the target path should be for the Simple API, but -TUF requires that the target paths be "file-like", in other words, a path like -``simple/PROJECT/`` is not acceptable, because it technically points to a -directory. - -The saving grace is that the target path does not *have* to actually match the URL -being fetched from the Simple API, and it can just be a sigil that the fetching code -knows how to transform into the actual URL that needs to be fetched. This same thing -can hold true for other aspects of the actual HTTP request, such as the ``Accept`` -header. - -Ultimately figuring out how to map a directory to a filename is out of scope for this -spec (but it would be in scope for :pep:`458`), and this spec defers making a decision -about how exactly to represent this inside of :pep:`458` metadata. - -However, it appears that the current WIP branch against pip that attempts to implement -:pep:`458` is using a target path like ``simple/PROJECT/index.html``. This could be -modified to include the API version and serialization format using something like -``simple/PROJECT/vnd.pypi.simple.vN.FORMAT``. So the v1 HTML format would be -``simple/PROJECT/vnd.pypi.simple.v1.html`` and the v1 JSON format would be -``simple/PROJECT/vnd.pypi.simple.v1.json``. - -In this case, since ``text/html`` is an alias to ``application/vnd.pypi.simple.v1+html`` -when interacting through TUF, it likely will make the most sense to normalize to the -more explicit name. - -Likewise the ``latest`` metaversion should not be included in the targets, only -explicitly declared versions should be supported. - Recommendations --------------- @@ -1027,3 +998,4 @@ History * November 2024: provenance metadata in the HTML and JSON formats, in :pep:`740` * July 2025: project status markers in the HTML and JSON formats, in :pep:`792` * July 2025: layout changes (dedicated page for file yanking, introduce concepts before API details) +* June 2026: :pep:`833` formally "freezes" the HTML representation of the simple API diff --git a/source/specifications/source-distribution-format.rst b/source/specifications/source-distribution-format.rst index 9ac93be7b..b877e87d5 100644 --- a/source/specifications/source-distribution-format.rst +++ b/source/specifications/source-distribution-format.rst @@ -74,7 +74,7 @@ at their respective paths relative to the root directory of the sdist No other content of a sdist is required or defined. Build systems can store whatever information they need in the sdist to build the project. -The tarball should use the modern POSIX.1-2001 pax tar format, which specifies +The tarball must use the modern POSIX.1-2001 pax tar format, which specifies UTF-8 based file names. In particular, source distribution files must be readable using the standard library tarfile module with the open flag 'r:gz'. diff --git a/source/specifications/version-specifiers.rst b/source/specifications/version-specifiers.rst index c0b544160..e05422ce2 100644 --- a/source/specifications/version-specifiers.rst +++ b/source/specifications/version-specifiers.rst @@ -394,6 +394,21 @@ from an earlier epoch:: 1!1.1 1!2.0 +.. note:: + + Use of nonzero epochs is discouraged. They are often not supported or + discouraged by downstream packaging where Python packages may need to be + consumed, and due to their scarce use they may also not be well supported by + Python packaging tools. + + When version scheme needs to be changed, it is preferable to continue with + monotonically increasing numbers in epoch zero. For example, the version + 2026.x could be unambiguously followed by 3000.x. + + See `Discouraging use of epoch segments in versions + `__ + for the relevant discussion. + .. _version-specifiers-normalization: @@ -590,8 +605,8 @@ and post-releases for minor corrections:: 1.0.dev2 1.0.dev3 1.0.dev4 - 1.0c1 - 1.0c2 + 1.0rc1 + 1.0rc2 1.0 1.0.post1 1.1.dev1 @@ -1016,8 +1031,9 @@ Arbitrary equality Arbitrary equality comparisons are simple string equality operations which do not take into account any of the semantic information such as zero padding or -local versions. This operator also does not support prefix matching as the -``==`` operator does. +local versions. The comparison MUST treat ASCII letters case-insensitively, e.g. +by lowercasing, and is unspecified for non-ASCII text. This operator also does +not support prefix matching as the ``==`` operator does. The primary use case for arbitrary equality is to allow for specifying a version which cannot otherwise be represented by this specification. This operator is @@ -1271,3 +1287,5 @@ History - August 2014: This specification was approved through :pep:`440`. - May 2025: Clarify that development releases are a form of pre-release when they are handled. +- Nov 2025: Make arbitrary equality case insensitivity explicit. +- Jan 2026: The use of epochs was discouraged. diff --git a/source/tutorials/installing-packages.rst b/source/tutorials/installing-packages.rst index 3a9aa23bb..4c9b95030 100644 --- a/source/tutorials/installing-packages.rst +++ b/source/tutorials/installing-packages.rst @@ -137,7 +137,7 @@ If that still doesn't allow you to run ``python -m pip``: `_ [1]_ * Run ``python get-pip.py``. [2]_ This will install or upgrade pip. - Additionally, it will install :ref:`setuptools` and :ref:`wheel` if they're + Additionally, it may install :ref:`setuptools` and :ref:`wheel` if they're not installed already. .. warning:: @@ -150,24 +150,23 @@ If that still doesn't allow you to run ``python -m pip``: software. -Ensure pip, setuptools, and wheel are up to date ------------------------------------------------- +Ensure pip is up to date +------------------------ -While ``pip`` alone is sufficient to install from pre-built binary archives, -up to date copies of the ``setuptools`` and ``wheel`` projects are useful -to ensure you can also install from source archives: +Make sure you have the latest features and fixes, and support for the latest +Python packaging specifications. .. tab:: Unix/macOS .. code-block:: bash - python3 -m pip install --upgrade pip setuptools wheel + python3 -m pip install --upgrade pip .. tab:: Windows .. code-block:: bat - py -m pip install --upgrade pip setuptools wheel + py -m pip install --upgrade pip Optionally, create a virtual environment ---------------------------------------- diff --git a/source/tutorials/packaging-projects.rst b/source/tutorials/packaging-projects.rst index f2c0851ba..4f69de20b 100644 --- a/source/tutorials/packaging-projects.rst +++ b/source/tutorials/packaging-projects.rst @@ -220,7 +220,7 @@ following this tutorial. your package will work on. For a complete list of classifiers, see https://pypi.org/classifiers/. - ``license`` is the :term:`SPDX license expression ` of - your package. + your :term:`Distribution Archive` files. - ``license-files`` is the list of glob paths to the license files, relative to the directory where :file:`pyproject.toml` is located. - ``urls`` lets you list any number of extra links to show on PyPI. @@ -250,12 +250,12 @@ if you'd like. Creating a LICENSE ------------------ -It's important for every package uploaded to the Python Package Index to include -a license. This tells users who install your package the terms under which they -can use your package. For help picking a license, see -https://choosealicense.com/. Once you have chosen a license, open -:file:`LICENSE` and enter the license text. For example, if you had chosen the -MIT license: +It's important for every :term:`Distribution Archive` uploaded to the Python +Package Index to include a license. This tells users who install your +:term:`Distribution Archive` the terms under which they can use it. For help +picking a license, see https://choosealicense.com/. Once you have chosen a +license, open :file:`LICENSE` and enter the license text. For example, if you +had chosen the MIT license: .. code-block:: text