diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml index 9b638ddd004609..0fe754bb071ea3 100644 --- a/.azure-pipelines/ci.yml +++ b/.azure-pipelines/ci.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +trigger: ['master', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./docs-steps.yml @@ -56,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.1f + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -69,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_CI_Tests - displayName: ManyLinux1 CI Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(build.sourceBranchName)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_CI_Tests displayName: Ubuntu CI Tests (coverage) dependsOn: Prebuild @@ -113,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1f + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -139,7 +104,7 @@ jobs: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(Build.SourceBranchName)-win32' testRunPlatform: win32 win64: diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml index 65f23eb62ee095..2d32e6d49bcc0e 100644 --- a/.azure-pipelines/pr.yml +++ b/.azure-pipelines/pr.yml @@ -1,18 +1,14 @@ variables: - manylinux: false coverage: false -resources: - containers: - - container: manylinux1 - image: pyca/cryptography-manylinux1:x86_64 +pr: ['master', '3.9', '3.8', '3.7'] jobs: - job: Prebuild displayName: Pre-build checks pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./prebuild-checks.yml @@ -24,7 +20,7 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['docs.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 steps: - template: ./docs-steps.yml @@ -56,12 +52,12 @@ jobs: condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.1f + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -69,37 +65,6 @@ jobs: dependencies: apt -- job: ManyLinux1_PR_Tests - displayName: ManyLinux1 PR Tests - dependsOn: Prebuild - condition: | - and( - and( - succeeded(), - eq(variables['manylinux'], 'true') - ), - eq(dependencies.Prebuild.outputs['tests.run'], 'true') - ) - - pool: - vmImage: ubuntu-16.04 - - container: manylinux1 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1' - testRunPlatform: manylinux1 - openssl_version: '' - - steps: - - template: ./posix-steps.yml - parameters: - dependencies: yum - sudo_dependencies: '' - xvfb: false - patchcheck: false - - - job: Ubuntu_Coverage_PR_Tests displayName: Ubuntu PR Tests (coverage) dependsOn: Prebuild @@ -113,12 +78,12 @@ jobs: ) pool: - vmImage: ubuntu-16.04 + vmImage: ubuntu-18.04 variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.1f + openssl_version: 1.1.1k steps: - template: ./posix-steps.yml @@ -139,7 +104,7 @@ jobs: matrix: win32: arch: win32 - buildOpt: + buildOpt: '-p Win32' testRunTitle: '$(System.PullRequest.TargetBranch)-win32' testRunPlatform: win32 win64: diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml index a460eb1bac8fe5..307510a40dd4e5 100644 --- a/.azure-pipelines/windows-release/msi-steps.yml +++ b/.azure-pipelines/windows-release/msi-steps.yml @@ -1,6 +1,12 @@ steps: - template: ./checkout.yml + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + - task: DownloadPipelineArtifact@1 displayName: 'Download artifact: doc' inputs: diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml index 07e343a0b4e0c7..f967cfdbe326f8 100644 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -105,19 +105,26 @@ jobs: clean: all steps: - - checkout: none + - template: ./checkout.yml - template: ./find-sdk.yml + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + - task: DownloadBuildArtifacts@0 displayName: 'Download Artifact: unsigned_msix' inputs: artifactName: unsigned_msix downloadPath: $(Build.BinariesDirectory) + # MSIX must be signed and timestamped simultaneously - powershell: | $failed = $true foreach ($retry in 1..3) { - signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix) if ($?) { $failed = $false break diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml index b100364820d95b..8dfea382c35622 100644 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -4,7 +4,7 @@ jobs: condition: and(succeeded(), eq(variables['DoNuget'], 'true')) pool: - vmImage: windows-2019 + name: 'Windows Release' workspace: clean: all @@ -36,6 +36,14 @@ jobs: nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive displayName: 'Create nuget package' + - powershell: | + gci *.nupkg | %{ + nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite + } + displayName: 'Sign nuget package' + workingDirectory: $(Build.ArtifactStagingDirectory) + condition: and(succeeded(), variables['SigningCertificate']) + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: nuget' inputs: diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml index 4d757ae8fca032..c21e1c9f2b0f9b 100644 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -26,6 +26,12 @@ jobs: - template: ./checkout.yml - template: ./find-sdk.yml + - powershell: | + $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; + Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" + displayName: 'Update signing description' + condition: and(succeeded(), not(variables['SigningDescription'])) + - powershell: | Write-Host "##vso[build.addbuildtag]signed" displayName: 'Add build tags' @@ -51,7 +57,7 @@ jobs: $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) $failed = $true foreach ($retry in 1..10) { - signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files if ($?) { $failed = $false break diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 4d80698eff39c4..eb6fc2c184236e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -74,8 +74,9 @@ Include/pytime.h @pganssle @abalkin # Parser/Pgen /Parser/pgen/ @pablogsal -/Parser/pegen/ @pablogsal -/Tools/peg_generator/ @pablogsal +/Parser/pegen/ @pablogsal @lysnikolaou +/Tools/peg_generator/ @pablogsal @lysnikolaou +/Lib/test/test_peg_generator/ @pablogsal @lysnikolaou # SQLite 3 **/*sqlite* @berkerpeksag diff --git a/.github/SECURITY.md b/.github/SECURITY.md index 28aea946623cc5..82ae4ca8c30977 100644 --- a/.github/SECURITY.md +++ b/.github/SECURITY.md @@ -10,9 +10,8 @@ https://devguide.python.org/#status-of-python-branches ## Reporting a Vulnerability Please read the guidelines on reporting security issues [on the -official website]( -https://www.python.org/news/security/#reporting-security-issues-in-python -) for instructions on how to report a security-related problem to +official website](https://www.python.org/dev/security/) for +instructions on how to report a security-related problem to the Python team responsibly. To reach the response team, email `security at python dot org`. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7c3bca3fc0671b..a42338edd59f50 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,11 +7,13 @@ on: push: branches: - master + - 3.9 - 3.8 - 3.7 pull_request: branches: - master + - 3.9 - 3.8 - 3.7 @@ -21,17 +23,83 @@ jobs: runs-on: ubuntu-latest outputs: run_tests: ${{ steps.check.outputs.run_tests }} + run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }} steps: - uses: actions/checkout@v2 - name: Check for source changes id: check run: | - if [ -z "GITHUB_BASE_REF" ]; then + if [ -z "$GITHUB_BASE_REF" ]; then echo '::set-output name=run_tests::true' + echo '::set-output name=run_ssl_tests::true' else git fetch origin $GITHUB_BASE_REF --depth=1 - git diff --name-only origin/$GITHUB_BASE_REF... | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc)' && echo '::set-output name=run_tests::true' || true + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo '::set-output name=run_ssl_tests::true' || true fi + + check_abi: + name: 'Check if the ABI has changed' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install Dependencies + run: | + sudo ./.github/workflows/posix-deps-apt.sh + sudo apt-get install -yq abigail-tools + - name: Build CPython + env: + CFLAGS: -g3 -O0 + run: | + # Build Python with the libpython dynamic library + ./configure --enable-shared + make -j4 + - name: Check for changes in the ABI + run: make check-abidump + + check_generated_files: + name: 'Check if generated files are up to date' + runs-on: ubuntu-latest + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Build CPython + run: | + ./configure --with-pydebug + make -j4 regen-all + - name: Check for changes + run: | + changes=$(git status --porcelain) + # Check for changes in regenerated files + if ! test -z "$changes" + then + echo "Generated files not up to date. Perhaps you forgot to run make regen-all ;)" + echo "$changes" + exit 1 + fi + - name: Check exported libpython symbols + run: make smelly + build_win32: name: 'Windows (x86)' runs-on: windows-latest @@ -44,7 +112,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p Win32 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 build_win_amd64: name: 'Windows (x64)' @@ -58,7 +126,7 @@ jobs: - name: Display build info run: .\python.bat -m test.pythoninfo - name: Tests - run: .\PCbuild\rt.bat -x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 + run: .\PCbuild\rt.bat -p x64 -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 build_macos: name: 'macOS' @@ -78,29 +146,85 @@ jobs: build_ubuntu: name: 'Ubuntu' - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 needs: check_source if: needs.check_source.outputs.run_tests == 'true' env: - OPENSSL_VER: 1.1.1f + OPENSSL_VER: 1.1.1k steps: - uses: actions/checkout@v2 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v1 + uses: actions/cache@v2.1.3 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - name: Install OpenSSL if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $OPENSSL_VER --system Linux + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 - name: Configure CPython - run: ./configure --with-pydebug --with-openssl=$PWD/multissl/openssl/$OPENSSL_VER + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR - name: Build CPython run: make -j4 - name: Display build info run: make pythoninfo - name: Tests run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" + + build_ubuntu_ssltests: + name: 'Ubuntu SSL tests with OpenSSL' + runs-on: ubuntu-20.04 + needs: check_source + if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_ssl_tests == 'true' + strategy: + fail-fast: false + matrix: + openssl_ver: [1.0.2u, 1.1.0l, 1.1.1k, 3.0.0-alpha14] + env: + OPENSSL_VER: ${{ matrix.openssl_ver }} + MULTISSL_DIR: ${{ github.workspace }}/multissl + OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} + LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib + steps: + - uses: actions/checkout@v2 + - name: Install Dependencies + run: sudo ./.github/workflows/posix-deps-apt.sh + - name: Configure OpenSSL env vars + run: | + echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV + echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV + - name: 'Restore OpenSSL build' + id: cache-openssl + uses: actions/cache@v2.1.4 + with: + path: ./multissl/openssl/${{ env.OPENSSL_VER }} + key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} + - name: Install OpenSSL + if: steps.cache-openssl.outputs.cache-hit != 'true' + run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux + - name: Add ccache to PATH + run: | + echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV + - name: Configure ccache action + uses: hendrikmuhs/ccache-action@v1 + - name: Configure CPython + run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR + - name: Build CPython + run: make -j4 + - name: Display build info + run: make pythoninfo + - name: SSL tests + run: ./python Lib/test/ssltests.py diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml index e9ecf547275627..3664ccb32e70eb 100644 --- a/.github/workflows/build_msi.yml +++ b/.github/workflows/build_msi.yml @@ -4,24 +4,28 @@ on: push: branches: - master + - 3.9 - 3.8 - 3.7 paths: - 'Tools/msi/**' + - 'Lib/ensurepip/**' pull_request: branches: - master + - 3.9 - 3.8 - 3.7 paths: - 'Tools/msi/**' + - 'Lib/ensurepip/**' jobs: build_win32: name: 'Windows (x86) Installer' runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Build CPython installer run: .\Tools\msi\build.bat -x86 @@ -29,6 +33,6 @@ jobs: name: 'Windows (x64) Installer' runs-on: windows-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Build CPython installer run: .\Tools\msi\build.bat -x64 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 6dd973bf8e4ad7..79c63e936f23fd 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - 3.9 - 3.8 - 3.7 paths-ignore: @@ -12,6 +13,7 @@ on: #pull_request: # branches: # - master + # - 3.9 # - 3.8 # - 3.7 # paths-ignore: @@ -23,14 +25,14 @@ jobs: name: 'Ubuntu (Coverage)' runs-on: ubuntu-latest env: - OPENSSL_VER: 1.1.1f + OPENSSL_VER: 1.1.1k steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: 'Restore OpenSSL build' id: cache-openssl - uses: actions/cache@v1 + uses: actions/cache@v2.1.3 with: path: ./multissl/openssl/${{ env.OPENSSL_VER }} key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} @@ -74,7 +76,7 @@ jobs: name: 'Ubuntu (C Coverage)' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: Install Dependencies run: sudo ./.github/workflows/posix-deps-apt.sh - name: Configure CPython diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index c8d395cea5156c..18efd7f328c251 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -4,6 +4,7 @@ on: #push: # branches: # - master + # - 3.9 # - 3.8 # - 3.7 # paths: @@ -11,6 +12,7 @@ on: pull_request: branches: - master + - 3.9 - 3.8 - 3.7 paths: @@ -22,7 +24,7 @@ jobs: name: 'Docs' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 - name: 'Install Dependencies' run: sudo ./.github/workflows/posix-deps-apt.sh && sudo apt-get install wamerican - name: 'Configure CPython' @@ -34,7 +36,7 @@ jobs: - name: 'Build documentation' run: xvfb-run make -C Doc/ PYTHON=../python SPHINXOPTS="-q -W -j4" doctest suspicious html - name: 'Upload' - uses: actions/upload-artifact@v1 + uses: actions/upload-artifact@v2.2.2 with: name: doc-html path: Doc/build/html diff --git a/.github/workflows/posix-deps-apt.sh b/.github/workflows/posix-deps-apt.sh index 2b879d32f8150d..56cc70edf60001 100755 --- a/.github/workflows/posix-deps-apt.sh +++ b/.github/workflows/posix-deps-apt.sh @@ -3,19 +3,20 @@ apt-get update apt-get -yq install \ build-essential \ - zlib1g-dev \ + ccache \ + gdb \ + lcov \ libbz2-dev \ + libffi-dev \ + libgdbm-dev \ liblzma-dev \ libncurses5-dev \ libreadline6-dev \ libsqlite3-dev \ libssl-dev \ - libgdbm-dev \ - tk-dev \ lzma \ lzma-dev \ - liblzma-dev \ - libffi-dev \ + tk-dev \ uuid-dev \ xvfb \ - lcov + zlib1g-dev diff --git a/.gitignore b/.gitignore index 0962d3a1841fc4..80dcf34bf47a6f 100644 --- a/.gitignore +++ b/.gitignore @@ -103,6 +103,8 @@ Tools/unicode/data/ /config.status /config.status.lineno /platform +/profile-clean-stamp +/profile-run-stamp /pybuilddir.txt /pyconfig.h /python-config diff --git a/.travis.yml b/.travis.yml index 133385fbf5c719..02b2afa4816c77 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,7 +10,7 @@ cache: env: global: - - OPENSSL=1.1.1f + - OPENSSL=1.1.1k - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - CFLAGS="-I${OPENSSL_DIR}/include" @@ -51,10 +51,7 @@ matrix: env: TESTING=docs before_script: - cd Doc - # Sphinx is pinned so that new versions that introduce new warnings won't suddenly cause build failures. - # (Updating the version is fine as long as no warnings are raised by doing so.) - # The theme used by the docs is stored separately, so we need to install that as well. - - python -m pip install sphinx==2.2.0 blurb python-docs-theme + - make venv PYTHON=python script: - make check suspicious html SPHINXOPTS="-q -W -j4" - name: "Documentation tests" @@ -82,6 +79,12 @@ matrix: packages: - xvfb before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run Python coverage on pull requests." + exit + fi - ./configure - make -j4 # Need a venv that can parse covered code. @@ -107,6 +110,12 @@ matrix: - lcov - xvfb before_script: + - | + if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] + then + echo "Don't run C coverage on pull requests." + exit + fi - ./configure script: - xvfb-run make -j4 coverage-report diff --git a/Doc/Makefile b/Doc/Makefile index 05361f2ee2c82f..f653d70674eb1c 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -143,7 +143,7 @@ clean: venv: $(PYTHON) -m venv $(VENVDIR) $(VENVDIR)/bin/python3 -m pip install -U pip setuptools - $(VENVDIR)/bin/python3 -m pip install -U Sphinx==2.2.0 blurb python-docs-theme + $(VENVDIR)/bin/python3 -m pip install -r requirements.txt @echo "The venv has been created in the $(VENVDIR) directory" dist: diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index b7baad589a72c8..7a3188009cf959 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -131,12 +131,12 @@ which disallows mutable objects such as :class:`bytearray`. ``S`` (:class:`bytes`) [PyBytesObject \*] Requires that the Python object is a :class:`bytes` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a bytes object. The C variable may also be declared as :c:type:`PyObject\*`. + a bytes object. The C variable may also be declared as :c:type:`PyObject*`. ``Y`` (:class:`bytearray`) [PyByteArrayObject \*] Requires that the Python object is a :class:`bytearray` object, without attempting any conversion. Raises :exc:`TypeError` if the object is not - a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject\*`. + a :class:`bytearray` object. The C variable may also be declared as :c:type:`PyObject*`. ``u`` (:class:`str`) [const Py_UNICODE \*] Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of @@ -151,7 +151,7 @@ which disallows mutable objects such as :class:`bytearray`. Previously, :exc:`TypeError` was raised when embedded null code points were encountered in the Python string. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -160,7 +160,7 @@ which disallows mutable objects such as :class:`bytearray`. Unicode data buffer, the second one its length. This variant allows null code points. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -168,7 +168,7 @@ which disallows mutable objects such as :class:`bytearray`. Like ``u``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. @@ -176,14 +176,14 @@ which disallows mutable objects such as :class:`bytearray`. Like ``u#``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to ``NULL``. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsWideCharString`. ``U`` (:class:`str`) [PyObject \*] Requires that the Python object is a Unicode object, without attempting any conversion. Raises :exc:`TypeError` if the object is not a Unicode - object. The C variable may also be declared as :c:type:`PyObject\*`. + object. The C variable may also be declared as :c:type:`PyObject*`. ``w*`` (read-write :term:`bytes-like object`) [Py_buffer] This format accepts any object which implements the read-write buffer @@ -196,10 +196,10 @@ which disallows mutable objects such as :class:`bytearray`. It only works for encoded data without embedded NUL bytes. This format requires two arguments. The first is only used as input, and - must be a :c:type:`const char\*` which points to the name of an encoding as a + must be a :c:type:`const char*` which points to the name of an encoding as a NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. @@ -219,10 +219,10 @@ which disallows mutable objects such as :class:`bytearray`. characters. It requires three arguments. The first is only used as input, and must be a - :c:type:`const char\*` which points to the name of an encoding as a + :c:type:`const char*` which points to the name of an encoding as a NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:type:`char\*\*`; the value of the pointer it + second argument must be a :c:type:`char**`; the value of the pointer it references will be set to a buffer with the contents of the argument text. The text will be encoded in the encoding specified by the first argument. The third argument must be a pointer to an integer; the referenced integer @@ -322,7 +322,7 @@ Other objects ``O!`` (object) [*typeobject*, PyObject \*] Store a Python object in a C object pointer. This is similar to ``O``, but takes two C arguments: the first is the address of a Python type object, the - second is the address of the C variable (of type :c:type:`PyObject\*`) into which + second is the address of the C variable (of type :c:type:`PyObject*`) into which the object pointer is stored. If the Python object does not have the required type, :exc:`TypeError` is raised. @@ -331,13 +331,13 @@ Other objects ``O&`` (object) [*converter*, *anything*] Convert a Python object to a C variable through a *converter* function. This takes two arguments: the first is a function, the second is the address of a C - variable (of arbitrary type), converted to :c:type:`void \*`. The *converter* + variable (of arbitrary type), converted to :c:type:`void *`. The *converter* function in turn is called as follows:: status = converter(object, address); where *object* is the Python object to be converted and *address* is the - :c:type:`void\*` argument that was passed to the :c:func:`PyArg_Parse\*` function. + :c:type:`void*` argument that was passed to the :c:func:`PyArg_Parse\*` function. The returned *status* should be ``1`` for a successful conversion and ``0`` if the conversion has failed. When the conversion fails, the *converter* function should raise an exception and leave the content of *address* unmodified. @@ -483,7 +483,7 @@ API Functions *args*; it must actually be a tuple. The length of the tuple must be at least *min* and no more than *max*; *min* and *max* may be equal. Additional arguments must be passed to the function, each of which should be a pointer to a - :c:type:`PyObject\*` variable; these will be filled in with the values from + :c:type:`PyObject*` variable; these will be filled in with the values from *args*; they will contain borrowed references. The variables which correspond to optional parameters not given by *args* will not be filled in; these should be initialized by the caller. This function returns true on success and false if @@ -652,8 +652,8 @@ Building values ``O&`` (object) [*converter*, *anything*] Convert *anything* to a Python object through a *converter* function. The - function is called with *anything* (which should be compatible with :c:type:`void - \*`) as its argument and should return a "new" Python object, or ``NULL`` if an + function is called with *anything* (which should be compatible with :c:type:`void*`) + as its argument and should return a "new" Python object, or ``NULL`` if an error occurred. ``(items)`` (:class:`tuple`) [*matching-items*] diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst index ce8de6e22f44ca..c197d447e9618c 100644 --- a/Doc/c-api/bool.rst +++ b/Doc/c-api/bool.rst @@ -13,7 +13,8 @@ are available, however. .. c:function:: int PyBool_Check(PyObject *o) - Return true if *o* is of type :c:data:`PyBool_Type`. + Return true if *o* is of type :c:data:`PyBool_Type`. This function always + succeeds. .. c:var:: PyObject* Py_False diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index fc1430efa8a4bb..e32719373cc716 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -89,7 +89,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. .. c:type:: Py_buffer - .. c:member:: void \*buf + .. c:member:: void *buf A pointer to the start of the logical structure described by the buffer fields. This can be any location within the underlying physical memory @@ -99,7 +99,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. For :term:`contiguous` arrays, the value points to the beginning of the memory block. - .. c:member:: void \*obj + .. c:member:: void *obj A new reference to the exporting object. The reference is owned by the consumer and automatically decremented and set to ``NULL`` by @@ -145,7 +145,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - .. c:member:: const char \*format + .. c:member:: const char *format A *NUL* terminated string in :mod:`struct` module style syntax describing the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) @@ -164,7 +164,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. to 64. Exporters MUST respect this limit, consumers of multi-dimensional buffers SHOULD be able to handle up to :c:macro:`PyBUF_MAX_NDIM` dimensions. - .. c:member:: Py_ssize_t \*shape + .. c:member:: Py_ssize_t *shape An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` indicating the shape of the memory as an n-dimensional array. Note that @@ -177,7 +177,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The shape array is read-only for the consumer. - .. c:member:: Py_ssize_t \*strides + .. c:member:: Py_ssize_t *strides An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` giving the number of bytes to skip to get to a new element in each @@ -189,7 +189,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The strides array is read-only for the consumer. - .. c:member:: Py_ssize_t \*suboffsets + .. c:member:: Py_ssize_t *suboffsets An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`. If ``suboffsets[n] >= 0``, the values stored along the nth dimension are @@ -207,7 +207,7 @@ a buffer, see :c:func:`PyObject_GetBuffer`. The suboffsets array is read-only for the consumer. - .. c:member:: void \*internal + .. c:member:: void *internal This is for use internally by the exporting object. For example, this might be re-cast as an integer by the exporter and used to store flags @@ -301,7 +301,7 @@ must be C-contiguous. +-----------------------------------+-------+---------+------------+--------+ | .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F | +-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | C | +| :c:macro:`PyBUF_ND` | yes | NULL | NULL | C | +-----------------------------------+-------+---------+------------+--------+ @@ -438,12 +438,12 @@ Buffer-related functions Send a request to *exporter* to fill in *view* as specified by *flags*. If the exporter cannot provide a buffer of the exact type, it MUST raise - :c:data:`PyExc_BufferError`, set :c:member:`view->obj` to ``NULL`` and + :c:data:`PyExc_BufferError`, set ``view->obj`` to ``NULL`` and return ``-1``. - On success, fill in *view*, set :c:member:`view->obj` to a new reference + On success, fill in *view*, set ``view->obj`` to a new reference to *exporter* and return 0. In the case of chained buffer providers - that redirect requests to a single object, :c:member:`view->obj` MAY + that redirect requests to a single object, ``view->obj`` MAY refer to this object instead of *exporter* (See :ref:`Buffer Object Structures `). Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls @@ -455,7 +455,7 @@ Buffer-related functions .. c:function:: void PyBuffer_Release(Py_buffer *view) Release the buffer *view* and decrement the reference count for - :c:member:`view->obj`. This function MUST be called when the buffer + ``view->obj``. This function MUST be called when the buffer is no longer being used, otherwise reference leaks may occur. It is an error to call this function on a buffer that was not obtained via @@ -516,9 +516,9 @@ Buffer-related functions *view* as specified by flags, unless *buf* has been designated as read-only and :c:macro:`PyBUF_WRITABLE` is set in *flags*. - On success, set :c:member:`view->obj` to a new reference to *exporter* and + On success, set ``view->obj`` to a new reference to *exporter* and return 0. Otherwise, raise :c:data:`PyExc_BufferError`, set - :c:member:`view->obj` to ``NULL`` and return ``-1``; + ``view->obj`` to ``NULL`` and return ``-1``; If this function is used as part of a :ref:`getbufferproc `, *exporter* MUST be set to the exporting object and *flags* must be passed diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index b2f409c15abb7a..30bcfc7cf9f500 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -25,13 +25,13 @@ Type check macros .. c:function:: int PyByteArray_Check(PyObject *o) Return true if the object *o* is a bytearray object or an instance of a - subtype of the bytearray type. + subtype of the bytearray type. This function always succeeds. .. c:function:: int PyByteArray_CheckExact(PyObject *o) Return true if the object *o* is a bytearray object, but not an instance of a - subtype of the bytearray type. + subtype of the bytearray type. This function always succeeds. Direct API functions diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst index 0e33ed2c7c9403..de65701037a7c1 100644 --- a/Doc/c-api/bytes.rst +++ b/Doc/c-api/bytes.rst @@ -25,13 +25,13 @@ called with a non-bytes parameter. .. c:function:: int PyBytes_Check(PyObject *o) Return true if the object *o* is a bytes object or an instance of a subtype - of the bytes type. + of the bytes type. This function always succeeds. .. c:function:: int PyBytes_CheckExact(PyObject *o) Return true if the object *o* is a bytes object, but not an instance of a - subtype of the bytes type. + subtype of the bytes type. This function always succeeds. .. c:function:: PyObject* PyBytes_FromString(const char *v) diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst index 06db12666d787c..31dc9c8031fdb6 100644 --- a/Doc/c-api/call.rst +++ b/Doc/c-api/call.rst @@ -84,7 +84,7 @@ This is a pointer to a function with the following signature: and they must be unique. If there are no keyword arguments, then *kwnames* can instead be *NULL*. -.. c:var:: PY_VECTORCALL_ARGUMENTS_OFFSET +.. c:macro:: PY_VECTORCALL_ARGUMENTS_OFFSET If this flag is set in a vectorcall *nargsf* argument, the callee is allowed to temporarily change ``args[-1]``. In other words, *args* points to @@ -144,7 +144,7 @@ Vectorcall Support API However, the function ``PyVectorcall_NARGS`` should be used to allow for future extensions. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.8 @@ -158,7 +158,7 @@ Vectorcall Support API This is mostly useful to check whether or not *op* supports vectorcall, which can be done by checking ``PyVectorcall_Function(op) != NULL``. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.8 @@ -172,7 +172,7 @@ Vectorcall Support API It does not check the :const:`Py_TPFLAGS_HAVE_VECTORCALL` flag and it does not fall back to ``tp_call``. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.8 @@ -256,7 +256,7 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 @@ -283,7 +283,7 @@ please see individual documentation for details. This is the equivalent of the Python expression: ``callable(*args)``. - Note that if you only pass :c:type:`PyObject \*` args, + Note that if you only pass :c:type:`PyObject *` args, :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. .. versionchanged:: 3.4 @@ -304,17 +304,17 @@ please see individual documentation for details. This is the equivalent of the Python expression: ``obj.name(arg1, arg2, ...)``. - Note that if you only pass :c:type:`PyObject \*` args, + Note that if you only pass :c:type:`PyObject *` args, :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. .. versionchanged:: 3.4 The types of *name* and *format* were changed from ``char *``. -.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ..., NULL) +.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ...) Call a callable Python object *callable*, with a variable number of - :c:type:`PyObject \*` arguments. The arguments are provided as a variable number + :c:type:`PyObject *` arguments. The arguments are provided as a variable number of parameters followed by *NULL*. Return the result of the call on success, or raise an exception and return @@ -324,11 +324,11 @@ please see individual documentation for details. ``callable(arg1, arg2, ...)``. -.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ..., NULL) +.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) Call a method of the Python object *obj*, where the name of the method is given as a Python string object in *name*. It is called with a variable number of - :c:type:`PyObject \*` arguments. The arguments are provided as a variable number + :c:type:`PyObject *` arguments. The arguments are provided as a variable number of parameters followed by *NULL*. Return the result of the call on success, or raise an exception and return @@ -343,7 +343,7 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 @@ -357,7 +357,7 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 @@ -372,7 +372,7 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 @@ -388,7 +388,7 @@ please see individual documentation for details. already has a dictionary ready to use for the keyword arguments, but not a tuple for the positional arguments. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 @@ -410,7 +410,7 @@ please see individual documentation for details. Return the result of the call on success, or raise an exception and return *NULL* on failure. - This function is not part of the `limited API `_. + This function is not part of the :ref:`limited API `. .. versionadded:: 3.9 diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst index 78e21140b2f80c..908e92653dd483 100644 --- a/Doc/c-api/capsule.rst +++ b/Doc/c-api/capsule.rst @@ -15,7 +15,7 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:type:: PyCapsule This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:type:`void\*` + extension modules who need to pass an opaque value (as a :c:type:`void*` pointer) through Python code to other C code. It is often used to make a C function pointer defined in one module available to other modules, so the regular import mechanism can be used to access C APIs defined in dynamically @@ -34,7 +34,8 @@ Refer to :ref:`using-capsules` for more information on using these objects. .. c:function:: int PyCapsule_CheckExact(PyObject *p) - Return true if its argument is a :c:type:`PyCapsule`. + Return true if its argument is a :c:type:`PyCapsule`. This function always + succeeds. .. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst index 8408f7e398db7b..ac4ef5adc5cc20 100644 --- a/Doc/c-api/cell.rst +++ b/Doc/c-api/cell.rst @@ -27,7 +27,8 @@ Cell objects are not likely to be useful elsewhere. .. c:function:: int PyCell_Check(ob) - Return true if *ob* is a cell object; *ob* must not be ``NULL``. + Return true if *ob* is a cell object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: PyObject* PyCell_New(PyObject *ob) diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst index 6f8c41ccbf6e85..b3a17f1898e8e1 100644 --- a/Doc/c-api/code.rst +++ b/Doc/c-api/code.rst @@ -27,7 +27,7 @@ bound into a function. .. c:function:: int PyCode_Check(PyObject *co) - Return true if *co* is a :class:`code` object. + Return true if *co* is a :class:`code` object. This function always succeeds. .. c:function:: int PyCode_GetNumFree(PyCodeObject *co) diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst index 06dbb2572725ee..c25894681bca35 100644 --- a/Doc/c-api/complex.rst +++ b/Doc/c-api/complex.rst @@ -46,9 +46,9 @@ pointers. This is consistent throughout the API. :c:type:`Py_complex` representation. -.. c:function:: Py_complex _Py_c_neg(Py_complex complex) +.. c:function:: Py_complex _Py_c_neg(Py_complex num) - Return the negation of the complex number *complex*, using the C + Return the negation of the complex number *num*, using the C :c:type:`Py_complex` representation. @@ -94,13 +94,13 @@ Complex Numbers as Python Objects .. c:function:: int PyComplex_Check(PyObject *p) Return true if its argument is a :c:type:`PyComplexObject` or a subtype of - :c:type:`PyComplexObject`. + :c:type:`PyComplexObject`. This function always succeeds. .. c:function:: int PyComplex_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyComplexObject`, but not a subtype of - :c:type:`PyComplexObject`. + :c:type:`PyComplexObject`. This function always succeeds. .. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst index c1d9fa1b41a3fe..84224dcca523b9 100644 --- a/Doc/c-api/concrete.rst +++ b/Doc/c-api/concrete.rst @@ -115,3 +115,4 @@ Other Objects coro.rst contextvars.rst datetime.rst + typehints.rst diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst index 9c088814314a81..d970f5443b1df5 100644 --- a/Doc/c-api/contextvars.rst +++ b/Doc/c-api/contextvars.rst @@ -107,9 +107,9 @@ Context variable functions: .. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def) Create a new ``ContextVar`` object. The *name* parameter is used - for introspection and debug purposes. The *def* parameter may optionally - specify the default value for the context variable. If an error has - occurred, this function returns ``NULL``. + for introspection and debug purposes. The *def* parameter specifies + a default value for the context variable, or ``NULL`` for no default. + If an error has occurred, this function returns ``NULL``. .. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) @@ -124,13 +124,12 @@ Context variable functions: - the default value of *var*, if not ``NULL``; - ``NULL`` - If the value was found, the function will create a new reference to it. + Except for ``NULL``, the function returns a new reference. .. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value) - Set the value of *var* to *value* in the current context. Returns a - pointer to a :c:type:`PyObject` object, or ``NULL`` if an error - has occurred. + Set the value of *var* to *value* in the current context. Returns + a new token object for this change, or ``NULL`` if an error has occurred. .. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token) diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst index b310fcb5e4f91e..0cc836b70a018f 100644 --- a/Doc/c-api/conversion.rst +++ b/Doc/c-api/conversion.rst @@ -11,21 +11,21 @@ Functions for number conversion and formatted string output. .. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) Output not more than *size* bytes to *str* according to the format string - *format* and the extra arguments. See the Unix man page :manpage:`snprintf(2)`. + *format* and the extra arguments. See the Unix man page :manpage:`snprintf(3)`. .. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) Output not more than *size* bytes to *str* according to the format string *format* and the variable argument list *va*. Unix man page - :manpage:`vsnprintf(2)`. + :manpage:`vsnprintf(3)`. :c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to guarantee consistent behavior in corner cases, which the Standard C functions do not. -The wrappers ensure that *str*[*size*-1] is always ``'\0'`` upon return. They +The wrappers ensure that ``str[size-1]`` is always ``'\0'`` upon return. They never write more than *size* bytes (including the trailing ``'\0'``) into str. Both functions require that ``str != NULL``, ``size > 0`` and ``format != NULL``. @@ -38,13 +38,13 @@ The return value (*rv*) for these functions should be interpreted as follows: * When ``0 <= rv < size``, the output conversion was successful and *rv* characters were written to *str* (excluding the trailing ``'\0'`` byte at - *str*[*rv*]). + ``str[rv]``). * When ``rv >= size``, the output conversion was truncated and a buffer with - ``rv + 1`` bytes would have been needed to succeed. *str*[*size*-1] is ``'\0'`` + ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` in this case. -* When ``rv < 0``, "something bad happened." *str*[*size*-1] is ``'\0'`` in +* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in this case too, but the rest of *str* is undefined. The exact cause of the error depends on the underlying platform. diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst index 2260944a9a93a9..caa855a10d20ca 100644 --- a/Doc/c-api/coro.rst +++ b/Doc/c-api/coro.rst @@ -24,6 +24,7 @@ return. .. c:function:: int PyCoro_CheckExact(PyObject *ob) Return true if *ob*'s type is :c:type:`PyCoro_Type`; *ob* must not be ``NULL``. + This function always succeeds. .. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst index bd4f1ff446bcf4..66f148df286807 100644 --- a/Doc/c-api/datetime.rst +++ b/Doc/c-api/datetime.rst @@ -28,61 +28,66 @@ Type-check macros: .. c:function:: int PyDate_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DateType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDate_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyDateTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDateTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be ``NULL``. + be ``NULL``. This function always succeeds. .. c:function:: int PyTime_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyTime_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyDelta_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyDelta_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. .. c:function:: int PyTZInfo_Check(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. + :c:data:`PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always + succeeds. .. c:function:: int PyTZInfo_CheckExact(PyObject *ob) Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - ``NULL``. + ``NULL``. This function always succeeds. Macros to create objects: diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst index 2fb29cdd617789..5803ac2ad97adb 100644 --- a/Doc/c-api/dict.rst +++ b/Doc/c-api/dict.rst @@ -22,13 +22,13 @@ Dictionary Objects .. c:function:: int PyDict_Check(PyObject *p) Return true if *p* is a dict object or an instance of a subtype of the dict - type. + type. This function always succeeds. .. c:function:: int PyDict_CheckExact(PyObject *p) Return true if *p* is a dict object, but not an instance of a subtype of - the dict type. + the dict type. This function always succeeds. .. c:function:: PyObject* PyDict_New() @@ -73,7 +73,7 @@ Dictionary Objects .. index:: single: PyUnicode_FromString() Insert *val* into the dictionary *p* using *key* as a key. *key* should - be a :c:type:`const char\*`. The key object is created using + be a :c:type:`const char*`. The key object is created using ``PyUnicode_FromString(key)``. Return ``0`` on success or ``-1`` on failure. This function *does not* steal a reference to *val*. @@ -81,14 +81,16 @@ Dictionary Objects .. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) Remove the entry in dictionary *p* with key *key*. *key* must be hashable; - if it isn't, :exc:`TypeError` is raised. Return ``0`` on success or ``-1`` - on failure. + if it isn't, :exc:`TypeError` is raised. + If *key* is not in the dictionary, :exc:`KeyError` is raised. + Return ``0`` on success or ``-1`` on failure. .. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - Remove the entry in dictionary *p* which has a key specified by the string - *key*. Return ``0`` on success or ``-1`` on failure. + Remove the entry in dictionary *p* which has a key specified by the string *key*. + If *key* is not in the dictionary, :exc:`KeyError` is raised. + Return ``0`` on success or ``-1`` on failure. .. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) @@ -112,7 +114,7 @@ Dictionary Objects .. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:type:`const char\*`, rather than a :c:type:`PyObject\*`. + :c:type:`const char*`, rather than a :c:type:`PyObject*`. Note that exceptions which occur while calling :meth:`__hash__` and :meth:`__eq__` methods and creating a temporary string object @@ -161,7 +163,7 @@ Dictionary Objects prior to the first call to this function to start the iteration; the function returns true for each pair in the dictionary, and false once all pairs have been reported. The parameters *pkey* and *pvalue* should either - point to :c:type:`PyObject\*` variables that will be filled in with each key + point to :c:type:`PyObject*` variables that will be filled in with each key and value, respectively, or may be ``NULL``. Any references returned through them are borrowed. *ppos* should not be altered during iteration. Its value represents offsets within the internal dictionary structure, and diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index e7805ba143c584..247b6d68eceae9 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -637,11 +637,21 @@ The following functions are used to create and modify Unicode exceptions from C. *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are UTF-8 encoded strings. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeEncodeError, "sOnns", ...)``. + .. c:function:: PyObject* PyUnicodeTranslateError_Create(const Py_UNICODE *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) Create a :class:`UnicodeTranslateError` object with the attributes *object*, *length*, *start*, *end* and *reason*. *reason* is a UTF-8 encoded string. + .. deprecated:: 3.3 3.11 + + ``Py_UNICODE`` is deprecated since Python 3.3. Please migrate to + ``PyObject_CallFunction(PyExc_UnicodeTranslateError, "Onns", ...)``. + .. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) @@ -773,7 +783,7 @@ Standard Exceptions All standard Python exceptions are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: @@ -993,7 +1003,7 @@ Standard Warning Categories All standard Python warning categories are available as global variables whose names are ``PyExc_`` followed by the Python exception name. These have the type -:c:type:`PyObject\*`; they are all class objects. For completeness, here are all +:c:type:`PyObject*`; they are all class objects. For completeness, here are all the variables: .. index:: diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst index 5370c4e350a0b5..ed3735aa83608a 100644 --- a/Doc/c-api/file.rst +++ b/Doc/c-api/file.rst @@ -8,7 +8,7 @@ File Objects .. index:: object: file These APIs are a minimal emulation of the Python 2 C API for built-in file -objects, which used to rely on the buffered I/O (:c:type:`FILE\*`) support +objects, which used to rely on the buffered I/O (:c:type:`FILE*`) support from the C standard library. In Python 3, files and streams use the new :mod:`io` module, which defines several layers over the low-level unbuffered I/O of the operating system. The functions described below are @@ -17,7 +17,7 @@ error reporting in the interpreter; third-party code is advised to access the :mod:`io` APIs instead. -.. c:function:: PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) +.. c:function:: PyObject* PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) Create a Python file object from the file descriptor of an already opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* @@ -82,6 +82,8 @@ the :mod:`io` APIs instead. This function is safe to call before :c:func:`Py_Initialize`. + .. audit-event:: setopencodehook "" c.PyFile_SetOpenCodeHook + .. versionadded:: 3.8 diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst index b29937dbecdcfd..c107243a88dfc6 100644 --- a/Doc/c-api/float.rst +++ b/Doc/c-api/float.rst @@ -22,13 +22,13 @@ Floating Point Objects .. c:function:: int PyFloat_Check(PyObject *p) Return true if its argument is a :c:type:`PyFloatObject` or a subtype of - :c:type:`PyFloatObject`. + :c:type:`PyFloatObject`. This function always succeeds. .. c:function:: int PyFloat_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyFloatObject`, but not a subtype of - :c:type:`PyFloatObject`. + :c:type:`PyFloatObject`. This function always succeeds. .. c:function:: PyObject* PyFloat_FromString(PyObject *str) diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst index bb416f4bb63aa2..20968828e0bb36 100644 --- a/Doc/c-api/function.rst +++ b/Doc/c-api/function.rst @@ -26,7 +26,7 @@ There are a few functions specific to Python functions. .. c:function:: int PyFunction_Check(PyObject *o) Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be ``NULL``. + The parameter must not be ``NULL``. This function always succeeds. .. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst index eee114c19d5904..f821b45090c470 100644 --- a/Doc/c-api/gcsupport.rst +++ b/Doc/c-api/gcsupport.rst @@ -33,6 +33,18 @@ Constructors for container types must conform to two rules: #. Once all the fields which may contain references to other containers are initialized, it must call :c:func:`PyObject_GC_Track`. + .. warning:: + If a type adds the Py_TPFLAGS_HAVE_GC, then it *must* implement at least + a :c:member:`~PyTypeObject.tp_traverse` handler or explicitly use one + from its subclass or subclasses. + + When calling :c:func:`PyType_Ready` or some of the APIs that indirectly + call it like :c:func:`PyType_FromSpecWithBases` or + :c:func:`PyType_FromSpec` the interpreter will automatically populate the + :c:member:`~PyTypeObject.tp_flags`, :c:member:`~PyTypeObject.tp_traverse` + and :c:member:`~PyTypeObject.tp_clear` fields if the type inherits from a + class that implements the garbage collector protocol and the child class + does *not* include the :const:`Py_TPFLAGS_HAVE_GC` flag. .. c:function:: TYPE* PyObject_GC_New(TYPE, PyTypeObject *type) diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst index 74410927bfde10..0eb5922f6da75f 100644 --- a/Doc/c-api/gen.rst +++ b/Doc/c-api/gen.rst @@ -22,12 +22,14 @@ than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. .. c:function:: int PyGen_Check(PyObject *ob) - Return true if *ob* is a generator object; *ob* must not be ``NULL``. + Return true if *ob* is a generator object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: int PyGen_CheckExact(PyObject *ob) - Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be ``NULL``. + Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be + ``NULL``. This function always succeeds. .. c:function:: PyObject* PyGen_New(PyFrameObject *frame) diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index c6fc33076f0f51..d2ae6b6d4e4711 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -299,4 +299,8 @@ Importing Modules field; failure to provide the sentinel value can result in a memory fault. Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to extend the internal table. In the event of failure, no modules are added to the - internal table. This should be called before :c:func:`Py_Initialize`. + internal table. This must be called before :c:func:`Py_Initialize`. + + If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or + :c:func:`PyImport_ExtendInittab` must be called before each Python + initialization. diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 68fed2acc447ee..eb0bd14587145b 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -81,7 +81,7 @@ When a flag is set by an option, the value of the flag is the number of times that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. -.. c:var:: Py_BytesWarningFlag +.. c:var:: int Py_BytesWarningFlag Issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater @@ -89,7 +89,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-b` option. -.. c:var:: Py_DebugFlag +.. c:var:: int Py_DebugFlag Turn on parser debugging output (for expert only, depending on compilation options). @@ -97,7 +97,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment variable. -.. c:var:: Py_DontWriteBytecodeFlag +.. c:var:: int Py_DontWriteBytecodeFlag If set to non-zero, Python won't try to write ``.pyc`` files on the import of source modules. @@ -105,14 +105,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. -.. c:var:: Py_FrozenFlag +.. c:var:: int Py_FrozenFlag Suppress error messages when calculating the module search path in :c:func:`Py_GetPath`. Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs. -.. c:var:: Py_HashRandomizationFlag +.. c:var:: int Py_HashRandomizationFlag Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to a non-empty string. @@ -120,14 +120,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment variable to initialize the secret hash seed. -.. c:var:: Py_IgnoreEnvironmentFlag +.. c:var:: int Py_IgnoreEnvironmentFlag Ignore all :envvar:`PYTHON*` environment variables, e.g. :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. Set by the :option:`-E` and :option:`-I` options. -.. c:var:: Py_InspectFlag +.. c:var:: int Py_InspectFlag When a script is passed as first argument or the :option:`-c` option is used, enter interactive mode after executing the script or the command, even when @@ -136,11 +136,11 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment variable. -.. c:var:: Py_InteractiveFlag +.. c:var:: int Py_InteractiveFlag Set by the :option:`-i` option. -.. c:var:: Py_IsolatedFlag +.. c:var:: int Py_IsolatedFlag Run Python in isolated mode. In isolated mode :data:`sys.path` contains neither the script's directory nor the user's site-packages directory. @@ -149,7 +149,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.4 -.. c:var:: Py_LegacyWindowsFSEncodingFlag +.. c:var:: int Py_LegacyWindowsFSEncodingFlag If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 encoding for the filesystem encoding. @@ -161,7 +161,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_LegacyWindowsStdioFlag +.. c:var:: int Py_LegacyWindowsStdioFlag If the flag is non-zero, use :class:`io.FileIO` instead of :class:`WindowsConsoleIO` for :mod:`sys` standard streams. @@ -173,7 +173,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. availability:: Windows. -.. c:var:: Py_NoSiteFlag +.. c:var:: int Py_NoSiteFlag Disable the import of the module :mod:`site` and the site-dependent manipulations of :data:`sys.path` that it entails. Also disable these @@ -182,7 +182,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-S` option. -.. c:var:: Py_NoUserSiteDirectory +.. c:var:: int Py_NoUserSiteDirectory Don't add the :data:`user site-packages directory ` to :data:`sys.path`. @@ -190,12 +190,12 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. Set by the :option:`-s` and :option:`-I` options, and the :envvar:`PYTHONNOUSERSITE` environment variable. -.. c:var:: Py_OptimizeFlag +.. c:var:: int Py_OptimizeFlag Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment variable. -.. c:var:: Py_QuietFlag +.. c:var:: int Py_QuietFlag Don't display the copyright and version messages even in interactive mode. @@ -203,14 +203,14 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. versionadded:: 3.2 -.. c:var:: Py_UnbufferedStdioFlag +.. c:var:: int Py_UnbufferedStdioFlag Force the stdout and stderr streams to be unbuffered. Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` environment variable. -.. c:var:: Py_VerboseFlag +.. c:var:: int Py_VerboseFlag Print a message each time a module is initialized, showing the place (filename or built-in module) from which it is loaded. If greater or equal @@ -830,7 +830,7 @@ code, or when embedding the Python interpreter: .. c:type:: PyThreadState This data structure represents the state of a single thread. The only public - data member is :c:type:`PyInterpreterState \*`:attr:`interp`, which points to + data member is :attr:`interp` (:c:type:`PyInterpreterState *`), which points to this thread's interpreter state. @@ -1155,7 +1155,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. versionadded:: 3.9 -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame); +.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) Set the frame evaluation function. @@ -1619,7 +1619,7 @@ The Python interpreter provides low-level support for thread-local storage (TLS) which wraps the underlying native TLS implementation to support the Python-level thread local storage API (:class:`threading.local`). The CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:type:`void\*` value per +use a thread key and functions to associate a :c:type:`void*` value per thread. The GIL does *not* need to be held when calling these functions; they supply @@ -1630,8 +1630,8 @@ you need to include :file:`pythread.h` to use thread-local storage. .. note:: None of these API functions handle memory management on behalf of the - :c:type:`void\*` values. You need to allocate and deallocate them yourself. - If the :c:type:`void\*` values happen to be :c:type:`PyObject\*`, these + :c:type:`void*` values. You need to allocate and deallocate them yourself. + If the :c:type:`void*` values happen to be :c:type:`PyObject*`, these functions don't do refcount operations on them either. .. _thread-specific-storage-api: @@ -1727,14 +1727,14 @@ undefined if the given :c:type:`Py_tss_t` has not been initialized by .. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - Return a zero value to indicate successfully associating a :c:type:`void\*` + Return a zero value to indicate successfully associating a :c:type:`void*` value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:type:`void\*` value. + mapping of the key to a :c:type:`void*` value. .. c:function:: void* PyThread_tss_get(Py_tss_t *key) - Return the :c:type:`void\*` value associated with a TSS key in the current + Return the :c:type:`void*` value associated with a TSS key in the current thread. This returns ``NULL`` if no value is associated with the key in the current thread. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index fc82c3eb590248..23009e4191f88c 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -43,6 +43,7 @@ Functions: * :c:func:`Py_PreInitializeFromArgs` * :c:func:`Py_PreInitializeFromBytesArgs` * :c:func:`Py_RunMain` +* :c:func:`Py_GetArgcArgv` The preconfiguration (``PyPreConfig`` type) is stored in ``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in @@ -196,12 +197,12 @@ PyPreConfig Function to initialize a preconfiguration: - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Python Configuration `. - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) + .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) Initialize the preconfiguration with :ref:`Isolated Configuration `. @@ -436,6 +437,14 @@ PyConfig :data:`sys.base_prefix`. + .. c:member:: wchar_t* platlibdir + + :data:`sys.platlibdir`: platform library directory name, set at configure time + by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR`` + environment variable. + + .. versionadded:: 3.9 + .. c:member:: int buffered_stdio If equals to 0, enable unbuffered mode, making the stdout and stderr @@ -721,9 +730,12 @@ Function to initialize Python: The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or -``PyImport_ExtendInittab()`` are used, they must be set or called after Python -preinitialization and before the Python initialization. +If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` are used, they must be set or called after +Python preinitialization and before the Python initialization. If Python is +initialized multiple times, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` must be called before each Python +initialization. Example setting the program name:: @@ -884,6 +896,7 @@ Path Configuration * Path configuration inputs: * :c:member:`PyConfig.home` + * :c:member:`PyConfig.platlibdir` * :c:member:`PyConfig.pathconfig_warnings` * :c:member:`PyConfig.program_name` * :c:member:`PyConfig.pythonpath_env` @@ -975,6 +988,14 @@ customized Python always running in isolated mode using :c:func:`Py_RunMain`. +Py_GetArgcArgv() +---------------- + +.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) + + Get the original command line arguments, before Python modified them. + + Multi-Phase Initialization Private Provisional API -------------------------------------------------- diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index e89a788de0d503..7ca8693afab79c 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -229,13 +229,13 @@ Objects, Types and Reference Counts .. index:: object: type Most Python/C API functions have one or more arguments as well as a return value -of type :c:type:`PyObject\*`. This type is a pointer to an opaque data type +of type :c:type:`PyObject*`. This type is a pointer to an opaque data type representing an arbitrary Python object. Since all Python object types are treated the same way by the Python language in most situations (e.g., assignments, scope rules, and argument passing), it is only fitting that they should be represented by a single C type. Almost all Python objects live on the heap: you never declare an automatic or static variable of type -:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject\*` can be +:c:type:`PyObject`, only pointer variables of type :c:type:`PyObject*` can be declared. The sole exception are the type objects; since these must never be deallocated, they are typically static :c:type:`PyTypeObject` objects. @@ -496,7 +496,7 @@ Types There are few other data types that play a significant role in the Python/C API; most are simple C types such as :c:type:`int`, :c:type:`long`, -:c:type:`double` and :c:type:`char\*`. A few structure types are used to +:c:type:`double` and :c:type:`char*`. A few structure types are used to describe static tables used to list the functions exported by a module or the data attributes of a new object type, and another is used to describe the value of a complex number. These will be discussed together with the functions that diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst index a2992b3452f91c..189f80b9b81932 100644 --- a/Doc/c-api/iter.rst +++ b/Doc/c-api/iter.rst @@ -9,7 +9,8 @@ There are two functions specifically for working with iterators. .. c:function:: int PyIter_Check(PyObject *o) - Return true if the object *o* supports the iterator protocol. + Return true if the object *o* supports the iterator protocol. This + function always succeeds. .. c:function:: PyObject* PyIter_Next(PyObject *o) diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst index 4d91e4a224fe82..3fcf099134d4dd 100644 --- a/Doc/c-api/iterator.rst +++ b/Doc/c-api/iterator.rst @@ -21,7 +21,8 @@ sentinel value is returned. .. c:function:: int PySeqIter_Check(op) - Return true if the type of *op* is :c:data:`PySeqIter_Type`. + Return true if the type of *op* is :c:data:`PySeqIter_Type`. This function + always succeeds. .. c:function:: PyObject* PySeqIter_New(PyObject *seq) @@ -39,7 +40,8 @@ sentinel value is returned. .. c:function:: int PyCallIter_Check(op) - Return true if the type of *op* is :c:data:`PyCallIter_Type`. + Return true if the type of *op* is :c:data:`PyCallIter_Type`. This + function always succeeds. .. c:function:: PyObject* PyCallIter_New(PyObject *callable, PyObject *sentinel) diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst index 0bc0785f200d4c..f338e2ae066895 100644 --- a/Doc/c-api/list.rst +++ b/Doc/c-api/list.rst @@ -22,13 +22,13 @@ List Objects .. c:function:: int PyList_Check(PyObject *p) Return true if *p* is a list object or an instance of a subtype of the list - type. + type. This function always succeeds. .. c:function:: int PyList_CheckExact(PyObject *p) Return true if *p* is a list object, but not an instance of a subtype of - the list type. + the list type. This function always succeeds. .. c:function:: PyObject* PyList_New(Py_ssize_t len) diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index c5c2aa60dcc35c..60e1791df9a451 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -27,13 +27,13 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: int PyLong_Check(PyObject *p) Return true if its argument is a :c:type:`PyLongObject` or a subtype of - :c:type:`PyLongObject`. + :c:type:`PyLongObject`. This function always succeeds. .. c:function:: int PyLong_CheckExact(PyObject *p) Return true if its argument is a :c:type:`PyLongObject`, but not a subtype of - :c:type:`PyLongObject`. + :c:type:`PyLongObject`. This function always succeeds. .. c:function:: PyObject* PyLong_FromLong(long v) @@ -96,11 +96,9 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicode(Py_UNICODE *u, Py_ssize_t length, int base) - Convert a sequence of Unicode digits to a Python integer value. The Unicode - string is first encoded to a byte string using :c:func:`PyUnicode_EncodeDecimal` - and then converted using :c:func:`PyLong_FromString`. + Convert a sequence of Unicode digits to a Python integer value. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.10 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyLong_FromUnicodeObject`. @@ -108,9 +106,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) Convert a sequence of Unicode digits in the string *u* to a Python integer - value. The Unicode string is first encoded to a byte string using - :c:func:`PyUnicode_EncodeDecimal` and then converted using - :c:func:`PyLong_FromString`. + value. .. versionadded:: 3.3 diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst index 7b179e22e290e1..7bb0dad2b6b6d5 100644 --- a/Doc/c-api/marshal.rst +++ b/Doc/c-api/marshal.rst @@ -43,7 +43,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - Return a C :c:type:`long` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`long` from the data stream in a :c:type:`FILE*` opened for reading. Only a 32-bit value can be read in using this function, regardless of the native size of :c:type:`long`. @@ -53,7 +53,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - Return a C :c:type:`short` from the data stream in a :c:type:`FILE\*` opened + Return a C :c:type:`short` from the data stream in a :c:type:`FILE*` opened for reading. Only a 16-bit value can be read in using this function, regardless of the native size of :c:type:`short`. @@ -63,7 +63,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` @@ -72,7 +72,7 @@ The following functions allow marshalled values to be read back in. .. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) - Return a Python object from the data stream in a :c:type:`FILE\*` opened for + Return a Python object from the data stream in a :c:type:`FILE*` opened for reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function assumes that no further objects will be read from the file, allowing it to aggressively load file data into memory so that the de-serialization can diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 8a8542f0479ec5..91b901d726e61c 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -109,7 +109,7 @@ zero bytes. .. c:function:: void* PyMem_RawMalloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as @@ -120,7 +120,7 @@ zero bytes. .. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or ``NULL`` if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct @@ -180,7 +180,7 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as @@ -191,7 +191,7 @@ The :ref:`default memory allocator ` uses the .. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or ``NULL`` if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct @@ -233,14 +233,14 @@ The following type-oriented macros are provided for convenience. Note that .. c:function:: TYPE* PyMem_New(TYPE, size_t n) Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:type:`TYPE\*`. The memory will not have + memory. Returns a pointer cast to :c:type:`TYPE*`. The memory will not have been initialized in any way. .. c:function:: TYPE* PyMem_Resize(void *p, TYPE, size_t n) Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE\*`. On return, + sizeof(TYPE))`` bytes. Returns a pointer cast to :c:type:`TYPE*`. On return, *p* will be a pointer to the new memory area, or ``NULL`` in the event of failure. @@ -282,7 +282,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Malloc(size_t n) - Allocates *n* bytes and returns a pointer of type :c:type:`void\*` to the + Allocates *n* bytes and returns a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as @@ -293,7 +293,7 @@ The :ref:`default object allocator ` uses the .. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:type:`void\*` to the allocated memory, or ``NULL`` if the + a pointer of type :c:type:`void*` to the allocated memory, or ``NULL`` if the request fails. The memory is initialized to zeros. Requesting zero elements or elements of size zero bytes returns a distinct @@ -388,7 +388,7 @@ Customize Memory Allocators Enum used to identify an allocator domain. Domains: - .. c:var:: PYMEM_DOMAIN_RAW + .. c:macro:: PYMEM_DOMAIN_RAW Functions: @@ -397,7 +397,7 @@ Customize Memory Allocators * :c:func:`PyMem_RawCalloc` * :c:func:`PyMem_RawFree` - .. c:var:: PYMEM_DOMAIN_MEM + .. c:macro:: PYMEM_DOMAIN_MEM Functions: @@ -406,7 +406,7 @@ Customize Memory Allocators * :c:func:`PyMem_Calloc` * :c:func:`PyMem_Free` - .. c:var:: PYMEM_DOMAIN_OBJ + .. c:macro:: PYMEM_DOMAIN_OBJ Functions: @@ -516,14 +516,14 @@ Customize pymalloc Arena Allocator +--------------------------------------------------+---------------------------------------+ | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, size_t size, void *ptr)`` | free an arena | + | ``void free(void *ctx, void *ptr, size_t size)`` | free an arena | +--------------------------------------------------+---------------------------------------+ -.. c:function:: PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) Get the arena allocator. -.. c:function:: PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) +.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) Set the arena allocator. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst index de429e5c11dc76..24f8c935302e8e 100644 --- a/Doc/c-api/memoryview.rst +++ b/Doc/c-api/memoryview.rst @@ -45,7 +45,8 @@ any other object. .. c:function:: int PyMemoryView_Check(PyObject *obj) Return true if the object *obj* is a memoryview object. It is not - currently allowed to create subclasses of :class:`memoryview`. + currently allowed to create subclasses of :class:`memoryview`. This + function always succeeds. .. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *mview) diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst index 0a5341cbbdf152..23852251dfe020 100644 --- a/Doc/c-api/method.rst +++ b/Doc/c-api/method.rst @@ -22,6 +22,7 @@ to bind a :c:data:`PyCFunction` to a class object. It replaces the former call Return true if *o* is an instance method object (has type :c:data:`PyInstanceMethod_Type`). The parameter must not be ``NULL``. + This function always succeeds. .. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) @@ -64,7 +65,7 @@ no longer available. .. c:function:: int PyMethod_Check(PyObject *o) Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be ``NULL``. + parameter must not be ``NULL``. This function always succeeds. .. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 8a415dfa30a35e..90766dca78bfa7 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -19,12 +19,13 @@ Module Objects .. c:function:: int PyModule_Check(PyObject *p) Return true if *p* is a module object, or a subtype of a module object. + This function always succeeds. .. c:function:: int PyModule_CheckExact(PyObject *p) Return true if *p* is a module object, but not a subtype of - :c:data:`PyModule_Type`. + :c:data:`PyModule_Type`. This function always succeeds. .. c:function:: PyObject* PyModule_NewObject(PyObject *name) @@ -325,7 +326,7 @@ The *m_slots* array must be terminated by a slot with id 0. The available slot types are: -.. c:var:: Py_mod_create +.. c:macro:: Py_mod_create Specifies a function that is called to create the module object itself. The *value* pointer of this slot must point to a function of the signature: @@ -357,7 +358,7 @@ The available slot types are: ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. -.. c:var:: Py_mod_exec +.. c:macro:: Py_mod_exec Specifies a function that is called to *execute* the module. This is equivalent to executing the code of a Python module: typically, diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst index b9c137ea352e78..05faa72346ed6b 100644 --- a/Doc/c-api/object.rst +++ b/Doc/c-api/object.rst @@ -291,7 +291,7 @@ Object Protocol is equivalent to the Python expression ``type(o)``. This function increments the reference count of the return value. There's really no reason to use this function instead of the common expression ``o->ob_type``, which returns a - pointer of type :c:type:`PyTypeObject\*`, except when the incremented reference + pointer of type :c:type:`PyTypeObject*`, except when the incremented reference count is needed. @@ -311,12 +311,12 @@ Object Protocol returned. This is the equivalent to the Python expression ``len(o)``. -.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t default) +.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) Return an estimated length for the object *o*. First try to return its actual length, then an estimate using :meth:`~object.__length_hint__`, and finally return the default value. On error return ``-1``. This is the - equivalent to the Python expression ``operator.length_hint(o, default)``. + equivalent to the Python expression ``operator.length_hint(o, defaultvalue)``. .. versionadded:: 3.4 diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst index 879f394d966cd0..84f34e7dae80be 100644 --- a/Doc/c-api/set.rst +++ b/Doc/c-api/set.rst @@ -53,28 +53,29 @@ the constructor functions work with any iterable Python object. .. c:function:: int PySet_Check(PyObject *p) Return true if *p* is a :class:`set` object or an instance of a subtype. + This function always succeeds. .. c:function:: int PyFrozenSet_Check(PyObject *p) Return true if *p* is a :class:`frozenset` object or an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: int PyAnySet_Check(PyObject *p) Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an - instance of a subtype. + instance of a subtype. This function always succeeds. .. c:function:: int PyAnySet_CheckExact(PyObject *p) Return true if *p* is a :class:`set` object or a :class:`frozenset` object but - not an instance of a subtype. + not an instance of a subtype. This function always succeeds. .. c:function:: int PyFrozenSet_CheckExact(PyObject *p) Return true if *p* is a :class:`frozenset` object but not an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: PyObject* PySet_New(PyObject *iterable) diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst index 48a58c6c6f7e36..8271d9acfb645e 100644 --- a/Doc/c-api/slice.rst +++ b/Doc/c-api/slice.rst @@ -14,7 +14,8 @@ Slice Objects .. c:function:: int PySlice_Check(PyObject *ob) - Return true if *ob* is a slice object; *ob* must not be ``NULL``. + Return true if *ob* is a slice object; *ob* must not be ``NULL``. This + function always succeeds. .. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 634e971952e8eb..0a0e03ff77ce64 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -141,7 +141,7 @@ Implementing functions and methods .. c:type:: PyCFunction Type of the functions used to implement most Python callables in C. - Functions of this type take two :c:type:`PyObject\*` parameters and return + Functions of this type take two :c:type:`PyObject*` parameters and return one such value. If the return value is ``NULL``, an exception shall have been set. If not ``NULL``, the return value is interpreted as the return value of the function as exposed in Python. The function must return a new @@ -220,10 +220,10 @@ Implementing functions and methods +------------------+---------------+-------------------------------+ The :attr:`ml_meth` is a C function pointer. The functions may be of different -types, but they always return :c:type:`PyObject\*`. If the function is not of +types, but they always return :c:type:`PyObject*`. If the function is not of the :c:type:`PyCFunction`, the compiler will require a cast in the method table. Even though :c:type:`PyCFunction` defines the first parameter as -:c:type:`PyObject\*`, it is common that the method implementation uses the +:c:type:`PyObject*`, it is common that the method implementation uses the specific C type of the *self* object. The :attr:`ml_flags` field is a bitfield which can include the following flags. @@ -235,7 +235,7 @@ There are these calling conventions: .. data:: METH_VARARGS This is the typical calling convention, where the methods have the type - :c:type:`PyCFunction`. The function expects two :c:type:`PyObject\*` values. + :c:type:`PyCFunction`. The function expects two :c:type:`PyObject*` values. The first one is the *self* object for methods; for module functions, it is the module object. The second parameter (often called *args*) is a tuple object representing all arguments. This parameter is typically processed @@ -256,7 +256,7 @@ There are these calling conventions: Fast calling convention supporting only positional arguments. The methods have the type :c:type:`_PyCFunctionFast`. The first parameter is *self*, the second parameter is a C array - of :c:type:`PyObject\*` values indicating the arguments and the third + of :c:type:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). This is not part of the :ref:`limited API `. @@ -270,7 +270,7 @@ There are these calling conventions: with methods of type :c:type:`_PyCFunctionFastWithKeywords`. Keyword arguments are passed the same way as in the :ref:`vectorcall protocol `: - there is an additional fourth :c:type:`PyObject\*` parameter + there is an additional fourth :c:type:`PyObject*` parameter which is a tuple representing the names of the keyword arguments (which are guaranteed to be strings) or possibly ``NULL`` if there are no keywords. The values of the keyword @@ -308,7 +308,7 @@ There are these calling conventions: Methods with a single object argument can be listed with the :const:`METH_O` flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. They have the type :c:type:`PyCFunction`, with the *self* parameter, and a - :c:type:`PyObject\*` parameter representing the single argument. + :c:type:`PyObject*` parameter representing the single argument. These two constants are not used to indicate the calling convention but the @@ -459,7 +459,7 @@ Accessing attributes of extension types | | | getter and setter | +-------------+------------------+-----------------------------------+ - The ``get`` function takes one :c:type:`PyObject\*` parameter (the + The ``get`` function takes one :c:type:`PyObject*` parameter (the instance) and a function pointer (the associated ``closure``):: typedef PyObject *(*getter)(PyObject *, void *); @@ -467,7 +467,7 @@ Accessing attributes of extension types It should return a new reference on success or ``NULL`` with a set exception on failure. - ``set`` functions take two :c:type:`PyObject\*` parameters (the instance and + ``set`` functions take two :c:type:`PyObject*` parameters (the instance and the value to be set) and a function pointer (the associated ``closure``):: typedef int (*setter)(PyObject *, PyObject *, void *); diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst index c14cb2d38fd54a..1dbf7dbb67ccd4 100644 --- a/Doc/c-api/tuple.rst +++ b/Doc/c-api/tuple.rst @@ -21,14 +21,14 @@ Tuple Objects .. c:function:: int PyTuple_Check(PyObject *p) - Return true if *p* is a tuple object or an instance of a subtype of the tuple - type. + Return true if *p* is a tuple object or an instance of a subtype of the + tuple type. This function always succeeds. .. c:function:: int PyTuple_CheckExact(PyObject *p) Return true if *p* is a tuple object, but not an instance of a subtype of the - tuple type. + tuple type. This function always succeeds. .. c:function:: PyObject* PyTuple_New(Py_ssize_t len) @@ -161,7 +161,7 @@ type. .. c:type:: PyStructSequence_Field Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:type:`PyObject\*`. The index in the + tuple, all fields are typed as :c:type:`PyObject*`. The index in the :attr:`fields` array of the :c:type:`PyStructSequence_Desc` determines which field of the struct sequence is described. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index f387279d143eec..d7f4bd67def131 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -23,12 +23,14 @@ Type Objects Return non-zero if the object *o* is a type object, including instances of types derived from the standard type object. Return 0 in all other cases. + This function always succeeds. .. c:function:: int PyType_CheckExact(PyObject *o) - Return non-zero if the object *o* is a type object, but not a subtype of the - standard type object. Return 0 in all other cases. + Return non-zero if the object *o* is a type object, but not a subtype of + the standard type object. Return 0 in all other cases. This function + always succeeds. .. c:function:: unsigned int PyType_ClearCache() @@ -95,6 +97,15 @@ Type Objects from a type's base class. Return ``0`` on success, or return ``-1`` and sets an exception on error. + .. note:: + If some of the base classes implements the GC protocol and the provided + type does not include the :const:`Py_TPFLAGS_HAVE_GC` in its flags, then + the GC protocol will be automatically implemented from its parents. On + the contrary, if the type being created does include + :const:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the + GC protocol itself by at least implementing the + :c:member:`~PyTypeObject.tp_traverse` handle. + .. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) Return the function pointer stored in the given slot. If the @@ -117,6 +128,13 @@ Type Objects If no module is associated with the given type, sets :py:class:`TypeError` and returns ``NULL``. + This function is usually used to get the module in which a method is defined. + Note that in such a method, ``PyType_GetModule(Py_TYPE(self))`` + may not return the intended result. + ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses + are not necessarily defined in the same module as their superclass. + See :c:type:`PyCMethod` to get the class that defines the method. + .. versionadded:: 3.9 .. c:function:: void* PyType_GetModuleState(PyTypeObject *type) @@ -148,12 +166,16 @@ The following functions and structs are used to create If *bases* is a tuple, the created heap type contains all types contained in it as base types. - If *bases* is ``NULL``, the *Py_tp_base* slot is used instead. + If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. + If that also is ``NULL``, the *Py_tp_base* slot is used instead. If that also is ``NULL``, the new type derives from :class:`object`. - The *module* must be a module object or ``NULL``. + The *module* argument can be used to record the module in which the new + class is defined. It must be a module object or ``NULL``. If not ``NULL``, the module is associated with the new type and can later be retreived with :c:func:`PyType_GetModule`. + The associated module is not inherited by subclasses; it must be specified + for each class individually. This function calls :c:func:`PyType_Ready` on the new type. @@ -215,7 +237,8 @@ The following functions and structs are used to create * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - The following fields cannot be set using :c:type:`PyType_Spec` and :c:type:`PyType_Slot`: + The following fields cannot be set at all using :c:type:`PyType_Spec` and + :c:type:`PyType_Slot`: * :c:member:`~PyTypeObject.tp_dict` * :c:member:`~PyTypeObject.tp_mro` @@ -229,13 +252,22 @@ The following functions and structs are used to create (see :ref:`PyMemberDef `) * :c:member:`~PyTypeObject.tp_vectorcall_offset` (see :ref:`PyMemberDef `) + + The following fields cannot be set using :c:type:`PyType_Spec` and + :c:type:`PyType_Slot` under the limited API: + * :c:member:`~PyBufferProcs.bf_getbuffer` * :c:member:`~PyBufferProcs.bf_releasebuffer` - Setting :c:data:`Py_tp_bases` may be problematic on some platforms. + Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be + problematic on some platforms. To avoid issues, use the *bases* argument of :py:func:`PyType_FromSpecWithBases` instead. + .. versionchanged:: 3.9 + + Slots in :c:type:`PyBufferProcs` in may be set in the unlimited API. + .. c:member:: void *PyType_Slot.pfunc The desired value of the slot. In most cases, this is a pointer diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst new file mode 100644 index 00000000000000..2d1175f4738b10 --- /dev/null +++ b/Doc/c-api/typehints.rst @@ -0,0 +1,46 @@ +.. highlight:: c + +.. _typehintobjects: + +Objects for Type Hinting +------------------------ + +Various built-in types for type hinting are provided. +Only :ref:`GenericAlias ` is exposed to C. + +.. c:function:: PyObject* Py_GenericAlias(PyObject *origin, PyObject *args) + + Create a :ref:`GenericAlias ` object. + Equivalent to calling the Python class + :class:`types.GenericAlias`. The *origin* and *args* arguments set the + ``GenericAlias``\ 's ``__origin__`` and ``__args__`` attributes respectively. + *origin* should be a :c:type:`PyTypeObject*`, and *args* can be a + :c:type:`PyTupleObject*` or any ``PyObject*``. If *args* passed is + not a tuple, a 1-tuple is automatically constructed and ``__args__`` is set + to ``(args,)``. + Minimal checking is done for the arguments, so the function will succeed even + if *origin* is not a type. + The ``GenericAlias``\ 's ``__parameters__`` attribute is constructed lazily + from ``__args__``. On failure, an exception is raised and ``NULL`` is + returned. + + Here's an example of how to make an extension type generic:: + + ... + static PyMethodDef my_obj_methods[] = { + // Other methods. + ... + {"__class_getitem__", (PyCFunction)Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} + ... + } + + .. seealso:: The data model method :meth:`__class_getitem__`. + + .. versionadded:: 3.9 + +.. c:var:: PyTypeObject Py_GenericAliasType + + The C type of the object returned by :c:func:`Py_GenericAlias`. Equivalent to + :class:`types.GenericAlias` in Python. + + .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ce4e8c926b2943..ddcb8ae3d0950c 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1223,11 +1223,25 @@ and :c:type:`PyType_Type` effectively act as defaults.) but the instance has no strong reference to the elements inside it, as they are allowed to be removed even if the instance is still alive). - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to :c:func:`local_traverse` to have these specific names; don't name them just anything. + Heap-allocated types (:const:`Py_TPFLAGS_HEAPTYPE`, such as those created + with :c:func:`PyType_FromSpec` and similar APIs) hold a reference to their + type. Their traversal function must therefore either visit + :c:func:`Py_TYPE(self) `, or delegate this responsibility by + calling ``tp_traverse`` of another heap-allocated type (such as a + heap-allocated superclass). + If they do not, the type object may not be garbage-collected. + + .. versionchanged:: 3.9 + + Heap-allocated types are expected to visit ``Py_TYPE(self)`` in + ``tp_traverse``. In earlier versions of Python, due to + `bug 40217 `_, doing this + may lead to crashes in subclasses. + **Inheritance:** Group: :const:`Py_TPFLAGS_HAVE_GC`, :attr:`tp_traverse`, :attr:`tp_clear` @@ -1334,7 +1348,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) The following macro is defined to ease writing rich comparison functions: - .. c:function:: PyObject \*Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, int op) + .. c:macro:: Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, op) Return ``Py_True`` or ``Py_False`` from the function, depending on the result of a comparison. @@ -1372,7 +1386,7 @@ and :c:type:`PyType_Type` effectively act as defaults.) than zero and contains the offset in the instance structure of the weak reference list head (ignoring the GC header, if present); this offset is used by :c:func:`PyObject_ClearWeakRefs` and the :c:func:`PyWeakref_\*` functions. The - instance structure needs to include a field of type :c:type:`PyObject\*` which is + instance structure needs to include a field of type :c:type:`PyObject*` which is initialized to ``NULL``. Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index b1787ed1ce89cf..8b1ed0810fb0cd 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -19,8 +19,7 @@ points must be below 1114112 (which is the full Unicode range). :c:type:`Py_UNICODE*` and UTF-8 representations are created on demand and cached in the Unicode object. The :c:type:`Py_UNICODE*` representation is deprecated -and inefficient; it should be avoided in performance- or memory-sensitive -situations. +and inefficient. Due to the transition between the old APIs and the new APIs, Unicode objects can internally be in two states depending on how they were created: @@ -34,6 +33,11 @@ can internally be in two states depending on how they were created: :c:type:`Py_UNICODE*` representation; you will have to call :c:func:`PyUnicode_READY` on them before calling any other API. +.. note:: + The "legacy" Unicode object will be removed in Python 3.12 with deprecated + APIs. All Unicode objects will be "canonical" since then. See :pep:`623` + for more information. + Unicode Type """""""""""" @@ -86,13 +90,13 @@ access internal read-only data of Unicode objects: .. c:function:: int PyUnicode_Check(PyObject *o) Return true if the object *o* is a Unicode object or an instance of a Unicode - subtype. + subtype. This function always succeeds. .. c:function:: int PyUnicode_CheckExact(PyObject *o) Return true if the object *o* is a Unicode object, but not an instance of a - subtype. + subtype. This function always succeeds. .. c:function:: int PyUnicode_READY(PyObject *o) @@ -107,6 +111,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + This API will be removed with :c:func:`PyUnicode_FromUnicode`. + .. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) @@ -138,6 +145,9 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 + .. deprecated-removed:: 3.10 3.12 + ``PyUnicode_WCHAR_KIND`` is deprecated. + .. c:function:: int PyUnicode_KIND(PyObject *o) @@ -188,7 +198,7 @@ access internal read-only data of Unicode objects: .. versionadded:: 3.3 -.. c:function:: PyUnicode_MAX_CHAR_VALUE(PyObject *o) +.. c:macro:: PyUnicode_MAX_CHAR_VALUE(o) Return the maximum code point that is suitable for creating another string based on *o*, which must be in the "canonical" representation. This is @@ -203,7 +213,7 @@ access internal read-only data of Unicode objects: code units (this includes surrogate pairs as 2 units). *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -213,7 +223,7 @@ access internal read-only data of Unicode objects: Return the size of the deprecated :c:type:`Py_UNICODE` representation in bytes. *o* has to be a Unicode object (not checked). - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using :c:func:`PyUnicode_GET_LENGTH`. @@ -235,7 +245,7 @@ access internal read-only data of Unicode objects: code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.12 Part of the old-style Unicode API, please migrate to using the :c:func:`PyUnicode_nBYTE_DATA` family of macros. @@ -423,7 +433,7 @@ APIs: If *u* is ``NULL``, this function behaves like :c:func:`PyUnicode_FromUnicode` with the buffer set to ``NULL``. This usage is deprecated in favor of - :c:func:`PyUnicode_New`. + :c:func:`PyUnicode_New`, and will be removed in Python 3.12. .. c:function:: PyObject *PyUnicode_FromString(const char *u) @@ -665,7 +675,7 @@ APIs: Deprecated Py_UNICODE APIs """""""""""""""""""""""""" -.. deprecated-removed:: 3.3 4.0 +.. deprecated-removed:: 3.3 3.12 These API functions are deprecated with the implementation of :pep:`393`. Extension modules can continue using them, as they will not be removed in Python @@ -687,8 +697,10 @@ Extension modules can continue using them, as they will not be removed in Python string content has been filled before using any of the access macros such as :c:func:`PyUnicode_KIND`. - Please migrate to using :c:func:`PyUnicode_FromKindAndData`, - :c:func:`PyUnicode_FromWideChar` or :c:func:`PyUnicode_New`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_FromKindAndData`, :c:func:`PyUnicode_FromWideChar`, or + :c:func:`PyUnicode_New`. .. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) @@ -701,9 +713,12 @@ Extension modules can continue using them, as they will not be removed in Python embedded null code points, which would cause the string to be truncated when used in most C functions. - Please migrate to using :c:func:`PyUnicode_AsUCS4`, - :c:func:`PyUnicode_AsWideChar`, :c:func:`PyUnicode_ReadChar` or similar new - APIs. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. + + .. deprecated-removed:: 3.3 3.10 .. c:function:: PyObject* PyUnicode_TransformDecimalToASCII(Py_UNICODE *s, Py_ssize_t size) @@ -712,6 +727,10 @@ Extension modules can continue using them, as they will not be removed in Python :c:type:`Py_UNICODE` buffer of the given *size* by ASCII digits 0--9 according to their decimal value. Return ``NULL`` if an exception occurs. + .. deprecated-removed:: 3.3 3.11 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`Py_UNICODE_TODECIMAL`. + .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) @@ -723,6 +742,11 @@ Extension modules can continue using them, as they will not be removed in Python .. versionadded:: 3.3 + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, + :c:func:`PyUnicode_ReadChar` or similar new APIs. + .. c:function:: Py_UNICODE* PyUnicode_AsUnicodeCopy(PyObject *unicode) @@ -743,7 +767,9 @@ Extension modules can continue using them, as they will not be removed in Python Return the size of the deprecated :c:type:`Py_UNICODE` representation, in code units (this includes surrogate pairs as 2 units). - Please migrate to using :c:func:`PyUnicode_GetLength`. + .. deprecated-removed:: 3.3 3.12 + Part of the old-style Unicode API, please migrate to using + :c:func:`PyUnicode_GET_LENGTH`. .. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj) @@ -1038,7 +1064,7 @@ These are the generic codec APIs: to be used is looked up using the Python codec registry. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1108,7 +1134,7 @@ These are the UTF-8 codec APIs: return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF8String`, :c:func:`PyUnicode_AsUTF8AndSize` or :c:func:`PyUnicode_AsEncodedString`. @@ -1182,7 +1208,7 @@ These are the UTF-32 codec APIs: Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF32String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1257,7 +1283,7 @@ These are the UTF-16 codec APIs: Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUTF16String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1295,7 +1321,7 @@ These are the UTF-7 codec APIs: nonzero, whitespace will be encoded in base-64. Both are set to zero for the Python "utf-7" codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsEncodedString`. @@ -1325,7 +1351,7 @@ These are the "Unicode Escape" codec APIs: Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Unicode-Escape and return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsUnicodeEscapeString`. @@ -1356,7 +1382,7 @@ These are the "Raw Unicode Escape" codec APIs: Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape and return a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsRawUnicodeEscapeString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1388,7 +1414,7 @@ ordinals and only these are accepted by the codecs during encoding. return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsLatin1String` or :c:func:`PyUnicode_AsEncodedString`. @@ -1420,7 +1446,7 @@ codes generate errors. return a Python bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsASCIIString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1472,7 +1498,7 @@ These are the mapping codec APIs: *mapping* object and return the result as a bytes object. Return ``NULL`` if an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_AsCharmapString` or :c:func:`PyUnicode_AsEncodedString`. @@ -1480,17 +1506,21 @@ These are the mapping codec APIs: The following codec API is special in that maps Unicode to Unicode. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *unicode, \ - PyObject *mapping, const char *errors) +.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) - Translate a Unicode object using the given *mapping* object and return the - resulting Unicode object. Return ``NULL`` if an exception was raised by the + Translate a string by applying a character mapping table to it and return the + resulting Unicode object. Return ``NULL`` if an exception was raised by the codec. - The *mapping* object must map Unicode ordinal integers to Unicode strings, - integers (which are then interpreted as Unicode ordinals) or ``None`` - (causing deletion of the character). Unmapped character ordinals (ones - which cause a :exc:`LookupError`) are left untouched and are copied as-is. + The mapping table must map Unicode ordinal integers to Unicode ordinal integers + or ``None`` (causing deletion of the character). + + Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries + and sequences work well. Unmapped character ordinals (ones which cause a + :exc:`LookupError`) are left untouched and are copied as-is. + + *errors* has the usual meaning for codecs. It may be ``NULL`` which indicates to + use the default error handling. .. c:function:: PyObject* PyUnicode_TranslateCharmap(const Py_UNICODE *s, Py_ssize_t size, \ @@ -1500,7 +1530,7 @@ The following codec API is special in that maps Unicode to Unicode. character *mapping* table to it and return the resulting Unicode object. Return ``NULL`` when an exception was raised by the codec. - .. deprecated-removed:: 3.3 4.0 + .. deprecated-removed:: 3.3 3.11 Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using :c:func:`PyUnicode_Translate`. or :ref:`generic codec based API ` @@ -1593,23 +1623,6 @@ They all return ``NULL`` or ``-1`` if an exception occurs. characters are not included in the resulting strings. -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, \ - const char *errors) - - Translate a string by applying a character mapping table to it and return the - resulting Unicode object. - - The mapping table must map Unicode ordinal integers to Unicode ordinal integers - or ``None`` (causing deletion of the character). - - Mapping tables need only provide the :meth:`__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. - - *errors* has the usual meaning for codecs. It may be ``NULL`` which indicates to - use the default error handling. - - .. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) Join a sequence of strings using the given *separator* and return the resulting diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index d6aecc394c9e70..551846ea6d7205 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -16,11 +16,11 @@ parameter. The available start symbols are :const:`Py_eval_input`, :const:`Py_file_input`, and :const:`Py_single_input`. These are described following the functions which accept them as parameters. -Note also that several of these functions take :c:type:`FILE\*` parameters. One +Note also that several of these functions take :c:type:`FILE*` parameters. One particular issue which needs to be handled carefully is that the :c:type:`FILE` structure for different C libraries can be different and incompatible. Under Windows (at least), it is possible for dynamically linked extensions to actually -use different libraries, so care should be taken that :c:type:`FILE\*` parameters +use different libraries, so care should be taken that :c:type:`FILE*` parameters are only passed to these functions if it is certain that they were created by the same library that the Python runtime is using. @@ -193,6 +193,8 @@ the same library that the Python runtime is using. :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set to ``NULL`` and *flags* set to ``0``. + .. deprecated-removed:: 3.9 3.10 + .. c:function:: struct _node* PyParser_SimpleParseStringFlags( const char *str, int start, int flags) @@ -200,6 +202,8 @@ the same library that the Python runtime is using. :c:func:`PyParser_SimpleParseStringFlagsFilename` below, leaving *filename* set to ``NULL``. + .. deprecated-removed:: 3.9 3.10 + .. c:function:: struct _node* PyParser_SimpleParseStringFlagsFilename( const char *str, const char *filename, int start, int flags) @@ -209,18 +213,24 @@ the same library that the Python runtime is using. many times. *filename* is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). + .. deprecated-removed:: 3.9 3.10 + .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) This is a simplified interface to :c:func:`PyParser_SimpleParseFileFlags` below, leaving *flags* set to ``0``. + .. deprecated-removed:: 3.9 3.10 + .. c:function:: struct _node* PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int flags) Similar to :c:func:`PyParser_SimpleParseStringFlagsFilename`, but the Python source code is read from *fp* instead of an in-memory string. + .. deprecated-removed:: 3.9 3.10 + .. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index e3a9bda54d671a..67698de77fef1d 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -13,17 +13,18 @@ as much as it can. .. c:function:: int PyWeakref_Check(ob) - Return true if *ob* is either a reference or proxy object. + Return true if *ob* is either a reference or proxy object. This function + always succeeds. .. c:function:: int PyWeakref_CheckRef(ob) - Return true if *ob* is a reference object. + Return true if *ob* is a reference object. This function always succeeds. .. c:function:: int PyWeakref_CheckProxy(ob) - Return true if *ob* is a proxy object. + Return true if *ob* is a proxy object. This function always succeeds. .. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback) diff --git a/Doc/conf.py b/Doc/conf.py index 12d74ea24ce4ac..079d17717f381c 100644 --- a/Doc/conf.py +++ b/Doc/conf.py @@ -15,7 +15,7 @@ extensions = ['sphinx.ext.coverage', 'sphinx.ext.doctest', 'pyspecific', 'c_annotations', 'escape4chm', - 'asdl_highlight'] + 'asdl_highlight', 'peg_highlight'] doctest_global_setup = ''' @@ -228,3 +228,13 @@ # Relative filename of the reference count data file. refcount_file = 'data/refcounts.dat' + +# Sphinx 2 and Sphinx 3 compatibility +# ----------------------------------- + +# bpo-40204: Allow Sphinx 2 syntax in the C domain +c_allow_pre_v3 = True + +# bpo-40204: Disable warnings on Sphinx 2 syntax of the C domain since the +# documentation is built with -W (warnings treated as errors). +c_warn_on_allowed_pre_v3 = False diff --git a/Doc/copyright.rst b/Doc/copyright.rst index 1b90d9f172c992..4191c0bb63a2c1 100644 --- a/Doc/copyright.rst +++ b/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2020 Python Software Foundation. All rights reserved. +Copyright © 2001-2021 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/Doc/data/python3.9.abi b/Doc/data/python3.9.abi new file mode 100644 index 00000000000000..228fefcd870056 --- /dev/null +++ b/Doc/data/python3.9.abi @@ -0,0 +1,13843 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index 4dacbe201d22a5..aac135113ad7eb 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -2306,6 +2306,11 @@ PyType_CheckExact:PyObject*:o:0: PyType_FromSpec:PyObject*::+1: PyType_FromSpec:PyType_Spec*:spec:: +PyType_FromModuleAndSpec:PyObject*::+1: +PyType_FromModuleAndSpec:PyObject*:module:+1: +PyType_FromModuleAndSpec:PyType_Spec*:spec:: +PyType_FromModuleAndSpec:PyObject*:bases:0: + PyType_FromSpecWithBases:PyObject*::+1: PyType_FromSpecWithBases:PyType_Spec*:spec:: PyType_FromSpecWithBases:PyObject*:bases:0: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst index 5f7b3bbc4f9174..02379946244d84 100644 --- a/Doc/distributing/index.rst +++ b/Doc/distributing/index.rst @@ -128,6 +128,7 @@ involved in creating and publishing a project: * `Project structure`_ * `Building and packaging the project`_ * `Uploading the project to the Python Packaging Index`_ +* `The .pypirc file`_ .. _Project structure: \ https://packaging.python.org/tutorials/distributing-packages/ @@ -135,6 +136,8 @@ involved in creating and publishing a project: https://packaging.python.org/tutorials/distributing-packages/#packaging-your-project .. _Uploading the project to the Python Packaging Index: \ https://packaging.python.org/tutorials/distributing-packages/#uploading-your-project-to-pypi +.. _The .pypirc file: \ + https://packaging.python.org/specifications/pypirc/ How do I...? diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index 25dc2934d29ef6..bc85a05ff2f850 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -410,7 +410,7 @@ optionally followed by an import of the module:: /* Optionally import the module; alternatively, import can be deferred until the embedded script imports it. */ - pmodule = PyImport_ImportModule("spam"); + PyObject *pmodule = PyImport_ImportModule("spam"); if (!pmodule) { PyErr_Print(); fprintf(stderr, "Error: could not import module 'spam'\n"); diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst index 0eb6ffd026f495..4da77e797d222c 100644 --- a/Doc/extending/newtypes_tutorial.rst +++ b/Doc/extending/newtypes_tutorial.rst @@ -416,7 +416,7 @@ But this would be risky. Our type doesn't restrict the type of the ``first`` member, so it could be any kind of object. It could have a destructor that causes code to be executed that tries to access the ``first`` member; or that destructor could release the -:term:`Global interpreter Lock` and let arbitrary code run in other +:term:`Global interpreter Lock ` and let arbitrary code run in other threads that accesses and modifies our object. To be paranoid and protect ourselves against this possibility, we almost diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst index 4e3cc575ee1964..d2b868ebb8290a 100644 --- a/Doc/faq/design.rst +++ b/Doc/faq/design.rst @@ -573,8 +573,7 @@ whether an instance or a class implements a particular ABC. The :class:`~collections.abc.MutableMapping`. For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. There is also a tool, -PyChecker, which can be used to find problems due to subclassing. +by an appropriate test discipline for components. A good test suite for a module can both provide a regression test and serve as a module interface specification and a set of examples. Many Python modules can @@ -602,7 +601,15 @@ test cases at all. Why is there no goto? --------------------- -You can use exceptions to provide a "structured goto" that even works across +In the 1970s people realized that unrestricted goto could lead +to messy "spaghetti" code that was hard to understand and revise. +In a high-level language, it is also unneeded as long as there +are ways to branch (in Python, with ``if`` statements and ``or``, +``and``, and ``if-else`` expressions) and loop (with ``while`` +and ``for`` statements, possibly containing ``continue`` and ``break``). + +One can also use exceptions to provide a "structured goto" +that works even across function calls. Many feel that exceptions can conveniently emulate all reasonable uses of the "go" or "goto" constructs of C, Fortran, and other languages. For example:: @@ -702,6 +709,15 @@ bindings are resolved at run-time in Python, and the second version only needs to perform the resolution once. +Why don't generators support the with statement? +------------------------------------------------ + +For technical reasons, a generator used directly as a context manager +would not work correctly. When, as is most common, a generator is used as +an iterator run to completion, no closing is needed. When it is, wrap +it as "contextlib.closing(generator)" in the 'with' statment. + + Why are colons required for the if/while/def/class statements? -------------------------------------------------------------- diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst index 3ef553e8acb43c..cf70f16c6fe328 100644 --- a/Doc/faq/general.rst +++ b/Doc/faq/general.rst @@ -17,12 +17,13 @@ What is Python? Python is an interpreted, interactive, object-oriented programming language. It incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. Python combines remarkable power with very clear syntax. -It has interfaces to many system calls and libraries, as well as to various -window systems, and is extensible in C or C++. It is also usable as an -extension language for applications that need a programmable interface. -Finally, Python is portable: it runs on many Unix variants, on the Mac, and on -Windows 2000 and later. +types, and classes. It supports multiple programming paradigms beyond +object-oriented programming, such as procedural and functional programming. +Python combines remarkable power with very clear syntax. It has interfaces to +many system calls and libraries, as well as to various window systems, and is +extensible in C or C++. It is also usable as an extension language for +applications that need a programmable interface. Finally, Python is portable: +it runs on many Unix variants including Linux and macOS, and on Windows. To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to Python `_ links to other @@ -141,9 +142,9 @@ to fix critical bugs. Alpha, beta and release candidate versions have an additional suffix. The suffix for an alpha version is "aN" for some small number N, the suffix for a beta version is "bN" for some small number N, and the suffix for a release -candidate version is "cN" for some small number N. In other words, all versions +candidate version is "rcN" for some small number N. In other words, all versions labeled 2.0aN precede the versions labeled 2.0bN, which precede versions labeled -2.0cN, and *those* precede 2.0. +2.0rcN, and *those* precede 2.0. You may also find version numbers with a "+" suffix, e.g. "2.2+". These are unreleased versions, built directly from the CPython development repository. In @@ -295,8 +296,8 @@ How stable is Python? --------------------- Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. Currently there are -usually around 18 months between major releases. +months since 1991, and this seems likely to continue. As of version 3.9, +Python will have a major new release every 12 months (:pep:`602`). The developers issue "bugfix" releases of older versions, so the stability of existing releases gradually improves. Bugfix releases, indicated by a third @@ -308,14 +309,14 @@ releases. The latest stable releases can always be found on the `Python download page `_. There are two production-ready versions of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it will not -be maintained after January 1, 2020 `_. +most widely used libraries. Although 2.x is still widely used, `it is not +maintained anymore `_. How many people are using Python? --------------------------------- -There are probably tens of thousands of users, though it's difficult to obtain -an exact count. +There are probably millions of users, though it's difficult to obtain an exact +count. Python is available for free download, so there are no sales figures, and it's available from many different sites and packaged with many Linux distributions, diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst index 781da467d18013..a322fa231669bc 100644 --- a/Doc/faq/gui.rst +++ b/Doc/faq/gui.rst @@ -92,7 +92,7 @@ FLTK Python bindings for `the FLTK toolkit `_, a simple yet powerful and mature cross-platform windowing system, are available from `the -PyFLTK project `_. +PyFLTK project `_. OpenGL ------ diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst index 61ffc5dbdaa773..ad51bb6ebf5952 100644 --- a/Doc/faq/programming.rst +++ b/Doc/faq/programming.rst @@ -29,50 +29,36 @@ Python distribution (normally available as Tools/scripts/idle), includes a graphical debugger. PythonWin is a Python IDE that includes a GUI debugger based on pdb. The -Pythonwin debugger colors breakpoints and has quite a few cool features such as -debugging non-Pythonwin programs. Pythonwin is available as part of the `Python -for Windows Extensions `__ project and -as a part of the ActivePython distribution (see -https://www.activestate.com/activepython\ ). - -`Boa Constructor `_ is an IDE and GUI -builder that uses wxWidgets. It offers visual frame creation and manipulation, -an object inspector, many views on the source like object browsers, inheritance -hierarchies, doc string generated html documentation, an advanced debugger, -integrated help, and Zope support. +PythonWin debugger colors breakpoints and has quite a few cool features such as +debugging non-PythonWin programs. PythonWin is available as part of +`pywin32 `_ project and +as a part of the +`ActivePython `_ distribution. `Eric `_ is an IDE built on PyQt and the Scintilla editing component. -Pydb is a version of the standard Python debugger pdb, modified for use with DDD -(Data Display Debugger), a popular graphical debugger front end. Pydb can be -found at http://bashdb.sourceforge.net/pydb/ and DDD can be found at -https://www.gnu.org/software/ddd. +`trepan3k `_ is a gdb-like debugger. + +`Visual Studio Code `_ is an IDE with debugging +tools that integrates with version-control software. There are a number of commercial Python IDEs that include graphical debuggers. They include: -* Wing IDE (https://wingware.com/) -* Komodo IDE (https://komodoide.com/) -* PyCharm (https://www.jetbrains.com/pycharm/) +* `Wing IDE `_ +* `Komodo IDE `_ +* `PyCharm `_ -Is there a tool to help find bugs or perform static analysis? +Are there tools to help find bugs or perform static analysis? ------------------------------------------------------------- Yes. -PyChecker is a static analysis tool that finds bugs in Python source code and -warns about code complexity and style. You can get PyChecker from -http://pychecker.sourceforge.net/. - -`Pylint `_ is another tool that checks -if a module satisfies a coding standard, and also makes it possible to write -plug-ins to add a custom feature. In addition to the bug checking that -PyChecker performs, Pylint offers some additional features such as checking line -length, whether variable names are well-formed according to your coding -standard, whether declared interfaces are fully implemented, and more. -https://docs.pylint.org/ provides a full list of Pylint's features. +`Pylint `_ and +`Pyflakes `_ do basic checking that will +help you catch bugs sooner. Static type checkers such as `Mypy `_, `Pyre `_, and @@ -80,6 +66,8 @@ Static type checkers such as `Mypy `_, source code. +.. _faq-create-standalone-binary: + How can I create a stand-alone binary from a Python script? ----------------------------------------------------------- @@ -103,13 +91,15 @@ only contains those built-in modules which are actually used in the program. It then compiles the generated C code and links it with the rest of the Python interpreter to form a self-contained binary which acts exactly like your script. -Obviously, freeze requires a C compiler. There are several other utilities -which don't. One is Thomas Heller's py2exe (Windows only) at - - http://www.py2exe.org/ - -Another tool is Anthony Tuininga's `cx_Freeze `_. +The following packages can help with the creation of console and GUI +executables: +* `Nuitka `_ (Cross-platform) +* `PyInstaller `_ (Cross-platform) +* `PyOxidizer `_ (Cross-platform) +* `cx_Freeze `_ (Cross-platform) +* `py2app `_ (macOS only) +* `py2exe `_ (Windows only) Are there coding standards or a style guide for Python programs? ---------------------------------------------------------------- @@ -518,14 +508,14 @@ desired effect in a number of ways. 1) By returning a tuple of the results:: - def func2(a, b): - a = 'new-value' # a and b are local names - b = b + 1 # assigned to new objects - return a, b # return new values - - x, y = 'old-value', 99 - x, y = func2(x, y) - print(x, y) # output: new-value 100 + >>> def func1(a, b): + ... a = 'new-value' # a and b are local names + ... b = b + 1 # assigned to new objects + ... return a, b # return new values + ... + >>> x, y = 'old-value', 99 + >>> func1(x, y) + ('new-value', 100) This is almost always the clearest solution. @@ -533,38 +523,41 @@ desired effect in a number of ways. 3) By passing a mutable (changeable in-place) object:: - def func1(a): - a[0] = 'new-value' # 'a' references a mutable list - a[1] = a[1] + 1 # changes a shared object - - args = ['old-value', 99] - func1(args) - print(args[0], args[1]) # output: new-value 100 + >>> def func2(a): + ... a[0] = 'new-value' # 'a' references a mutable list + ... a[1] = a[1] + 1 # changes a shared object + ... + >>> args = ['old-value', 99] + >>> func2(args) + >>> args + ['new-value', 100] 4) By passing in a dictionary that gets mutated:: - def func3(args): - args['a'] = 'new-value' # args is a mutable dictionary - args['b'] = args['b'] + 1 # change it in-place - - args = {'a': 'old-value', 'b': 99} - func3(args) - print(args['a'], args['b']) + >>> def func3(args): + ... args['a'] = 'new-value' # args is a mutable dictionary + ... args['b'] = args['b'] + 1 # change it in-place + ... + >>> args = {'a': 'old-value', 'b': 99} + >>> func3(args) + >>> args + {'a': 'new-value', 'b': 100} 5) Or bundle up values in a class instance:: - class callByRef: - def __init__(self, /, **args): - for key, value in args.items(): - setattr(self, key, value) - - def func4(args): - args.a = 'new-value' # args is a mutable callByRef - args.b = args.b + 1 # change object in-place - - args = callByRef(a='old-value', b=99) - func4(args) - print(args.a, args.b) + >>> class Namespace: + ... def __init__(self, /, **args): + ... for key, value in args.items(): + ... setattr(self, key, value) + ... + >>> def func4(args): + ... args.a = 'new-value' # args is a mutable Namespace + ... args.b = args.b + 1 # change object in-place + ... + >>> args = Namespace(a='old-value', b=99) + >>> func4(args) + >>> vars(args) + {'a': 'new-value', 'b': 100} There's almost never a good reason to get this complicated. @@ -953,7 +946,7 @@ There are various techniques. f() -* Use :func:`locals` or :func:`eval` to resolve the function name:: +* Use :func:`locals` to resolve the function name:: def myFunc(): print("hello") @@ -963,12 +956,6 @@ There are various techniques. f = locals()[fname] f() - f = eval(fname) - f() - - Note: Using :func:`eval` is slow and dangerous. If you don't have absolute - control over the contents of the string, someone could pass a string that - resulted in an arbitrary function being executed. Is there an equivalent to Perl's chomp() for removing trailing newlines from strings? ------------------------------------------------------------------------------------- @@ -1133,7 +1120,7 @@ trailing newline from a string. How do I iterate over a sequence in reverse order? -------------------------------------------------- -Use the :func:`reversed` built-in function, which is new in Python 2.4:: +Use the :func:`reversed` built-in function:: for x in reversed(sequence): ... # do something with x ... @@ -1141,11 +1128,6 @@ Use the :func:`reversed` built-in function, which is new in Python 2.4:: This won't touch your original sequence, but build a new copy with reversed order to iterate over. -With Python 2.3, you can use an extended slice syntax:: - - for x in sequence[::-1]: - ... # do something with x ... - How do you remove duplicates from a list? ----------------------------------------- @@ -1175,6 +1157,21 @@ This converts the list into a set, thereby removing duplicates, and then back into a list. +How do you remove multiple items from a list +-------------------------------------------- + +As with removing duplicates, explicitly iterating in reverse with a +delete condition is one possibility. However, it is easier and faster +to use slice replacement with an implicit or explicit forward iteration. +Here are three variations.:: + + mylist[:] = filter(keep_function, mylist) + mylist[:] = (x for x in mylist if keep_condition) + mylist[:] = [x for x in mylist if keep_condition] + +The list comprehension may be fastest. + + How do you make an array in Python? ----------------------------------- @@ -1377,20 +1374,6 @@ out the element you want. :: ['else', 'sort', 'to', 'something'] -An alternative for the last step is:: - - >>> result = [] - >>> for p in pairs: result.append(p[1]) - -If you find this more legible, you might prefer to use this instead of the final -list comprehension. However, it is almost twice as slow for long lists. Why? -First, the ``append()`` operation has to reallocate memory, and while it uses -some tricks to avoid doing that each time, it still has to do it occasionally, -and that costs quite a bit. Second, the expression "result.append" requires an -extra attribute lookup, and third, there's a speed reduction from having to make -all those function calls. - - Objects ======= @@ -1441,6 +1424,41 @@ single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also check whether an object is one of Python's built-in types, e.g. ``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. +Note that :func:`isinstance` also checks for virtual inheritance from an +:term:`abstract base class`. So, the test will return ``True`` for a +registered class even if hasn't directly or indirectly inherited from it. To +test for "true inheritance", scan the :term:`MRO` of the class: + +.. testcode:: + + from collections.abc import Mapping + + class P: + pass + + class C(P): + pass + + Mapping.register(P) + +.. doctest:: + + >>> c = C() + >>> isinstance(c, C) # direct + True + >>> isinstance(c, P) # indirect + True + >>> isinstance(c, Mapping) # virtual + True + + # Actual inheritance chain + >>> type(c).__mro__ + (, , ) + + # Test for "true inheritance" + >>> Mapping in type(c).__mro__ + False + Note that most programs do not use :func:`isinstance` on user-defined classes very often. If you are developing the classes yourself, a more proper object-oriented style is to define methods on the classes that encapsulate a @@ -1534,18 +1552,18 @@ provide the ``self`` argument. How can I organize my code to make it easier to change the base class? ---------------------------------------------------------------------- -You could define an alias for the base class, assign the real base class to it -before your class definition, and use the alias throughout your class. Then all +You could assign the base class to an alias and derive from the alias. Then all you have to change is the value assigned to the alias. Incidentally, this trick is also handy if you want to decide dynamically (e.g. depending on availability of resources) which base class to use. Example:: - BaseAlias = + class Base: + ... + + BaseAlias = Base class Derived(BaseAlias): - def meth(self): - BaseAlias.meth(self) - ... + ... How do I create static class data and static class methods? @@ -1723,6 +1741,93 @@ to the object: 13891296 +When can I rely on identity tests with the *is* operator? +--------------------------------------------------------- + +The ``is`` operator tests for object identity. The test ``a is b`` is +equivalent to ``id(a) == id(b)``. + +The most important property of an identity test is that an object is always +identical to itself, ``a is a`` always returns ``True``. Identity tests are +usually faster than equality tests. And unlike equality tests, identity tests +are guaranteed to return a boolean ``True`` or ``False``. + +However, identity tests can *only* be substituted for equality tests when +object identity is assured. Generally, there are three circumstances where +identity is guaranteed: + +1) Assignments create new names but do not change object identity. After the +assignment ``new = old``, it is guaranteed that ``new is old``. + +2) Putting an object in a container that stores object references does not +change object identity. After the list assignment ``s[0] = x``, it is +guaranteed that ``s[0] is x``. + +3) If an object is a singleton, it means that only one instance of that object +can exist. After the assignments ``a = None`` and ``b = None``, it is +guaranteed that ``a is b`` because ``None`` is a singleton. + +In most other circumstances, identity tests are inadvisable and equality tests +are preferred. In particular, identity tests should not be used to check +constants such as :class:`int` and :class:`str` which aren't guaranteed to be +singletons:: + + >>> a = 1000 + >>> b = 500 + >>> c = b + 500 + >>> a is c + False + + >>> a = 'Python' + >>> b = 'Py' + >>> c = b + 'thon' + >>> a is c + False + +Likewise, new instances of mutable containers are never identical:: + + >>> a = [] + >>> b = [] + >>> a is b + False + +In the standard library code, you will see several common patterns for +correctly using identity tests: + +1) As recommended by :pep:`8`, an identity test is the preferred way to check +for ``None``. This reads like plain English in code and avoids confusion with +other objects that may have boolean values that evaluate to false. + +2) Detecting optional arguments can be tricky when ``None`` is a valid input +value. In those situations, you can create an singleton sentinel object +guaranteed to be distinct from other objects. For example, here is how +to implement a method that behaves like :meth:`dict.pop`:: + + _sentinel = object() + + def pop(self, key, default=_sentinel): + if key in self: + value = self[key] + del self[key] + return value + if default is _sentinel: + raise KeyError(key) + return default + +3) Container implementations sometimes need to augment equality tests with +identity tests. This prevents the code from being confused by objects such as +``float('NaN')`` that are not equal to themselves. + +For example, here is the implementation of +:meth:`collections.abc.Sequence.__contains__`:: + + def __contains__(self, value): + for v in self: + if v is value or v == value: + return True + return False + + Modules ======= diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index a181086e9ce627..0153a4f316ee82 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -140,11 +140,8 @@ offender. How do I make an executable from a Python script? ------------------------------------------------- -See `cx_Freeze `_ for a distutils extension -that allows you to create console and GUI executables from Python code. -`py2exe `_, the most popular extension for building -Python 2.x-based executables, does not yet support Python 3 but a version that -does is in development. +See :ref:`faq-create-standalone-binary` for a list of tools that can be used to +make executables. Is a ``*.pyd`` file the same as a DLL? @@ -279,7 +276,7 @@ in batch mode. How do I check for a keypress without blocking? ----------------------------------------------- -Use the msvcrt module. This is a standard Windows-specific extension module. +Use the :mod:`msvcrt` module. This is a standard Windows-specific extension module. It defines a function ``kbhit()`` which checks whether a keyboard hit is present, and ``getch()`` which gets one character without echoing it. diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 6189cb045049c2..4fd01e0160c26c 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -189,6 +189,10 @@ Glossary A list of bytecode instructions can be found in the documentation for :ref:`the dis module `. + callback + A subroutine function which is passed as an argument to be executed at + some point in the future. + class A template for creating user-defined objects. Class definitions normally contain method definitions which operate on instances of the @@ -297,13 +301,20 @@ Glossary including functions, methods, properties, class methods, static methods, and reference to super classes. - For more information about descriptors' methods, see :ref:`descriptors`. + For more information about descriptors' methods, see :ref:`descriptors` + or the :ref:`Descriptor How To Guide `. dictionary An associative array, where arbitrary keys are mapped to values. The keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. Called a hash in Perl. + dictionary comprehension + A compact way to process all or part of the elements in an iterable and + return a dictionary with the results. ``results = {n: n ** 2 for n in + range(10)}`` generates a dictionary containing key ``n`` mapped to + value ``n ** 2``. See :ref:`comprehensions`. + dictionary view The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and :meth:`dict.items` are called dictionary views. They provide a dynamic @@ -472,6 +483,13 @@ Glossary See also the :term:`single dispatch` glossary entry, the :func:`functools.singledispatch` decorator, and :pep:`443`. + generic type + A :term:`type` that can be parameterized; typically a container like + :class:`list`. Used for :term:`type hints ` and + :term:`annotations `. + + See :pep:`483` for more details, and :mod:`typing` or + :ref:`generic alias type ` for its uses. GIL See :term:`global interpreter lock`. @@ -583,7 +601,7 @@ Glossary and :class:`tuple`) and some non-sequence types like :class:`dict`, :term:`file objects `, and objects of any classes you define with an :meth:`__iter__` method or with a :meth:`__getitem__` method - that implements :term:`Sequence` semantics. + that implements :term:`Sequence ` semantics. Iterables can be used in a :keyword:`for` loop and in many other places where a sequence is @@ -1020,7 +1038,13 @@ Glossary :meth:`index`, :meth:`__contains__`, and :meth:`__reversed__`. Types that implement this expanded interface can be registered explicitly using - :func:`~abc.register`. + :func:`~abc.ABCMeta.register`. + + set comprehension + A compact way to process all or part of the elements in an iterable and + return a set with the results. ``results = {c for c in 'abracadabra' if + c not in 'abc'}`` generates the set of strings ``{'r', 'd'}``. See + :ref:`comprehensions`. single dispatch A form of :term:`generic function` dispatch where the implementation is @@ -1080,19 +1104,15 @@ Glossary Type aliases are useful for simplifying :term:`type hints `. For example:: - from typing import List, Tuple - def remove_gray_shades( - colors: List[Tuple[int, int, int]]) -> List[Tuple[int, int, int]]: + colors: list[tuple[int, int, int]]) -> list[tuple[int, int, int]]: pass could be made more readable like this:: - from typing import List, Tuple - - Color = Tuple[int, int, int] + Color = tuple[int, int, int] - def remove_gray_shades(colors: List[Color]) -> List[Color]: + def remove_gray_shades(colors: list[Color]) -> list[Color]: pass See :mod:`typing` and :pep:`484`, which describe this functionality. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index 6928917dbe4200..9f0dd2f1f15506 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -1,3 +1,5 @@ +.. _descriptorhowto: + ====================== Descriptor HowTo Guide ====================== @@ -7,45 +9,516 @@ Descriptor HowTo Guide .. Contents:: + +:term:`Descriptors ` let objects customize attribute lookup, +storage, and deletion. + +This guide has four major sections: + +1) The "primer" gives a basic overview, moving gently from simple examples, + adding one feature at a time. Start here if you're new to descriptors. + +2) The second section shows a complete, practical descriptor example. If you + already know the basics, start there. + +3) The third section provides a more technical tutorial that goes into the + detailed mechanics of how descriptors work. Most people don't need this + level of detail. + +4) The last section has pure Python equivalents for built-in descriptors that + are written in C. Read this if you're curious about how functions turn + into bound methods or about the implementation of common tools like + :func:`classmethod`, :func:`staticmethod`, :func:`property`, and + :term:`__slots__`. + + +Primer +^^^^^^ + +In this primer, we start with the most basic possible example and then we'll +add new capabilities one by one. + + +Simple example: A descriptor that returns a constant +---------------------------------------------------- + +The :class:`Ten` class is a descriptor that always returns the constant ``10`` +from its :meth:`__get__` method: + +.. testcode:: + + class Ten: + def __get__(self, obj, objtype=None): + return 10 + +To use the descriptor, it must be stored as a class variable in another class: + +.. testcode:: + + class A: + x = 5 # Regular class attribute + y = Ten() # Descriptor instance + +An interactive session shows the difference between normal attribute lookup +and descriptor lookup: + +.. doctest:: + + >>> a = A() # Make an instance of class A + >>> a.x # Normal attribute lookup + 5 + >>> a.y # Descriptor lookup + 10 + +In the ``a.x`` attribute lookup, the dot operator finds the key ``x`` and the +value ``5`` in the class dictionary. In the ``a.y`` lookup, the dot operator +finds a descriptor instance, recognized by its ``__get__`` method, and calls +that method which returns ``10``. + +Note that the value ``10`` is not stored in either the class dictionary or the +instance dictionary. Instead, the value ``10`` is computed on demand. + +This example shows how a simple descriptor works, but it isn't very useful. +For retrieving constants, normal attribute lookup would be better. + +In the next section, we'll create something more useful, a dynamic lookup. + + +Dynamic lookups +--------------- + +Interesting descriptors typically run computations instead of returning +constants: + +.. testcode:: + + import os + + class DirectorySize: + + def __get__(self, obj, objtype=None): + return len(os.listdir(obj.dirname)) + + class Directory: + + size = DirectorySize() # Descriptor instance + + def __init__(self, dirname): + self.dirname = dirname # Regular instance attribute + +An interactive session shows that the lookup is dynamic — it computes +different, updated answers each time:: + + >>> s = Directory('songs') + >>> g = Directory('games') + >>> s.size # The songs directory has twenty files + 20 + >>> g.size # The games directory has three files + 3 + >>> os.remove('games/chess') # Delete a game + >>> g.size # File count is automatically updated + 2 + +Besides showing how descriptors can run computations, this example also +reveals the purpose of the parameters to :meth:`__get__`. The *self* +parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that +lets the :meth:`__get__` method learn the target directory. The *objtype* +parameter is the class *Directory*. + + +Managed attributes +------------------ + +A popular use for descriptors is managing access to instance data. The +descriptor is assigned to a public attribute in the class dictionary while the +actual data is stored as a private attribute in the instance dictionary. The +descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when +the public attribute is accessed. + +In the following example, *age* is the public attribute and *_age* is the +private attribute. When the public attribute is accessed, the descriptor logs +the lookup or update: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAgeAccess: + + def __get__(self, obj, objtype=None): + value = obj._age + logging.info('Accessing %r giving %r', 'age', value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', 'age', value) + obj._age = value + + class Person: + + age = LoggedAgeAccess() # Descriptor instance + + def __init__(self, name, age): + self.name = name # Regular instance attribute + self.age = age # Calls __set__() + + def birthday(self): + self.age += 1 # Calls both __get__() and __set__() + + +An interactive session shows that all access to the managed attribute *age* is +logged, but that the regular attribute *name* is not logged: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> mary = Person('Mary M', 30) # The initial age update is logged + INFO:root:Updating 'age' to 30 + >>> dave = Person('David D', 40) + INFO:root:Updating 'age' to 40 + + >>> vars(mary) # The actual data is in a private attribute + {'name': 'Mary M', '_age': 30} + >>> vars(dave) + {'name': 'David D', '_age': 40} + + >>> mary.age # Access the data and log the lookup + INFO:root:Accessing 'age' giving 30 + 30 + >>> mary.birthday() # Updates are logged as well + INFO:root:Accessing 'age' giving 30 + INFO:root:Updating 'age' to 31 + + >>> dave.name # Regular attribute lookup isn't logged + 'David D' + >>> dave.age # Only the managed attribute is logged + INFO:root:Accessing 'age' giving 40 + 40 + +One major issue with this example is that the private name *_age* is hardwired in +the *LoggedAgeAccess* class. That means that each instance can only have one +logged attribute and that its name is unchangeable. In the next example, +we'll fix that problem. + + +Customized names +---------------- + +When a class uses descriptors, it can inform each descriptor about which +variable name was used. + +In this example, the :class:`Person` class has two descriptor instances, +*name* and *age*. When the :class:`Person` class is defined, it makes a +callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can +be recorded, giving each descriptor its own *public_name* and *private_name*: + +.. testcode:: + + import logging + + logging.basicConfig(level=logging.INFO) + + class LoggedAccess: + + def __set_name__(self, owner, name): + self.public_name = name + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + value = getattr(obj, self.private_name) + logging.info('Accessing %r giving %r', self.public_name, value) + return value + + def __set__(self, obj, value): + logging.info('Updating %r to %r', self.public_name, value) + setattr(obj, self.private_name, value) + + class Person: + + name = LoggedAccess() # First descriptor instance + age = LoggedAccess() # Second descriptor instance + + def __init__(self, name, age): + self.name = name # Calls the first descriptor + self.age = age # Calls the second descriptor + + def birthday(self): + self.age += 1 + +An interactive session shows that the :class:`Person` class has called +:meth:`__set_name__` so that the field names would be recorded. Here +we call :func:`vars` to look up the descriptor without triggering it: + +.. doctest:: + + >>> vars(vars(Person)['name']) + {'public_name': 'name', 'private_name': '_name'} + >>> vars(vars(Person)['age']) + {'public_name': 'age', 'private_name': '_age'} + +The new class now logs access to both *name* and *age*: + +.. testcode:: + :hide: + + import logging, sys + logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) + +.. doctest:: + + >>> pete = Person('Peter P', 10) + INFO:root:Updating 'name' to 'Peter P' + INFO:root:Updating 'age' to 10 + >>> kate = Person('Catherine C', 20) + INFO:root:Updating 'name' to 'Catherine C' + INFO:root:Updating 'age' to 20 + +The two *Person* instances contain only the private names: + +.. doctest:: + + >>> vars(pete) + {'_name': 'Peter P', '_age': 10} + >>> vars(kate) + {'_name': 'Catherine C', '_age': 20} + + +Closing thoughts +---------------- + +A :term:`descriptor` is what we call any object that defines :meth:`__get__`, +:meth:`__set__`, or :meth:`__delete__`. + +Optionally, descriptors can have a :meth:`__set_name__` method. This is only +used in cases where a descriptor needs to know either the class where it was +created or the name of class variable it was assigned to. (This method, if +present, is called even if the class is not a descriptor.) + +Descriptors get invoked by the dot "operator" during attribute lookup. If a +descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, +the descriptor instance is returned without invoking it. + +Descriptors only work when used as class variables. When put in instances, +they have no effect. + +The main motivation for descriptors is to provide a hook allowing objects +stored in class variables to control what happens during attribute lookup. + +Traditionally, the calling class controls what happens during lookup. +Descriptors invert that relationship and allow the data being looked-up to +have a say in the matter. + +Descriptors are used throughout the language. It is how functions turn into +bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, +:func:`property`, and :func:`functools.cached_property` are all implemented as +descriptors. + + +Complete Practical Example +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +In this example, we create a practical and powerful tool for locating +notoriously hard to find data corruption bugs. + + +Validator class +--------------- + +A validator is a descriptor for managed attribute access. Prior to storing +any data, it verifies that the new value meets various type and range +restrictions. If those restrictions aren't met, it raises an exception to +prevent data corruption at its source. + +This :class:`Validator` class is both an :term:`abstract base class` and a +managed attribute descriptor: + +.. testcode:: + + from abc import ABC, abstractmethod + + class Validator(ABC): + + def __set_name__(self, owner, name): + self.private_name = '_' + name + + def __get__(self, obj, objtype=None): + return getattr(obj, self.private_name) + + def __set__(self, obj, value): + self.validate(value) + setattr(obj, self.private_name, value) + + @abstractmethod + def validate(self, value): + pass + +Custom validators need to inherit from :class:`Validator` and must supply a +:meth:`validate` method to test various restrictions as needed. + + +Custom validators +----------------- + +Here are three practical data validation utilities: + +1) :class:`OneOf` verifies that a value is one of a restricted set of options. + +2) :class:`Number` verifies that a value is either an :class:`int` or + :class:`float`. Optionally, it verifies that a value is between a given + minimum or maximum. + +3) :class:`String` verifies that a value is a :class:`str`. Optionally, it + validates a given minimum or maximum length. It can validate a + user-defined `predicate + `_ as well. + +.. testcode:: + + class OneOf(Validator): + + def __init__(self, *options): + self.options = set(options) + + def validate(self, value): + if value not in self.options: + raise ValueError(f'Expected {value!r} to be one of {self.options!r}') + + class Number(Validator): + + def __init__(self, minvalue=None, maxvalue=None): + self.minvalue = minvalue + self.maxvalue = maxvalue + + def validate(self, value): + if not isinstance(value, (int, float)): + raise TypeError(f'Expected {value!r} to be an int or float') + if self.minvalue is not None and value < self.minvalue: + raise ValueError( + f'Expected {value!r} to be at least {self.minvalue!r}' + ) + if self.maxvalue is not None and value > self.maxvalue: + raise ValueError( + f'Expected {value!r} to be no more than {self.maxvalue!r}' + ) + + class String(Validator): + + def __init__(self, minsize=None, maxsize=None, predicate=None): + self.minsize = minsize + self.maxsize = maxsize + self.predicate = predicate + + def validate(self, value): + if not isinstance(value, str): + raise TypeError(f'Expected {value!r} to be an str') + if self.minsize is not None and len(value) < self.minsize: + raise ValueError( + f'Expected {value!r} to be no smaller than {self.minsize!r}' + ) + if self.maxsize is not None and len(value) > self.maxsize: + raise ValueError( + f'Expected {value!r} to be no bigger than {self.maxsize!r}' + ) + if self.predicate is not None and not self.predicate(value): + raise ValueError( + f'Expected {self.predicate} to be true for {value!r}' + ) + + +Practical application +--------------------- + +Here's how the data validators can be used in a real class: + +.. testcode:: + + class Component: + + name = String(minsize=3, maxsize=10, predicate=str.isupper) + kind = OneOf('wood', 'metal', 'plastic') + quantity = Number(minvalue=0) + + def __init__(self, name, kind, quantity): + self.name = name + self.kind = kind + self.quantity = quantity + +The descriptors prevent invalid instances from being created: + +.. doctest:: + + >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase + Traceback (most recent call last): + ... + ValueError: Expected to be true for 'Widget' + + >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled + Traceback (most recent call last): + ... + ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} + + >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative + Traceback (most recent call last): + ... + ValueError: Expected -5 to be at least 0 + >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number + Traceback (most recent call last): + ... + TypeError: Expected 'V' to be an int or float + + >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid + + +Technical Tutorial +^^^^^^^^^^^^^^^^^^ + +What follows is a more technical tutorial for the mechanics and details of how +descriptors work. + + Abstract -------- Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Examines a custom descriptor and several built-in Python descriptors -including functions, properties, static methods, and class methods. Shows how -each works by giving a pure Python equivalent and a sample application. +called. Provides an example showing how object relational mappings work. Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works and an appreciation for the -elegance of its design. +creates a deeper understanding of how Python works. -Definition and Introduction +Definition and introduction --------------------------- -In general, a descriptor is an object attribute with "binding behavior", one -whose attribute access has been overridden by methods in the descriptor -protocol. Those methods are :meth:`__get__`, :meth:`__set__`, and -:meth:`__delete__`. If any of those methods are defined for an object, it is -said to be a descriptor. +In general, a descriptor is an attribute value that has one of the methods in +the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, +and :meth:`__delete__`. If any of those methods are defined for an +attribute, it is said to be a :term:`descriptor`. The default behavior for attribute access is to get, set, or delete the attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the base classes of ``type(a)`` excluding metaclasses. If the +continuing through the method resolution order of ``type(a)``. If the looked-up value is an object defining one of the descriptor methods, then Python may override the default behavior and invoke the descriptor method instead. Where this occurs in the precedence chain depends on which descriptor methods were defined. Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and :func:`super()`. -They are used throughout Python itself to implement the new style classes -introduced in version 2.2. Descriptors simplify the underlying C-code and offer -a flexible set of new tools for everyday Python programs. +behind properties, methods, static methods, class methods, and +:func:`super()`. They are used throughout Python itself. Descriptors +simplify the underlying C code and offer a flexible set of new tools for +everyday Python programs. -Descriptor Protocol +Descriptor protocol ------------------- ``descr.__get__(self, obj, type=None) -> value`` @@ -60,7 +533,7 @@ as an attribute. If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are typically used for methods but other uses are +non-data descriptors (they are often used for methods but other uses are possible). Data and non-data descriptors differ in how overrides are calculated with @@ -75,124 +548,404 @@ called. Defining the :meth:`__set__` method with an exception raising placeholder is enough to make it a data descriptor. -Invoking Descriptors --------------------- +Overview of descriptor invocation +--------------------------------- + +A descriptor can be called directly with ``desc.__get__(obj)`` or +``desc.__get__(None, cls)``. + +But it is more common for a descriptor to be invoked automatically from +attribute access. + +The expression ``obj.x`` looks up the attribute ``x`` in the chain of +namespaces for ``obj``. If the search finds a descriptor outside of the +instance ``__dict__``, its :meth:`__get__` method is invoked according to the +precedence rules listed below. + +The details of invocation depend on whether ``obj`` is an object, class, or +instance of super. + + +Invocation from an instance +--------------------------- + +Instance lookup scans through a chain of namespaces giving data descriptors +the highest priority, followed by instance variables, then non-data +descriptors, then class variables, and lastly :meth:`__getattr__` if it is +provided. + +If a descriptor is found for ``a.x``, then it is invoked with: +``desc.__get__(a, type(a))``. + +The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is +a pure Python equivalent: + +.. testcode:: + + def object_getattribute(obj, name): + "Emulate PyObject_GenericGetAttr() in Objects/object.c" + null = object() + objtype = type(obj) + cls_var = getattr(objtype, name, null) + descr_get = getattr(type(cls_var), '__get__', null) + if descr_get is not null: + if (hasattr(type(cls_var), '__set__') + or hasattr(type(cls_var), '__delete__')): + return descr_get(cls_var, obj, objtype) # data descriptor + if hasattr(obj, '__dict__') and name in vars(obj): + return vars(obj)[name] # instance variable + if descr_get is not null: + return descr_get(cls_var, obj, objtype) # non-data descriptor + if cls_var is not null: + return cls_var # class variable + raise AttributeError(name) + + +.. testcode:: + :hide: + + # Test the fidelity of object_getattribute() by comparing it with the + # normal object.__getattribute__(). The former will be accessed by + # square brackets and the latter by the dot operator. + + class Object: + + def __getitem__(obj, name): + try: + return object_getattribute(obj, name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + + class DualOperator(Object): + + x = 10 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + @property + def p3(self): + return 3 * self.x + + def m5(self, y): + return 5 * y + + def m7(self, y): + return 7 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + class DualOperatorWithSlots: + + __getitem__ = Object.__getitem__ + + __slots__ = ['z'] + + x = 15 + + def __init__(self, z): + self.z = z + + @property + def p2(self): + return 2 * self.x + + def m5(self, y): + return 5 * y + + def __getattr__(self, name): + return ('getattr_hook', self, name) + + +.. doctest:: + :hide: + + >>> a = DualOperator(11) + >>> vars(a).update(p3 = '_p3', m7 = '_m7') + >>> a.x == a['x'] == 10 + True + >>> a.z == a['z'] == 11 + True + >>> a.p2 == a['p2'] == 20 + True + >>> a.p3 == a['p3'] == 30 + True + >>> a.m5(100) == a.m5(100) == 500 + True + >>> a.m7 == a['m7'] == '_m7' + True + >>> a.g == a['g'] == ('getattr_hook', a, 'g') + True + + >>> b = DualOperatorWithSlots(22) + >>> b.x == b['x'] == 15 + True + >>> b.z == b['z'] == 22 + True + >>> b.p2 == b['p2'] == 30 + True + >>> b.m5(200) == b['m5'](200) == 1000 + True + >>> b.g == b['g'] == ('getattr_hook', b, 'g') + True + + +Interestingly, attribute lookup doesn't call :meth:`object.__getattribute__` +directly. Instead, both the dot operator and the :func:`getattr` function +perform attribute lookup by way of a helper function: + +.. testcode:: + + def getattr_hook(obj, name): + "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" + try: + return obj.__getattribute__(name) + except AttributeError: + if not hasattr(type(obj), '__getattr__'): + raise + return type(obj).__getattr__(obj, name) # __getattr__ + +.. doctest:: + :hide: + + + >>> class ClassWithGetAttr: + ... x = 123 + ... def __getattr__(self, attr): + ... return attr.upper() + ... + >>> cw = ClassWithGetAttr() + >>> cw.y = 456 + >>> getattr_hook(cw, 'x') + 123 + >>> getattr_hook(cw, 'y') + 456 + >>> getattr_hook(cw, 'z') + 'Z' + + >>> class ClassWithoutGetAttr: + ... x = 123 + ... + >>> cwo = ClassWithoutGetAttr() + >>> cwo.y = 456 + >>> getattr_hook(cwo, 'x') + 123 + >>> getattr_hook(cwo, 'y') + 456 + >>> getattr_hook(cwo, 'z') + Traceback (most recent call last): + ... + AttributeError: 'ClassWithoutGetAttr' object has no attribute 'z' + +So if :meth:`__getattr__` exists, it is called whenever :meth:`__getattribute__` +raises :exc:`AttributeError` (either directly or in one of the descriptor calls). + +Also, if a user calls :meth:`object.__getattribute__` directly, the +:meth:`__getattr__` hook is bypassed entirely. + + +Invocation from a class +----------------------- -A descriptor can be called directly by its method name. For example, -``d.__get__(obj)``. +The logic for a dotted lookup such as ``A.x`` is in +:meth:`type.__getattribute__`. The steps are similar to those for +:meth:`object.__getattribute__` but the instance dictionary lookup is replaced +by a search through the class's :term:`method resolution order`. -Alternatively, it is more common for a descriptor to be invoked automatically -upon attribute access. For example, ``obj.d`` looks up ``d`` in the dictionary -of ``obj``. If ``d`` defines the method :meth:`__get__`, then ``d.__get__(obj)`` -is invoked according to the precedence rules listed below. +If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. -The details of invocation depend on whether ``obj`` is an object or a class. +The full C implementation can be found in :c:func:`type_getattro()` and +:c:func:`_PyType_Lookup()` in :source:`Objects/typeobject.c`. -For objects, the machinery is in :meth:`object.__getattribute__` which -transforms ``b.x`` into ``type(b).__dict__['x'].__get__(b, type(b))``. The -implementation works through a precedence chain that gives data descriptors -priority over instance variables, instance variables priority over non-data -descriptors, and assigns lowest priority to :meth:`__getattr__` if provided. -The full C implementation can be found in :c:func:`PyObject_GenericGetAttr()` in -:source:`Objects/object.c`. -For classes, the machinery is in :meth:`type.__getattribute__` which transforms -``B.x`` into ``B.__dict__['x'].__get__(None, B)``. In pure Python, it looks -like:: +Invocation from super +--------------------- + +The logic for super's dotted lookup is in the :meth:`__getattribute__` method for +object returned by :class:`super()`. + +A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` +for the base class ``B`` immediately following ``A`` and then returns +``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned +unchanged. - def __getattribute__(self, key): - "Emulate type_getattro() in Objects/typeobject.c" - v = object.__getattribute__(self, key) - if hasattr(v, '__get__'): - return v.__get__(None, self) - return v +The full C implementation can be found in :c:func:`super_getattro()` in +:source:`Objects/typeobject.c`. A pure Python equivalent can be found in +`Guido's Tutorial +`_. + + +Summary of invocation logic +--------------------------- + +The mechanism for descriptors is embedded in the :meth:`__getattribute__()` +methods for :class:`object`, :class:`type`, and :func:`super`. The important points to remember are: -* descriptors are invoked by the :meth:`__getattribute__` method -* overriding :meth:`__getattribute__` prevents automatic descriptor calls +* Descriptors are invoked by the :meth:`__getattribute__` method. + +* Classes inherit this machinery from :class:`object`, :class:`type`, or + :func:`super`. + +* Overriding :meth:`__getattribute__` prevents automatic descriptor calls + because all the descriptor logic is in that method. + * :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. -* data descriptors always override instance dictionaries. -* non-data descriptors may be overridden by instance dictionaries. + different calls to :meth:`__get__`. The first includes the instance and may + include the class. The second puts in ``None`` for the instance and always + includes the class. -The object returned by ``super()`` also has a custom :meth:`__getattribute__` -method for invoking descriptors. The attribute lookup ``super(B, obj).m`` searches -``obj.__class__.__mro__`` for the base class ``A`` immediately following ``B`` -and then returns ``A.__dict__['m'].__get__(obj, B)``. If not a descriptor, -``m`` is returned unchanged. If not in the dictionary, ``m`` reverts to a -search using :meth:`object.__getattribute__`. +* Data descriptors always override instance dictionaries. -The implementation details are in :c:func:`super_getattro()` in -:source:`Objects/typeobject.c`. and a pure Python equivalent can be found in -`Guido's Tutorial`_. +* Non-data descriptors may be overridden by instance dictionaries. -.. _`Guido's Tutorial`: https://www.python.org/download/releases/2.2.3/descrintro/#cooperation -The details above show that the mechanism for descriptors is embedded in the -:meth:`__getattribute__()` methods for :class:`object`, :class:`type`, and -:func:`super`. Classes inherit this machinery when they derive from -:class:`object` or if they have a meta-class providing similar functionality. -Likewise, classes can turn-off descriptor invocation by overriding -:meth:`__getattribute__()`. +Automatic name notification +--------------------------- +Sometimes it is desirable for a descriptor to know what class variable name it +was assigned to. When a new class is created, the :class:`type` metaclass +scans the dictionary of the new class. If any of the entries are descriptors +and if they define :meth:`__set_name__`, that method is called with two +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. -Descriptor Example ------------------- +The implementation details are in :c:func:`type_new()` and +:c:func:`set_names()` in :source:`Objects/typeobject.c`. -The following code creates a class whose objects are data descriptors which -print a message for each get or set. Overriding :meth:`__getattribute__` is -alternate approach that could do this for every attribute. However, this -descriptor is useful for monitoring just a few chosen attributes:: +Since the update logic is in :meth:`type.__new__`, notifications only take +place at the time of class creation. If descriptors are added to the class +afterwards, :meth:`__set_name__` will need to be called manually. - class RevealAccess: - """A data descriptor that sets and returns values - normally and prints a message logging their access. - """ - def __init__(self, initval=None, name='var'): - self.val = initval - self.name = name +ORM example +----------- - def __get__(self, obj, objtype): - print('Retrieving', self.name) - return self.val +The following code is simplified skeleton showing how data descriptors could +be used to implement an `object relational mapping +`_. - def __set__(self, obj, val): - print('Updating', self.name) - self.val = val +The essential idea is that the data is stored in an external database. The +Python instances only hold keys to the database's tables. Descriptors take +care of lookups or updates: - >>> class MyClass: - ... x = RevealAccess(10, 'var "x"') - ... y = 5 - ... - >>> m = MyClass() - >>> m.x - Retrieving var "x" - 10 - >>> m.x = 20 - Updating var "x" - >>> m.x - Retrieving var "x" - 20 - >>> m.y - 5 +.. testcode:: + + class Field: + + def __set_name__(self, owner, name): + self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' + self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' + + def __get__(self, obj, objtype=None): + return conn.execute(self.fetch, [obj.key]).fetchone()[0] -The protocol is simple and offers exciting possibilities. Several use cases are -so common that they have been packaged into individual function calls. -Properties, bound methods, static methods, and class methods are all -based on the descriptor protocol. + def __set__(self, obj, value): + conn.execute(self.store, [value, obj.key]) + conn.commit() + +We can use the :class:`Field` class to define `models +`_ that describe the schema for +each table in a database: + +.. testcode:: + + class Movie: + table = 'Movies' # Table name + key = 'title' # Primary key + director = Field() + year = Field() + + def __init__(self, key): + self.key = key + + class Song: + table = 'Music' + key = 'title' + artist = Field() + year = Field() + genre = Field() + + def __init__(self, key): + self.key = key + +To use the models, first connect to the database:: + + >>> import sqlite3 + >>> conn = sqlite3.connect('entertainment.db') + +An interactive session shows how data is retrieved from the database and how +it can be updated: + +.. testsetup:: + + song_data = [ + ('Country Roads', 'John Denver', 1972), + ('Me and Bobby McGee', 'Janice Joplin', 1971), + ('Coal Miners Daughter', 'Loretta Lynn', 1970), + ] + + movie_data = [ + ('Star Wars', 'George Lucas', 1977), + ('Jaws', 'Steven Spielberg', 1975), + ('Aliens', 'James Cameron', 1986), + ] + + import sqlite3 + + conn = sqlite3.connect(':memory:') + conn.execute('CREATE TABLE Music (title text, artist text, year integer);') + conn.execute('CREATE INDEX MusicNdx ON Music (title);') + conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) + conn.execute('CREATE TABLE Movies (title text, director text, year integer);') + conn.execute('CREATE INDEX MovieNdx ON Music (title);') + conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) + conn.commit() + +.. doctest:: + + >>> Movie('Star Wars').director + 'George Lucas' + >>> jaws = Movie('Jaws') + >>> f'Released in {jaws.year} by {jaws.director}' + 'Released in 1975 by Steven Spielberg' + + >>> Song('Country Roads').artist + 'John Denver' + + >>> Movie('Star Wars').director = 'J.J. Abrams' + >>> Movie('Star Wars').director + 'J.J. Abrams' + + +Pure Python Equivalents +^^^^^^^^^^^^^^^^^^^^^^^ + +The descriptor protocol is simple and offers exciting possibilities. Several +use cases are so common that they have been prepackaged into built-in tools. +Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are +all based on the descriptor protocol. Properties ---------- Calling :func:`property` is a succinct way of building a data descriptor that -triggers function calls upon access to an attribute. Its signature is:: +triggers a function call upon access to an attribute. Its signature is:: + + property(fget=None, fset=None, fdel=None, doc=None) -> property - property(fget=None, fset=None, fdel=None, doc=None) -> property attribute +The documentation shows a typical use to define a managed attribute ``x``: -The documentation shows a typical use to define a managed attribute ``x``:: +.. testcode:: class C: def getx(self): return self.__x @@ -200,8 +953,24 @@ The documentation shows a typical use to define a managed attribute ``x``:: def delx(self): del self.__x x = property(getx, setx, delx, "I'm the 'x' property.") +.. doctest:: + :hide: + + >>> C.x.__doc__ + "I'm the 'x' property." + >>> c.x = 2.71828 + >>> c.x + 2.71828 + >>> del c.x + >>> c.x + Traceback (most recent call last): + ... + AttributeError: 'C' object has no attribute '_C__x' + To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent:: +here is a pure Python equivalent: + +.. testcode:: class Property: "Emulate PyProperty_Type() in Objects/descrobject.c" @@ -240,6 +1009,57 @@ here is a pure Python equivalent:: def deleter(self, fdel): return type(self)(self.fget, self.fset, fdel, self.__doc__) +.. testcode:: + :hide: + + # Verify the Property() emulation + + class CC: + def getx(self): + return self.__x + def setx(self, value): + self.__x = value + def delx(self): + del self.__x + x = Property(getx, setx, delx, "I'm the 'x' property.") + + # Now do it again but use the decorator style + + class CCC: + @Property + def x(self): + return self.__x + @x.setter + def x(self, value): + self.__x = value + @x.deleter + def x(self): + del self.__x + + +.. doctest:: + :hide: + + >>> cc = CC() + >>> hasattr(cc, 'x') + False + >>> cc.x = 33 + >>> cc.x + 33 + >>> del cc.x + >>> hasattr(cc, 'x') + False + + >>> ccc = CCC() + >>> hasattr(ccc, 'x') + False + >>> ccc.x = 333 + >>> ccc.x == 333 + True + >>> del ccc.x + >>> hasattr(ccc, 'x') + False + The :func:`property` builtin helps whenever a user interface has granted attribute access and then subsequent changes require the intervention of a method. @@ -248,104 +1068,143 @@ For instance, a spreadsheet class may grant access to a cell value through ``Cell('b10').value``. Subsequent improvements to the program require the cell to be recalculated on every access; however, the programmer does not want to affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor:: +to wrap access to the value attribute in a property data descriptor: + +.. testcode:: class Cell: - . . . - def getvalue(self): + ... + + @property + def value(self): "Recalculate the cell before returning value" self.recalc() return self._value - value = property(getvalue) + +Either the built-in :func:`property` or our :func:`Property` equivalent would +work in this example. -Functions and Methods +Functions and methods --------------------- Python's object oriented features are built upon a function based environment. Using non-data descriptors, the two are merged seamlessly. -Class dictionaries store methods as functions. In a class definition, methods -are written using :keyword:`def` or :keyword:`lambda`, the usual tools for -creating functions. Methods only differ from regular functions in that the -first argument is reserved for the object instance. By Python convention, the -instance reference is called *self* but may be called *this* or any other -variable name. +Functions stored in class dictionaries get turned into methods when invoked. +Methods only differ from regular functions in that the object instance is +prepended to the other arguments. By convention, the instance is called +*self* but could be called *this* or any other variable name. + +Methods can be created manually with :class:`types.MethodType` which is +roughly equivalent to: + +.. testcode:: -To support method calls, functions include the :meth:`__get__` method for -binding methods during attribute access. This means that all functions are -non-data descriptors which return bound methods when they are invoked from an -object. In pure Python, it works like this:: + class MethodType: + "Emulate PyMethod_Type in Objects/classobject.c" + + def __init__(self, func, obj): + self.__func__ = func + self.__self__ = obj + + def __call__(self, *args, **kwargs): + func = self.__func__ + obj = self.__self__ + return func(obj, *args, **kwargs) + +To support automatic creation of methods, functions include the +:meth:`__get__` method for binding methods during attribute access. This +means that functions are non-data descriptors that return bound methods +during dotted lookup from an instance. Here's how it works: + +.. testcode:: class Function: - . . . + ... + def __get__(self, obj, objtype=None): "Simulate func_descr_get() in Objects/funcobject.c" if obj is None: return self - return types.MethodType(self, obj) + return MethodType(self, obj) -Running the interpreter shows how the function descriptor works in practice:: +Running the following class in the interpreter shows how the function +descriptor works in practice: - >>> class D: - ... def f(self, x): - ... return x - ... - >>> d = D() +.. testcode:: + + class D: + def f(self, x): + return x + +The function has a :term:`qualified name` attribute to support introspection: + +.. doctest:: + + >>> D.f.__qualname__ + 'D.f' + +Accessing the function through the class dictionary does not invoke +:meth:`__get__`. Instead, it just returns the underlying function object:: - # Access through the class dictionary does not invoke __get__. - # It just returns the underlying function object. >>> D.__dict__['f'] - # Dotted access from a class calls __get__() which just returns - # the underlying function unchanged. +Dotted access from a class calls :meth:`__get__` which just returns the +underlying function unchanged:: + >>> D.f - # The function has a __qualname__ attribute to support introspection - >>> D.f.__qualname__ - 'D.f' +The interesting behavior occurs during dotted access from an instance. The +dotted lookup calls :meth:`__get__` which returns a bound method object:: - # Dotted access from an instance calls __get__() which returns the - # function wrapped in a bound method object + >>> d = D() >>> d.f > - # Internally, the bound method stores the underlying function, - # the bound instance, and the class of the bound instance. +Internally, the bound method stores the underlying function and the bound +instance:: + >>> d.f.__func__ - + + >>> d.f.__self__ <__main__.D object at 0x1012e1f98> - >>> d.f.__class__ - +If you have ever wondered where *self* comes from in regular methods or where +*cls* comes from in class methods, this is it! -Static Methods and Class Methods --------------------------------- + +Kinds of methods +---------------- Non-data descriptors provide a simple mechanism for variations on the usual patterns of binding functions into methods. To recap, functions have a :meth:`__get__` method so that they can be converted to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``klass.f(*args)`` +``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` becomes ``f(*args)``. This chart summarizes the binding and its two most useful variants: +-----------------+----------------------+------------------+ | Transformation | Called from an | Called from a | - | | Object | Class | + | | object | class | +=================+======================+==================+ | function | f(obj, \*args) | f(\*args) | +-----------------+----------------------+------------------+ | staticmethod | f(\*args) | f(\*args) | +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(klass, \*args) | + | classmethod | f(type(obj), \*args) | f(cls, \*args) | +-----------------+----------------------+------------------+ + +Static methods +-------------- + Static methods return the underlying function without changes. Calling either ``c.f`` or ``C.f`` is the equivalent of a direct lookup into ``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a @@ -364,21 +1223,27 @@ in statistical work but does not directly depend on a particular dataset. It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or ``Sample.erf(1.5) --> .9332``. -Since staticmethods return the underlying function with no changes, the example -calls are unexciting:: +Since static methods return the underlying function with no changes, the +example calls are unexciting: + +.. testcode:: + + class E: + @staticmethod + def f(x): + return x * 10 + +.. doctest:: - >>> class E: - ... def f(x): - ... print(x) - ... f = staticmethod(f) - ... >>> E.f(3) - 3 + 30 >>> E().f(3) - 3 + 30 Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this:: +:func:`staticmethod` would look like this: + +.. testcode:: class StaticMethod: "Emulate PyStaticMethod_Type() in Objects/funcobject.c" @@ -389,44 +1254,75 @@ Using the non-data descriptor protocol, a pure Python version of def __get__(self, obj, objtype=None): return self.f +.. testcode:: + :hide: + + class E_sim: + @StaticMethod + def f(x): + return x * 10 + +.. doctest:: + :hide: + + >>> E_sim.f(3) + 30 + >>> E_sim().f(3) + 30 + + +Class methods +------------- + Unlike static methods, class methods prepend the class reference to the argument list before calling the function. This format is the same -for whether the caller is an object or a class:: +for whether the caller is an object or a class: - >>> class E: - ... def f(klass, x): - ... return klass.__name__, x - ... f = classmethod(f) - ... - >>> print(E.f(3)) - ('E', 3) - >>> print(E().f(3)) - ('E', 3) +.. testcode:: + + class F: + @classmethod + def f(cls, x): + return cls.__name__, x + +.. doctest:: + >>> F.f(3) + ('F', 3) + >>> F().f(3) + ('F', 3) -This behavior is useful whenever the function only needs to have a class -reference and does not care about any underlying data. One use for classmethods -is to create alternate class constructors. In Python 2.3, the classmethod -:func:`dict.fromkeys` creates a new dictionary from a list of keys. The pure -Python equivalent is:: +This behavior is useful whenever the method only needs to have a class +reference and does not rely on data stored in a specific instance. One use for +class methods is to create alternate class constructors. For example, the +classmethod :func:`dict.fromkeys` creates a new dictionary from a list of +keys. The pure Python equivalent is: - class Dict: - . . . - def fromkeys(klass, iterable, value=None): +.. testcode:: + + class Dict(dict): + @classmethod + def fromkeys(cls, iterable, value=None): "Emulate dict_fromkeys() in Objects/dictobject.c" - d = klass() + d = cls() for key in iterable: d[key] = value return d - fromkeys = classmethod(fromkeys) -Now a new dictionary of unique keys can be constructed like this:: +Now a new dictionary of unique keys can be constructed like this: + +.. doctest:: - >>> Dict.fromkeys('abracadabra') - {'a': None, 'r': None, 'b': None, 'c': None, 'd': None} + >>> d = Dict.fromkeys('abracadabra') + >>> type(d) is Dict + True + >>> d + {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this:: +:func:`classmethod` would look like this: + +.. testcode:: class ClassMethod: "Emulate PyClassMethod_Type() in Objects/funcobject.c" @@ -434,10 +1330,313 @@ Using the non-data descriptor protocol, a pure Python version of def __init__(self, f): self.f = f - def __get__(self, obj, klass=None): - if klass is None: - klass = type(obj) - def newfunc(*args): - return self.f(klass, *args) - return newfunc + def __get__(self, obj, cls=None): + if cls is None: + cls = type(obj) + if hasattr(type(self.f), '__get__'): + return self.f.__get__(cls) + return MethodType(self.f, cls) + +.. testcode:: + :hide: + + # Verify the emulation works + class T: + @ClassMethod + def cm(cls, x, y): + return (cls, x, y) + + @ClassMethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + + +.. doctest:: + :hide: + + >>> T.cm(11, 22) + (, 11, 22) + + # Also call it from an instance + >>> t = T() + >>> t.cm(11, 22) + (, 11, 22) + + # Check the alternate path for chained descriptors + >>> T.__doc__ + "A doc for 'T'" + +The code path for ``hasattr(type(self.f), '__get__')`` was added in +Python 3.9 and makes it possible for :func:`classmethod` to support +chained decorators. For example, a classmethod and property could be +chained together: + +.. testcode:: + + class G: + @classmethod + @property + def __doc__(cls): + return f'A doc for {cls.__name__!r}' + +.. doctest:: + + >>> G.__doc__ + "A doc for 'G'" + + +Member objects and __slots__ +---------------------------- + +When a class defines ``__slots__``, it replaces instance dictionaries with a +fixed-length array of slot values. From a user point of view that has +several effects: + +1. Provides immediate detection of bugs due to misspelled attribute +assignments. Only attribute names specified in ``__slots__`` are allowed: + +.. testcode:: + + class Vehicle: + __slots__ = ('id_number', 'make', 'model') + +.. doctest:: + + >>> auto = Vehicle() + >>> auto.id_nubmer = 'VYE483814LQEX' + Traceback (most recent call last): + ... + AttributeError: 'Vehicle' object has no attribute 'id_nubmer' + +2. Helps create immutable objects where descriptors manage access to private +attributes stored in ``__slots__``: + +.. testcode:: + + class Immutable: + + __slots__ = ('_dept', '_name') # Replace the instance dictionary + + def __init__(self, dept, name): + self._dept = dept # Store to private attribute + self._name = name # Store to private attribute + + @property # Read-only descriptor + def dept(self): + return self._dept + + @property + def name(self): # Read-only descriptor + return self._name + +.. doctest:: + + >>> mark = Immutable('Botany', 'Mark Watney') + >>> mark.dept + 'Botany' + >>> mark.dept = 'Space Pirate' + Traceback (most recent call last): + ... + AttributeError: can't set attribute + >>> mark.location = 'Mars' + Traceback (most recent call last): + ... + AttributeError: 'Immutable' object has no attribute 'location' + +3. Saves memory. On a 64-bit Linux build, an instance with two attributes +takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight +design pattern `_ likely only +matters when a large number of instances are going to be created. + +4. Blocks tools like :func:`functools.cached_property` which require an +instance dictionary to function correctly: + +.. testcode:: + + from functools import cached_property + + class CP: + __slots__ = () # Eliminates the instance dict + + @cached_property # Requires an instance dict + def pi(self): + return 4 * sum((-1.0)**n / (2.0*n + 1.0) + for n in reversed(range(100_000))) + +.. doctest:: + + >>> CP().pi + Traceback (most recent call last): + ... + TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. + +It is not possible to create an exact drop-in pure Python version of +``__slots__`` because it requires direct access to C structures and control +over object memory allocation. However, we can build a mostly faithful +simulation where the actual C structure for slots is emulated by a private +``_slotvalues`` list. Reads and writes to that private structure are managed +by member descriptors: + +.. testcode:: + + null = object() + + class Member: + + def __init__(self, name, clsname, offset): + 'Emulate PyMemberDef in Include/structmember.h' + # Also see descr_new() in Objects/descrobject.c + self.name = name + self.clsname = clsname + self.offset = offset + + def __get__(self, obj, objtype=None): + 'Emulate member_get() in Objects/descrobject.c' + # Also see PyMember_GetOne() in Python/structmember.c + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + return value + + def __set__(self, obj, value): + 'Emulate member_set() in Objects/descrobject.c' + obj._slotvalues[self.offset] = value + + def __delete__(self, obj): + 'Emulate member_delete() in Objects/descrobject.c' + value = obj._slotvalues[self.offset] + if value is null: + raise AttributeError(self.name) + obj._slotvalues[self.offset] = null + + def __repr__(self): + 'Emulate member_repr() in Objects/descrobject.c' + return f'' + +The :meth:`type.__new__` method takes care of adding member objects to class +variables: + +.. testcode:: + + class Type(type): + 'Simulate how the type metaclass adds member objects for slots' + + def __new__(mcls, clsname, bases, mapping): + 'Emuluate type_new() in Objects/typeobject.c' + # type_new() calls PyTypeReady() which calls add_methods() + slot_names = mapping.get('slot_names', []) + for offset, name in enumerate(slot_names): + mapping[name] = Member(name, clsname, offset) + return type.__new__(mcls, clsname, bases, mapping) + +The :meth:`object.__new__` method takes care of creating instances that have +slots instead of an instance dictionary. Here is a rough simulation in pure +Python: + +.. testcode:: + + class Object: + 'Simulate how object.__new__() allocates memory for __slots__' + + def __new__(cls, *args): + 'Emulate object_new() in Objects/typeobject.c' + inst = super().__new__(cls) + if hasattr(cls, 'slot_names'): + empty_slots = [null] * len(cls.slot_names) + object.__setattr__(inst, '_slotvalues', empty_slots) + return inst + + def __setattr__(self, name, value): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__setattr__(name, value) + + def __delattr__(self, name): + 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' + cls = type(self) + if hasattr(cls, 'slot_names') and name not in cls.slot_names: + raise AttributeError( + f'{type(self).__name__!r} object has no attribute {name!r}' + ) + super().__delattr__(name) + +To use the simulation in a real class, just inherit from :class:`Object` and +set the :term:`metaclass` to :class:`Type`: + +.. testcode:: + + class H(Object, metaclass=Type): + 'Instance variables stored in slots' + + slot_names = ['x', 'y'] + + def __init__(self, x, y): + self.x = x + self.y = y + +At this point, the metaclass has loaded member objects for *x* and *y*:: + + >>> from pprint import pp + >>> pp(dict(vars(H))) + {'__module__': '__main__', + '__doc__': 'Instance variables stored in slots', + 'slot_names': ['x', 'y'], + '__init__': , + 'x': , + 'y': } + +.. doctest:: + :hide: + + # We test this separately because the preceding section is not + # doctestable due to the hex memory address for the __init__ function + >>> isinstance(vars(H)['x'], Member) + True + >>> isinstance(vars(H)['y'], Member) + True + +When instances are created, they have a ``slot_values`` list where the +attributes are stored: + +.. doctest:: + + >>> h = H(10, 20) + >>> vars(h) + {'_slotvalues': [10, 20]} + >>> h.x = 55 + >>> vars(h) + {'_slotvalues': [55, 20]} + +Misspelled or unassigned attributes will raise an exception: + +.. doctest:: + + >>> h.xz + Traceback (most recent call last): + ... + AttributeError: 'H' object has no attribute 'xz' + +.. doctest:: + :hide: + + # Examples for deleted attributes are not shown because this section + # is already a bit lengthy. We still test that code here. + >>> del h.x + >>> hasattr(h, 'x') + False + + # Also test the code for uninitialized slots + >>> class HU(Object, metaclass=Type): + ... slot_names = ['x', 'y'] + ... + >>> hu = HU() + >>> hasattr(hu, 'x') + False + >>> hasattr(hu, 'y') + False diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index 909deb5fed33ff..f0081e4ec28905 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -272,9 +272,7 @@ should instead read: Available static markers ------------------------ -.. I'm reusing the "c:function" type for markers - -.. c:function:: function__entry(str filename, str funcname, int lineno) +.. object:: function__entry(str filename, str funcname, int lineno) This marker indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. @@ -290,7 +288,7 @@ Available static markers * ``$arg3`` : ``int`` line number -.. c:function:: function__return(str filename, str funcname, int lineno) +.. object:: function__return(str filename, str funcname, int lineno) This marker is the converse of :c:func:`function__entry`, and indicates that execution of a Python function has ended (either via ``return``, or via an @@ -298,7 +296,7 @@ Available static markers The arguments are the same as for :c:func:`function__entry` -.. c:function:: line(str filename, str funcname, int lineno) +.. object:: line(str filename, str funcname, int lineno) This marker indicates a Python line is about to be executed. It is the equivalent of line-by-line tracing with a Python profiler. It is @@ -306,24 +304,24 @@ Available static markers The arguments are the same as for :c:func:`function__entry`. -.. c:function:: gc__start(int generation) +.. object:: gc__start(int generation) Fires when the Python interpreter starts a garbage collection cycle. ``arg0`` is the generation to scan, like :func:`gc.collect()`. -.. c:function:: gc__done(long collected) +.. object:: gc__done(long collected) Fires when the Python interpreter finishes a garbage collection cycle. ``arg0`` is the number of collected objects. -.. c:function:: import__find__load__start(str modulename) +.. object:: import__find__load__start(str modulename) Fires before :mod:`importlib` attempts to find and load the module. ``arg0`` is the module name. .. versionadded:: 3.7 -.. c:function:: import__find__load__done(str modulename, int found) +.. object:: import__find__load__done(str modulename, int found) Fires after :mod:`importlib`'s find_and_load function is called. ``arg0`` is the module name, ``arg1`` indicates if module was @@ -332,7 +330,7 @@ Available static markers .. versionadded:: 3.7 -.. c:function:: audit(str event, void *tuple) +.. object:: audit(str event, void *tuple) Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` @@ -375,14 +373,14 @@ If this file is installed in SystemTap's tapset directory (e.g. ``/usr/share/systemtap/tapset``), then these additional probepoints become available: -.. c:function:: python.function.entry(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.entry(str filename, str funcname, int lineno, frameptr) This probe point indicates that execution of a Python function has begun. It is only triggered for pure-Python (bytecode) functions. -.. c:function:: python.function.return(str filename, str funcname, int lineno, frameptr) +.. object:: python.function.return(str filename, str funcname, int lineno, frameptr) - This probe point is the converse of :c:func:`python.function.return`, and + This probe point is the converse of ``python.function.return``, and indicates that execution of a Python function has ended (either via ``return``, or via an exception). It is only triggered for pure-Python (bytecode) functions. diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 17f4ff6e474c23..5777a4c5031f85 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -1188,7 +1188,7 @@ to the above, as in the following example:: class StyleAdapter(logging.LoggerAdapter): def __init__(self, logger, extra=None): - super(StyleAdapter, self).__init__(logger, extra or {}) + super().__init__(logger, extra or {}) def log(self, level, msg, /, *args, **kwargs): if self.isEnabledFor(level): @@ -1368,7 +1368,7 @@ An example dictionary-based configuration ----------------------------------------- Below is an example of a logging configuration dictionary - it's taken from -the `documentation on the Django project `_. +the `documentation on the Django project `_. This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: LOGGING = { @@ -1424,7 +1424,7 @@ This dictionary is passed to :func:`~config.dictConfig` to put the configuration } For more information about this configuration, you can see the `relevant -section `_ +section `_ of the Django documentation. .. _cookbook-rotator-namer: @@ -1783,7 +1783,7 @@ as in the following complete example:: return tuple(o) elif isinstance(o, unicode): return o.encode('unicode_escape').decode('ascii') - return super(Encoder, self).default(o) + return super().default(o) class StructuredMessage: def __init__(self, message, /, **kwargs): @@ -2175,11 +2175,11 @@ class, as shown in the following example:: """ Format an exception so that it prints on a single line. """ - result = super(OneLineExceptionFormatter, self).formatException(exc_info) + result = super().formatException(exc_info) return repr(result) # or format into one line however you want to def format(self, record): - s = super(OneLineExceptionFormatter, self).format(record) + s = super().format(record) if record.exc_text: s = s.replace('\n', '') + '|' return s @@ -2813,7 +2813,7 @@ refer to the comments in the code snippet for more detailed information. # class QtHandler(logging.Handler): def __init__(self, slotfunc, *args, **kwargs): - super(QtHandler, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) self.signaller = Signaller() self.signaller.signal.connect(slotfunc) @@ -2883,7 +2883,7 @@ refer to the comments in the code snippet for more detailed information. } def __init__(self, app): - super(Window, self).__init__() + super().__init__() self.app = app self.textedit = te = QtWidgets.QPlainTextEdit(self) # Set whatever the default monospace font is for the platform diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index 046a88af62f0b3..12d525771ddc28 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -97,7 +97,7 @@ schemes. For example, you can make an FTP request like so:: In the case of HTTP, there are two extra things that Request objects allow you to do: First, you can pass data to be sent to the server. Second, you can pass -extra information ("metadata") *about* the data or the about request itself, to +extra information ("metadata") *about* the data or about the request itself, to the server - this information is sent as HTTP "headers". Let's look at each of these in turn. diff --git a/Doc/includes/sqlite3/countcursors.py b/Doc/includes/sqlite3/countcursors.py deleted file mode 100644 index 112f47703a2ff4..00000000000000 --- a/Doc/includes/sqlite3/countcursors.py +++ /dev/null @@ -1,17 +0,0 @@ -import sqlite3 - -class CountCursorsConnection(sqlite3.Connection): - def __init__(self, *args, **kwargs): - sqlite3.Connection.__init__(self, *args, **kwargs) - self.numcursors = 0 - - def cursor(self, *args, **kwargs): - self.numcursors += 1 - return sqlite3.Connection.cursor(self, *args, **kwargs) - -con = sqlite3.connect(":memory:", factory=CountCursorsConnection) -cur1 = con.cursor() -cur2 = con.cursor() -print(con.numcursors) - -con.close() diff --git a/Doc/includes/sqlite3/createdb.py b/Doc/includes/sqlite3/createdb.py deleted file mode 100644 index ee2950bdf81646..00000000000000 --- a/Doc/includes/sqlite3/createdb.py +++ /dev/null @@ -1,28 +0,0 @@ -# Not referenced from the documentation, but builds the database file the other -# code snippets expect. - -import sqlite3 -import os - -DB_FILE = "mydb" - -if os.path.exists(DB_FILE): - os.remove(DB_FILE) - -con = sqlite3.connect(DB_FILE) -cur = con.cursor() -cur.execute(""" - create table people - ( - name_last varchar(20), - age integer - ) - """) - -cur.execute("insert into people (name_last, age) values ('Yeltsin', 72)") -cur.execute("insert into people (name_last, age) values ('Putin', 51)") - -con.commit() - -cur.close() -con.close() diff --git a/Doc/includes/sqlite3/ctx_manager.py b/Doc/includes/sqlite3/ctx_manager.py index 6db77d45046e1f..2e1175ef44c641 100644 --- a/Doc/includes/sqlite3/ctx_manager.py +++ b/Doc/includes/sqlite3/ctx_manager.py @@ -1,19 +1,19 @@ import sqlite3 con = sqlite3.connect(":memory:") -con.execute("create table person (id integer primary key, firstname varchar unique)") +con.execute("create table lang (id integer primary key, name varchar unique)") # Successful, con.commit() is called automatically afterwards with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) + con.execute("insert into lang(name) values (?)", ("Python",)) # con.rollback() is called after the with block finishes with an exception, the # exception is still raised and must be caught try: with con: - con.execute("insert into person(firstname) values (?)", ("Joe",)) + con.execute("insert into lang(name) values (?)", ("Python",)) except sqlite3.IntegrityError: - print("couldn't add Joe twice") + print("couldn't add Python twice") # Connection object used as context manager only commits or rollbacks transactions, # so the connection object should be closed manually diff --git a/Doc/includes/sqlite3/execsql_fetchonerow.py b/Doc/includes/sqlite3/execsql_fetchonerow.py deleted file mode 100644 index 115bcb50c7c754..00000000000000 --- a/Doc/includes/sqlite3/execsql_fetchonerow.py +++ /dev/null @@ -1,19 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() -SELECT = "select name_last, age from people order by age, name_last" - -# 1. Iterate over the rows available from the cursor, unpacking the -# resulting sequences to yield their elements (name_last, age): -cur.execute(SELECT) -for (name_last, age) in cur: - print('%s is %d years old.' % (name_last, age)) - -# 2. Equivalently: -cur.execute(SELECT) -for row in cur: - print('%s is %d years old.' % (row[0], row[1])) - -con.close() diff --git a/Doc/includes/sqlite3/execsql_printall_1.py b/Doc/includes/sqlite3/execsql_printall_1.py deleted file mode 100644 index 19306e6e3ca7d1..00000000000000 --- a/Doc/includes/sqlite3/execsql_printall_1.py +++ /dev/null @@ -1,15 +0,0 @@ -import sqlite3 - -# Create a connection to the database file "mydb": -con = sqlite3.connect("mydb") - -# Get a Cursor object that operates in the context of Connection con: -cur = con.cursor() - -# Execute the SELECT statement: -cur.execute("select * from people order by age") - -# Retrieve all rows as a sequence and print that sequence: -print(cur.fetchall()) - -con.close() diff --git a/Doc/includes/sqlite3/execute_1.py b/Doc/includes/sqlite3/execute_1.py index 3466b1265a5bf2..ee0000e2b94a32 100644 --- a/Doc/includes/sqlite3/execute_1.py +++ b/Doc/includes/sqlite3/execute_1.py @@ -2,17 +2,21 @@ con = sqlite3.connect(":memory:") cur = con.cursor() -cur.execute("create table people (name_last, age)") - -who = "Yeltsin" -age = 72 +cur.execute("create table lang (name, first_appeared)") # This is the qmark style: -cur.execute("insert into people values (?, ?)", (who, age)) +cur.execute("insert into lang values (?, ?)", ("C", 1972)) -# And this is the named style: -cur.execute("select * from people where name_last=:who and age=:age", {"who": who, "age": age}) +# The qmark style used with executemany(): +lang_list = [ + ("Fortran", 1957), + ("Python", 1991), + ("Go", 2009), +] +cur.executemany("insert into lang values (?, ?)", lang_list) -print(cur.fetchone()) +# And this is the named style: +cur.execute("select * from lang where first_appeared=:year", {"year": 1972}) +print(cur.fetchall()) con.close() diff --git a/Doc/includes/sqlite3/insert_more_people.py b/Doc/includes/sqlite3/insert_more_people.py deleted file mode 100644 index 10cf937243f6da..00000000000000 --- a/Doc/includes/sqlite3/insert_more_people.py +++ /dev/null @@ -1,18 +0,0 @@ -import sqlite3 - -con = sqlite3.connect("mydb") - -cur = con.cursor() - -newPeople = ( - ('Lebed' , 53), - ('Zhirinovsky' , 57), - ) - -for person in newPeople: - cur.execute("insert into people (name_last, age) values (?, ?)", person) - -# The changes will not be saved unless the transaction is committed explicitly: -con.commit() - -con.close() diff --git a/Doc/includes/sqlite3/parse_colnames.py b/Doc/includes/sqlite3/parse_colnames.py deleted file mode 100644 index 5f01dbfe1cb524..00000000000000 --- a/Doc/includes/sqlite3/parse_colnames.py +++ /dev/null @@ -1,10 +0,0 @@ -import sqlite3 -import datetime - -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute('select ? as "x [timestamp]"', (datetime.datetime.now(),)) -dt = cur.fetchone()[0] -print(dt, type(dt)) - -con.close() diff --git a/Doc/includes/sqlite3/shared_cache.py b/Doc/includes/sqlite3/shared_cache.py deleted file mode 100644 index 30e71c935ff62e..00000000000000 --- a/Doc/includes/sqlite3/shared_cache.py +++ /dev/null @@ -1,6 +0,0 @@ -import sqlite3 - -# The shared cache is only available in SQLite versions 3.3.3 or later -# See the SQLite documentation for details. - -sqlite3.enable_shared_cache(True) diff --git a/Doc/includes/sqlite3/shortcut_methods.py b/Doc/includes/sqlite3/shortcut_methods.py index 98a39411495cba..48ea6fad15a898 100644 --- a/Doc/includes/sqlite3/shortcut_methods.py +++ b/Doc/includes/sqlite3/shortcut_methods.py @@ -1,23 +1,23 @@ import sqlite3 -persons = [ - ("Hugo", "Boss"), - ("Calvin", "Klein") - ] +langs = [ + ("C++", 1985), + ("Objective-C", 1984), +] con = sqlite3.connect(":memory:") # Create the table -con.execute("create table person(firstname, lastname)") +con.execute("create table lang(name, first_appeared)") # Fill the table -con.executemany("insert into person(firstname, lastname) values (?, ?)", persons) +con.executemany("insert into lang(name, first_appeared) values (?, ?)", langs) # Print the table contents -for row in con.execute("select firstname, lastname from person"): +for row in con.execute("select name, first_appeared from lang"): print(row) -print("I just deleted", con.execute("delete from person").rowcount, "rows") +print("I just deleted", con.execute("delete from lang").rowcount, "rows") # close is not a shortcut method and it's not called automatically, # so the connection object should be closed manually diff --git a/Doc/includes/sqlite3/simple_tableprinter.py b/Doc/includes/sqlite3/simple_tableprinter.py deleted file mode 100644 index 148a1707f948bc..00000000000000 --- a/Doc/includes/sqlite3/simple_tableprinter.py +++ /dev/null @@ -1,28 +0,0 @@ -import sqlite3 - -FIELD_MAX_WIDTH = 20 -TABLE_NAME = 'people' -SELECT = 'select * from %s order by age, name_last' % TABLE_NAME - -con = sqlite3.connect("mydb") - -cur = con.cursor() -cur.execute(SELECT) - -# Print a header. -for fieldDesc in cur.description: - print(fieldDesc[0].ljust(FIELD_MAX_WIDTH), end=' ') -print() # Finish the header with a newline. -print('-' * 78) - -# For each row, print the value of each field left-justified within -# the maximum possible width of that field. -fieldIndices = range(len(cur.description)) -for row in cur: - for fieldIndex in fieldIndices: - fieldValue = str(row[fieldIndex]) - print(fieldValue.ljust(FIELD_MAX_WIDTH), end=' ') - - print() # Finish the row with a newline. - -con.close() diff --git a/Doc/install/index.rst b/Doc/install/index.rst index e6d5a3e6ebde60..ae0d0294ef7773 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -1064,8 +1064,7 @@ normal libraries do. .. [#] This also means you could replace all existing COFF-libraries with OMF-libraries of the same name. -.. [#] Check https://www.sourceware.org/cygwin/ and http://www.mingw.org/ for more - information +.. [#] Check https://www.sourceware.org/cygwin/ for more information .. [#] Then you have no POSIX emulation available, but you also don't need :file:`cygwin1.dll`. diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst index e3d749e6017847..41399942d30308 100644 --- a/Doc/library/__future__.rst +++ b/Doc/library/__future__.rst @@ -90,7 +90,7 @@ language using this mechanism: | generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | | | | | *StopIteration handling inside generators* | +------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | 4.0 | :pep:`563`: | +| annotations | 3.7.0b1 | 3.10 | :pep:`563`: | | | | | *Postponed evaluation of annotations* | +------------------+-------------+--------------+---------------------------------------------+ diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst index 7328907730fb10..2e917cf7321b85 100644 --- a/Doc/library/aifc.rst +++ b/Doc/library/aifc.rst @@ -208,6 +208,7 @@ number of frames must be filled in. .. method:: aifc.tell() + :noindex: Return the current write position in the output file. Useful in combination with :meth:`setmark`. @@ -232,6 +233,7 @@ number of frames must be filled in. .. method:: aifc.close() + :noindex: Close the AIFF file. The header of the file is updated to reflect the actual size of the audio data. After calling this method, the object can no longer be diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index f8e39189686208..aa4713e75cd471 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -659,7 +659,7 @@ exit_on_error Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` method of an :class:`ArgumentParser`, it will exit with error info. -If the user would like catch errors manually, the feature can be enable by setting +If the user would like to catch errors manually, the feature can be enabled by setting ``exit_on_error`` to ``False``:: >>> parser = argparse.ArgumentParser(exit_on_error=False) @@ -696,7 +696,7 @@ The add_argument() method * const_ - A constant value required by some action_ and nargs_ selections. * default_ - The value produced if the argument is absent from the - command line. + command line and if it is absent from the namespace object. * type_ - The type to which the command-line argument should be converted. @@ -863,7 +863,7 @@ An example of a custom action:: ... def __init__(self, option_strings, dest, nargs=None, **kwargs): ... if nargs is not None: ... raise ValueError("nargs not allowed") - ... super(FooAction, self).__init__(option_strings, dest, **kwargs) + ... super().__init__(option_strings, dest, **kwargs) ... def __call__(self, parser, namespace, values, option_string=None): ... print('%r %r %r' % (namespace, values, option_string)) ... setattr(namespace, self.dest, values) @@ -961,19 +961,6 @@ values are: usage: PROG [-h] foo [foo ...] PROG: error: the following arguments are required: foo -.. _`argparse.REMAINDER`: - -* ``argparse.REMAINDER``. All the remaining command-line arguments are gathered - into a list. This is commonly useful for command line utilities that dispatch - to other command line utilities:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo') - >>> parser.add_argument('command') - >>> parser.add_argument('args', nargs=argparse.REMAINDER) - >>> print(parser.parse_args('--foo B cmd --arg1 XX ZZ'.split())) - Namespace(args=['--arg1', 'XX', 'ZZ'], command='cmd', foo='B') - If the ``nargs`` keyword argument is not provided, the number of arguments consumed is determined by the action_. Generally this means a single command-line argument will be consumed and a single item (not a list) will be produced. @@ -1019,6 +1006,14 @@ was not present at the command line:: >>> parser.parse_args([]) Namespace(foo=42) +If the target namespace already has an attribute set, the action *default* +will not over write it:: + + >>> parser = argparse.ArgumentParser() + >>> parser.add_argument('--foo', default=42) + >>> parser.parse_args([], namespace=argparse.Namespace(foo=101)) + Namespace(foo=101) + If the ``default`` value is a string, the parser parses the value as if it were a command-line argument. In particular, the parser applies any type_ conversion argument, if provided, before setting the attribute on the @@ -1055,63 +1050,70 @@ command-line argument was not present:: type ^^^^ -By default, :class:`ArgumentParser` objects read command-line arguments in as simple +By default, the parser reads command-line arguments in as simple strings. However, quite often the command-line string should instead be -interpreted as another type, like a :class:`float` or :class:`int`. The -``type`` keyword argument of :meth:`~ArgumentParser.add_argument` allows any -necessary type-checking and type conversions to be performed. Common built-in -types and functions can be used directly as the value of the ``type`` argument:: +interpreted as another type, such as a :class:`float` or :class:`int`. The +``type`` keyword for :meth:`~ArgumentParser.add_argument` allows any +necessary type-checking and type conversions to be performed. - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', type=int) - >>> parser.add_argument('bar', type=open) - >>> parser.parse_args('2 temp.txt'.split()) - Namespace(bar=<_io.TextIOWrapper name='temp.txt' encoding='UTF-8'>, foo=2) +If the type_ keyword is used with the default_ keyword, the type converter +is only applied if the default is a string. -See the section on the default_ keyword argument for information on when the -``type`` argument is applied to default arguments. +The argument to ``type`` can be any callable that accepts a single string. +If the function raises :exc:`ArgumentTypeError`, :exc:`TypeError`, or +:exc:`ValueError`, the exception is caught and a nicely formatted error +message is displayed. No other exception types are handled. -To ease the use of various types of files, the argparse module provides the -factory FileType which takes the ``mode=``, ``bufsize=``, ``encoding=`` and -``errors=`` arguments of the :func:`open` function. For example, -``FileType('w')`` can be used to create a writable file:: +Common built-in types and functions can be used as type converters: - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('bar', type=argparse.FileType('w')) - >>> parser.parse_args(['out.txt']) - Namespace(bar=<_io.TextIOWrapper name='out.txt' encoding='UTF-8'>) - -``type=`` can take any callable that takes a single string argument and returns -the converted value:: - - >>> def perfect_square(string): - ... value = int(string) - ... sqrt = math.sqrt(value) - ... if sqrt != int(sqrt): - ... msg = "%r is not a perfect square" % string - ... raise argparse.ArgumentTypeError(msg) - ... return value +.. testcode:: + + import argparse + import pathlib + + parser = argparse.ArgumentParser() + parser.add_argument('count', type=int) + parser.add_argument('distance', type=float) + parser.add_argument('street', type=ascii) + parser.add_argument('code_point', type=ord) + parser.add_argument('source_file', type=open) + parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) + parser.add_argument('datapath', type=pathlib.Path) + +User defined functions can be used as well: + +.. doctest:: + + >>> def hyphenated(string): + ... return '-'.join([word[:4] for word in string.casefold().split()]) ... - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=perfect_square) - >>> parser.parse_args(['9']) - Namespace(foo=9) - >>> parser.parse_args(['7']) - usage: PROG [-h] foo - PROG: error: argument foo: '7' is not a perfect square + >>> parser = argparse.ArgumentParser() + >>> _ = parser.add_argument('short_title', type=hyphenated) + >>> parser.parse_args(['"The Tale of Two Cities"']) + Namespace(short_title='"the-tale-of-two-citi') -The choices_ keyword argument may be more convenient for type checkers that -simply check against a range of values:: +The :func:`bool` function is not recommended as a type converter. All it does +is convert empty strings to ``False`` and non-empty strings to ``True``. +This is usually not what is desired. - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', type=int, choices=range(5, 10)) - >>> parser.parse_args(['7']) - Namespace(foo=7) - >>> parser.parse_args(['11']) - usage: PROG [-h] {5,6,7,8,9} - PROG: error: argument foo: invalid choice: 11 (choose from 5, 6, 7, 8, 9) +In general, the ``type`` keyword is a convenience that should only be used for +simple conversions that can only raise one of the three supported exceptions. +Anything with more interesting error-handling or resource management should be +done downstream after the arguments are parsed. + +For example, JSON or YAML conversions have complex error cases that require +better reporting than can be given by the ``type`` keyword. An +:exc:`~json.JSONDecodeError` would not be well formatted and a +:exc:`FileNotFound` exception would not be handled at all. -See the choices_ section for more details. +Even :class:`~argparse.FileType` has its limitations for use with the ``type`` +keyword. If one argument uses *FileType* and then a subsequent argument fails, +an error is reported but the file is not automatically closed. In this case, it +would be better to wait until after the parser has run and then use the +:keyword:`with`-statement to manage the files. + +For type checkers that simply check against a fixed set of values, consider +using the choices_ keyword instead. choices @@ -1147,6 +1149,14 @@ container should match the type_ specified:: Any container can be passed as the *choices* value, so :class:`list` objects, :class:`set` objects, and custom containers are all supported. +Use of :class:`enum.Enum` is not recommended because it is difficult to +control its appearance in usage, help, and error messages. + +Formatted choices overrides the default *metavar* which is normally derived +from *dest*. This is usually what you want because the user never sees the +*dest* parameter. If this display isn't desirable (perhaps because there are +many choices), just specify an explicit metavar_. + required ^^^^^^^^ @@ -1161,8 +1171,8 @@ keyword argument to :meth:`~ArgumentParser.add_argument`:: >>> parser.parse_args(['--foo', 'BAR']) Namespace(foo='BAR') >>> parser.parse_args([]) - usage: argparse.py [-h] [--foo FOO] - argparse.py: error: option --foo is required + usage: [-h] --foo FOO + : error: the following arguments are required: --foo As the example shows, if an option is marked as ``required``, :meth:`~ArgumentParser.parse_args` will report an error if that option is not diff --git a/Doc/library/array.rst b/Doc/library/array.rst index 78020738bf4f75..f2f7894e1bf0f0 100644 --- a/Doc/library/array.rst +++ b/Doc/library/array.rst @@ -160,8 +160,7 @@ The following data items and methods are also supported: Read *n* items (as machine values) from the :term:`file object` *f* and append them to the end of the array. If less than *n* items are available, :exc:`EOFError` is raised, but the items that were available are still - inserted into the array. *f* must be a real built-in file object; something - else with a :meth:`read` method won't do. + inserted into the array. .. method:: array.fromlist(list) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index 6c6ad01b842c8e..9328faf53839f9 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -80,10 +80,11 @@ Node classes end_col_offset Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset` - attributes. The :attr:`lineno` and :attr:`end_lineno` are the first and - last line numbers of source text span (1-indexed so the first line is line 1) - and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and + :attr:`end_lineno` are the first and last line numbers of the source + text span (1-indexed so the first line is line 1), and the + :attr:`col_offset` and :attr:`end_col_offset` are the corresponding UTF-8 byte offsets of the first and last tokens that generated the node. The UTF-8 offset is recorded because the parser uses UTF-8 internally. @@ -139,6 +140,11 @@ Node classes In the meantime, instantiating them will return an instance of a different class. +.. note:: + The descriptions of the specific node classes displayed here + were initially adapted from the fantastic `Green Tree + Snakes `__ project and + all its contributors. Literals ^^^^^^^^ @@ -1503,6 +1509,13 @@ Async and await fields as :class:`For` and :class:`With`, respectively. Only valid in the body of an :class:`AsyncFunctionDef`. +.. note:: + When a string is parsed by :func:`ast.parse`, operator nodes (subclasses + of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, + :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree + will be singletons. Changes to one will be reflected in all other + occurrences of the same value (e.g. :class:`ast.Add`). + :mod:`ast` Helpers ------------------ @@ -1553,7 +1566,12 @@ and classes for traversing abstract syntax trees: .. warning:: The produced code string will not necessarily be equal to the original - code that generated the :class:`ast.AST` object. + code that generated the :class:`ast.AST` object (without any compiler + optimizations, such as constant tuples/frozensets). + + .. warning:: + Trying to unparse a highly complex expression would result with + :exc:`RecursionError`. .. versionadded:: 3.9 @@ -1748,6 +1766,34 @@ and classes for traversing abstract syntax trees: Added the *indent* option. +.. _ast-compiler-flags: + +Compiler Flags +-------------- + +The following flags may be passed to :func:`compile` in order to change +effects on the compilation of a program: + +.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT + + Enables support for top-level ``await``, ``async for``, ``async with`` + and async comprehensions. + + .. versionadded:: 3.8 + +.. data:: PyCF_ONLY_AST + + Generates and returns an abstract syntax tree instead of returning a + compiled code object. + +.. data:: PyCF_TYPE_COMMENTS + + Enables support for :pep:`484` and :pep:`526` style type comments + (``# type: ``, ``# type: ignore ``). + + .. versionadded:: 3.8 + + .. _ast-cli: Command-Line Usage @@ -1795,5 +1841,24 @@ to stdout. Otherwise, the content is read from stdin. .. seealso:: - `Green Tree Snakes `_, an external documentation resource, has good - details on working with Python ASTs. + `Green Tree Snakes `_, an external + documentation resource, has good details on working with Python ASTs. + + `ASTTokens `_ + annotates Python ASTs with the positions of tokens and text in the source + code that generated them. This is helpful for tools that make source code + transformations. + + `leoAst.py `_ unifies the + token-based and parse-tree-based views of python programs by inserting + two-way links between tokens and ast nodes. + + `LibCST `_ parses code as a Concrete Syntax + Tree that looks like an ast tree and keeps all formatting details. It's + useful for building automated refactoring (codemod) applications and + linters. + + `Parso `_ is a Python parser that supports + error recovery and round-trip parsing for different Python versions (in + multiple Python versions). Parso is also able to list multiple syntax errors + in your python file. \ No newline at end of file diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst index d5b5659abc65e2..047e5bbc58ccad 100644 --- a/Doc/library/asyncio-api-index.rst +++ b/Doc/library/asyncio-api-index.rst @@ -48,6 +48,9 @@ await on multiple things with timeouts. * - :class:`Task` - Task object. + * - :func:`to_thread` + - Asychronously run a function in a separate OS thread. + * - :func:`run_coroutine_threadsafe` - Schedule a coroutine from another OS thread. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst index ff51c4fa3b20f4..02a00033152aba 100644 --- a/Doc/library/asyncio-dev.rst +++ b/Doc/library/asyncio-dev.rst @@ -73,7 +73,7 @@ event loop, no other Tasks can run in the same thread. When a Task executes an ``await`` expression, the running Task gets suspended, and the event loop executes the next Task. -To schedule a callback from a different OS thread, the +To schedule a :term:`callback` from another OS thread, the :meth:`loop.call_soon_threadsafe` method should be used. Example:: loop.call_soon_threadsafe(callback, *args) @@ -107,6 +107,16 @@ The :meth:`loop.run_in_executor` method can be used with a blocking code in a different OS thread without blocking the OS thread that the event loop runs in. +There is currently no way to schedule coroutines or callbacks directly +from a different process (such as one started with +:mod:`multiprocessing`). The :ref:`Event Loop Methods ` +section lists APIs that can read from pipes and watch file descriptors +without blocking the event loop. In addition, asyncio's +:ref:`Subprocess ` APIs provide a way to start a +process and communicate with it from the event loop. Lastly, the +aforementioned :meth:`loop.run_in_executor` method can also be used +with a :class:`concurrent.futures.ProcessPoolExecutor` to execute +code in a different process. .. _asyncio-handle-blocking: diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index d60a6ce95cdd87..66eb3faecd896e 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -191,8 +191,8 @@ Scheduling callbacks .. method:: loop.call_soon(callback, *args, context=None) - Schedule a *callback* to be called with *args* arguments at - the next iteration of the event loop. + Schedule the *callback* :term:`callback` to be called with + *args* arguments at the next iteration of the event loop. Callbacks are called in the order in which they are registered. Each callback will be called exactly once. @@ -321,7 +321,7 @@ Creating Futures and Tasks .. versionadded:: 3.5.2 -.. method:: loop.create_task(coro, \*, name=None) +.. method:: loop.create_task(coro, *, name=None) Schedule the execution of a :ref:`coroutine`. Return a :class:`Task` object. @@ -356,7 +356,7 @@ Opening network connections ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_connection(protocol_factory, \ - host=None, port=None, \*, ssl=None, \ + host=None, port=None, *, ssl=None, \ family=0, proto=0, flags=0, sock=None, \ local_addr=None, server_hostname=None, \ ssl_handshake_timeout=None, \ @@ -440,7 +440,7 @@ Opening network connections and *local_addr* should be specified. * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* + to bind the socket locally. The *local_host* and *local_port* are looked up using ``getaddrinfo()``, similarly to *host* and *port*. * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds @@ -482,7 +482,7 @@ Opening network connections that can be used directly in async/await code. .. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ - local_addr=None, remote_addr=None, \*, \ + local_addr=None, remote_addr=None, *, \ family=0, proto=0, flags=0, \ reuse_address=None, reuse_port=None, \ allow_broadcast=None, sock=None) @@ -518,7 +518,7 @@ Opening network connections Other arguments: * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* + to bind the socket locally. The *local_host* and *local_port* are looked up using :meth:`getaddrinfo`. * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used @@ -559,7 +559,7 @@ Opening network connections Added support for Windows. .. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ - path=None, \*, ssl=None, sock=None, \ + path=None, *, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) Create a Unix connection. @@ -592,7 +592,7 @@ Creating network servers ^^^^^^^^^^^^^^^^^^^^^^^^ .. coroutinemethod:: loop.create_server(protocol_factory, \ - host=None, port=None, \*, \ + host=None, port=None, *, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, \ sock=None, backlog=100, ssl=None, \ @@ -683,7 +683,7 @@ Creating network servers .. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ - \*, sock=None, backlog=100, ssl=None, \ + *, sock=None, backlog=100, ssl=None, \ ssl_handshake_timeout=None, start_serving=True) Similar to :meth:`loop.create_server` but works with the @@ -708,7 +708,7 @@ Creating network servers The *path* parameter can now be a :class:`~pathlib.Path` object. .. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ - sock, \*, ssl=None, ssl_handshake_timeout=None) + sock, *, ssl=None, ssl_handshake_timeout=None) Wrap an already accepted connection into a transport/protocol pair. @@ -773,7 +773,7 @@ TLS Upgrade ^^^^^^^^^^^ .. coroutinemethod:: loop.start_tls(transport, protocol, \ - sslcontext, \*, server_side=False, \ + sslcontext, *, server_side=False, \ server_hostname=None, ssl_handshake_timeout=None) Upgrade an existing transport-based connection to TLS. @@ -806,7 +806,7 @@ TLS Upgrade Watching file descriptors ^^^^^^^^^^^^^^^^^^^^^^^^^ -.. method:: loop.add_reader(fd, callback, \*args) +.. method:: loop.add_reader(fd, callback, *args) Start monitoring the *fd* file descriptor for read availability and invoke *callback* with the specified arguments once *fd* is available for @@ -816,7 +816,7 @@ Watching file descriptors Stop monitoring the *fd* file descriptor for read availability. -.. method:: loop.add_writer(fd, callback, \*args) +.. method:: loop.add_writer(fd, callback, *args) Start monitoring the *fd* file descriptor for write availability and invoke *callback* with the specified arguments once *fd* is available for @@ -930,7 +930,7 @@ convenient. :meth:`loop.create_server` and :func:`start_server`. .. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ - \*, fallback=True) + *, fallback=True) Send a file using high-performance :mod:`os.sendfile` if possible. Return the total number of bytes sent. @@ -964,7 +964,7 @@ convenient. DNS ^^^ -.. coroutinemethod:: loop.getaddrinfo(host, port, \*, family=0, \ +.. coroutinemethod:: loop.getaddrinfo(host, port, *, family=0, \ type=0, proto=0, flags=0) Asynchronous version of :meth:`socket.getaddrinfo`. @@ -1029,7 +1029,7 @@ Working with pipes Unix signals ^^^^^^^^^^^^ -.. method:: loop.add_signal_handler(signum, callback, \*args) +.. method:: loop.add_signal_handler(signum, callback, *args) Set *callback* as the handler for the *signum* signal. @@ -1064,7 +1064,7 @@ Unix signals Executing code in thread or process pools ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. awaitablemethod:: loop.run_in_executor(executor, func, \*args) +.. awaitablemethod:: loop.run_in_executor(executor, func, *args) Arrange for *func* to be called in the specified executor. @@ -1234,9 +1234,9 @@ async/await code consider using the high-level subprocesses. See :ref:`Subprocess Support on Windows ` for details. -.. coroutinemethod:: loop.subprocess_exec(protocol_factory, \*args, \ +.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from one or more string arguments specified by *args*. @@ -1316,9 +1316,9 @@ async/await code consider using the high-level conforms to the :class:`asyncio.SubprocessTransport` base class and *protocol* is an object instantiated by the *protocol_factory*. -.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, \*, \ +.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, *, \ stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, \*\*kwargs) + stderr=subprocess.PIPE, **kwargs) Create a subprocess from *cmd*, which can be a :class:`str` or a :class:`bytes` string encoded to the diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst index e1ac18eaf09882..939d4c1a84523a 100644 --- a/Doc/library/asyncio-future.rst +++ b/Doc/library/asyncio-future.rst @@ -31,7 +31,7 @@ Future Functions .. versionadded:: 3.5 -.. function:: ensure_future(obj, \*, loop=None) +.. function:: ensure_future(obj, *, loop=None) Return: @@ -58,7 +58,7 @@ Future Functions The function accepts any :term:`awaitable` object. -.. function:: wrap_future(future, \*, loop=None) +.. function:: wrap_future(future, *, loop=None) Wrap a :class:`concurrent.futures.Future` object in a :class:`asyncio.Future` object. @@ -67,7 +67,7 @@ Future Functions Future Object ============= -.. class:: Future(\*, loop=None) +.. class:: Future(*, loop=None) A Future represents an eventual result of an asynchronous operation. Not thread-safe. diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst index d9d3232d2408b3..ef6a0588506b52 100644 --- a/Doc/library/asyncio-policy.rst +++ b/Doc/library/asyncio-policy.rst @@ -159,7 +159,7 @@ implementation used by the asyncio event loop: .. class:: AbstractChildWatcher - .. method:: add_child_handler(pid, callback, \*args) + .. method:: add_child_handler(pid, callback, *args) Register a new child handler. @@ -209,7 +209,7 @@ implementation used by the asyncio event loop: It works reliably even when the asyncio event loop is run in a non-main OS thread. There is no noticeable overhead when handling a big number of children (*O(1)* each - time a child terminates), but stating a thread per process requires extra memory. + time a child terminates), but starting a thread per process requires extra memory. This watcher is used by default. @@ -219,7 +219,7 @@ implementation used by the asyncio event loop: This implementation registers a :py:data:`SIGCHLD` signal handler on instantiation. That can break third-party code that installs a custom handler for - `SIGCHLD`. signal). + :py:data:`SIGCHLD` signal. The watcher avoids disrupting other code spawning processes by polling every process explicitly on a :py:data:`SIGCHLD` signal. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 3079716f03ecc0..8b67f4b8957ef6 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -683,7 +683,7 @@ factories passed to the :meth:`loop.create_datagram_endpoint` method. Subprocess Protocols -------------------- -Datagram Protocol instances should be constructed by protocol +Subprocess Protocol instances should be constructed by protocol factories passed to the :meth:`loop.subprocess_exec` and :meth:`loop.subprocess_shell` methods. @@ -993,7 +993,7 @@ loop.subprocess_exec() and SubprocessProtocol An example of a subprocess protocol used to get the output of a subprocess and to wait for the subprocess exit. -The subprocess is created by th :meth:`loop.subprocess_exec` method:: +The subprocess is created by the :meth:`loop.subprocess_exec` method:: import asyncio import sys diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst index b76ed379c7f4c8..714de8d41a3503 100644 --- a/Doc/library/asyncio-stream.rst +++ b/Doc/library/asyncio-stream.rst @@ -48,7 +48,7 @@ The following top-level asyncio functions can be used to create and work with streams: -.. coroutinefunction:: open_connection(host=None, port=None, \*, \ +.. coroutinefunction:: open_connection(host=None, port=None, *, \ loop=None, limit=None, ssl=None, family=0, \ proto=0, flags=0, sock=None, local_addr=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -74,7 +74,7 @@ and work with streams: The *ssl_handshake_timeout* parameter. .. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, \*, loop=None, limit=None, \ + port=None, *, loop=None, limit=None, \ family=socket.AF_UNSPEC, \ flags=socket.AI_PASSIVE, sock=None, \ backlog=100, ssl=None, reuse_address=None, \ @@ -109,7 +109,7 @@ and work with streams: .. rubric:: Unix Sockets -.. coroutinefunction:: open_unix_connection(path=None, \*, loop=None, \ +.. coroutinefunction:: open_unix_connection(path=None, *, loop=None, \ limit=None, ssl=None, sock=None, \ server_hostname=None, ssl_handshake_timeout=None) @@ -132,7 +132,7 @@ and work with streams: .. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ - \*, loop=None, limit=None, sock=None, \ + *, loop=None, limit=None, sock=None, \ backlog=100, ssl=None, ssl_handshake_timeout=None, \ start_serving=True) diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst index 1d87d2f8005ec6..6ba24249f28d84 100644 --- a/Doc/library/asyncio-subprocess.rst +++ b/Doc/library/asyncio-subprocess.rst @@ -61,9 +61,9 @@ See also the `Examples`_ subsection. Creating Subprocesses ===================== -.. coroutinefunction:: create_subprocess_exec(program, \*args, stdin=None, \ +.. coroutinefunction:: create_subprocess_exec(program, *args, stdin=None, \ stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) + limit=None, **kwds) Create a subprocess. @@ -82,7 +82,7 @@ Creating Subprocesses .. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ stdout=None, stderr=None, loop=None, \ - limit=None, \*\*kwds) + limit=None, **kwds) Run the *cmd* shell command. @@ -95,14 +95,14 @@ Creating Subprocesses See the documentation of :meth:`loop.subprocess_shell` for other parameters. -.. important:: + .. important:: - It is the application's responsibility to ensure that all whitespace and - special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and special shell characters in strings that are going - to be used to construct shell commands. + It is the application's responsibility to ensure that all whitespace and + special characters are quoted appropriately to avoid `shell injection + `_ + vulnerabilities. The :func:`shlex.quote` function can be used to properly + escape whitespace and special shell characters in strings that are going + to be used to construct shell commands. .. deprecated-removed:: 3.8 3.10 @@ -110,10 +110,8 @@ Creating Subprocesses .. note:: - The default asyncio event loop implementation on **Windows** does not - support subprocesses. Subprocesses are available for Windows if a - :class:`ProactorEventLoop` is used. - See :ref:`Subprocess Support on Windows ` + Subprocesses are available for Windows if a :class:`ProactorEventLoop` is + used. See :ref:`Subprocess Support on Windows ` for details. .. seealso:: @@ -253,7 +251,7 @@ their completion. .. method:: kill() - Kill the child. + Kill the child process. On POSIX systems this method sends :py:data:`SIGKILL` to the child process. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst index 84a52cb2d57571..f62ce670fccbf9 100644 --- a/Doc/library/asyncio-sync.rst +++ b/Doc/library/asyncio-sync.rst @@ -104,8 +104,8 @@ Event that some event has happened. An Event object manages an internal flag that can be set to *true* - with the :meth:`set` method and reset to *false* with the - :meth:`clear` method. The :meth:`wait` method blocks until the + with the :meth:`~Event.set` method and reset to *false* with the + :meth:`clear` method. The :meth:`~Event.wait` method blocks until the flag is set to *true*. The flag is set to *false* initially. @@ -142,7 +142,7 @@ Event Wait until the event is set. If the event is set, return ``True`` immediately. - Otherwise block until another task calls :meth:`set`. + Otherwise block until another task calls :meth:`~Event.set`. .. method:: set() @@ -155,8 +155,8 @@ Event Clear (unset) the event. - Tasks awaiting on :meth:`wait` will now block until the - :meth:`set` method is called again. + Tasks awaiting on :meth:`~Event.wait` will now block until the + :meth:`~Event.set` method is called again. .. method:: is_set() diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 2e963398d93001..cba0bae26d9062 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -210,7 +210,7 @@ is :meth:`loop.run_in_executor`. Running an asyncio Program ========================== -.. function:: run(coro, \*, debug=False) +.. function:: run(coro, *, debug=False) Execute the :term:`coroutine` *coro* and return the result. @@ -247,7 +247,7 @@ Running an asyncio Program Creating Tasks ============== -.. function:: create_task(coro, \*, name=None) +.. function:: create_task(coro, *, name=None) Wrap the *coro* :ref:`coroutine ` into a :class:`Task` and schedule its execution. Return the Task object. @@ -283,7 +283,7 @@ Creating Tasks Sleeping ======== -.. coroutinefunction:: sleep(delay, result=None, \*, loop=None) +.. coroutinefunction:: sleep(delay, result=None, *, loop=None) Block for *delay* seconds. @@ -293,6 +293,10 @@ Sleeping ``sleep()`` always suspends the current task, allowing other tasks to run. + Setting the delay to 0 provides an optimized path to allow other + tasks to run. This can be used by long-running functions to avoid + blocking the event loop for the full duration of the function call. + .. deprecated-removed:: 3.8 3.10 The *loop* parameter. @@ -319,7 +323,7 @@ Sleeping Running Tasks Concurrently ========================== -.. awaitablefunction:: gather(\*aws, loop=None, return_exceptions=False) +.. awaitablefunction:: gather(*aws, loop=None, return_exceptions=False) Run :ref:`awaitable objects ` in the *aws* sequence *concurrently*. @@ -360,32 +364,43 @@ Running Tasks Concurrently async def factorial(name, number): f = 1 for i in range(2, number + 1): - print(f"Task {name}: Compute factorial({i})...") + print(f"Task {name}: Compute factorial({number}), currently i={i}...") await asyncio.sleep(1) f *= i print(f"Task {name}: factorial({number}) = {f}") + return f async def main(): # Schedule three calls *concurrently*: - await asyncio.gather( + L = await asyncio.gather( factorial("A", 2), factorial("B", 3), factorial("C", 4), ) + print(L) asyncio.run(main()) # Expected output: # - # Task A: Compute factorial(2)... - # Task B: Compute factorial(2)... - # Task C: Compute factorial(2)... + # Task A: Compute factorial(2), currently i=2... + # Task B: Compute factorial(3), currently i=2... + # Task C: Compute factorial(4), currently i=2... # Task A: factorial(2) = 2 - # Task B: Compute factorial(3)... - # Task C: Compute factorial(3)... + # Task B: Compute factorial(3), currently i=3... + # Task C: Compute factorial(4), currently i=3... # Task B: factorial(3) = 6 - # Task C: Compute factorial(4)... + # Task C: Compute factorial(4), currently i=4... # Task C: factorial(4) = 24 + # [2, 6, 24] + + .. note:: + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. .. versionchanged:: 3.7 If the *gather* itself is cancelled, the cancellation is @@ -395,7 +410,7 @@ Running Tasks Concurrently Shielding From Cancellation =========================== -.. awaitablefunction:: shield(aw, \*, loop=None) +.. awaitablefunction:: shield(aw, *, loop=None) Protect an :ref:`awaitable object ` from being :meth:`cancelled `. @@ -435,7 +450,7 @@ Shielding From Cancellation Timeouts ======== -.. coroutinefunction:: wait_for(aw, timeout, \*, loop=None) +.. coroutinefunction:: wait_for(aw, timeout, *, loop=None) Wait for the *aw* :ref:`awaitable ` to complete with a timeout. @@ -492,14 +507,14 @@ Timeouts Waiting Primitives ================== -.. coroutinefunction:: wait(aws, \*, loop=None, timeout=None,\ +.. coroutinefunction:: wait(aws, *, loop=None, timeout=None,\ return_when=ALL_COMPLETED) Run :ref:`awaitable objects ` in the *aws* - set concurrently and block until the condition specified + iterable concurrently and block until the condition specified by *return_when*. - The *aws* set must not be empty. + The *aws* iterable must not be empty. Returns two sets of Tasks/Futures: ``(done, pending)``. @@ -582,12 +597,12 @@ Waiting Primitives deprecated. -.. function:: as_completed(aws, \*, loop=None, timeout=None) +.. function:: as_completed(aws, *, loop=None, timeout=None) Run :ref:`awaitable objects ` in the *aws* - set concurrently. Return an iterator of :class:`Future` objects. - Each Future object returned represents the earliest result - from the set of the remaining awaitables. + iterable concurrently. Return an iterator of coroutines. + Each coroutine returned can be awaited to get the earliest next + result from the iterable of the remaining awaitables. Raises :exc:`asyncio.TimeoutError` if the timeout occurs before all Futures are done. @@ -597,11 +612,70 @@ Waiting Primitives Example:: - for f in as_completed(aws): - earliest_result = await f + for coro in as_completed(aws): + earliest_result = await coro # ... +Running in Threads +================== + +.. coroutinefunction:: to_thread(func, /, *args, **kwargs) + + Asynchronously run function *func* in a separate thread. + + Any \*args and \*\*kwargs supplied for this function are directly passed + to *func*. Also, the current :class:`contextvars.Context` is propagated, + allowing context variables from the event loop thread to be accessed in the + separate thread. + + Return a coroutine that can be awaited to get the eventual result of *func*. + + This coroutine function is primarily intended to be used for executing + IO-bound functions/methods that would otherwise block the event loop if + they were run in the main thread. For example:: + + def blocking_io(): + print(f"start blocking_io at {time.strftime('%X')}") + # Note that time.sleep() can be replaced with any blocking + # IO-bound operation, such as file operations. + time.sleep(1) + print(f"blocking_io complete at {time.strftime('%X')}") + + async def main(): + print(f"started main at {time.strftime('%X')}") + + await asyncio.gather( + asyncio.to_thread(blocking_io), + asyncio.sleep(1)) + + print(f"finished main at {time.strftime('%X')}") + + + asyncio.run(main()) + + # Expected output: + # + # started main at 19:50:53 + # start blocking_io at 19:50:53 + # blocking_io complete at 19:50:54 + # finished main at 19:50:54 + + Directly calling `blocking_io()` in any coroutine would block the event loop + for its duration, resulting in an additional 1 second of run time. Instead, + by using `asyncio.to_thread()`, we can run it in a separate thread without + blocking the event loop. + + .. note:: + + Due to the :term:`GIL`, `asyncio.to_thread()` can typically only be used + to make IO-bound functions non-blocking. However, for extension modules + that release the GIL or alternative Python implementations that don't + have one, `asyncio.to_thread()` can also be used for CPU-bound functions. + + .. versionadded:: 3.9 + + Scheduling From Other Threads ============================= @@ -676,7 +750,7 @@ Introspection Task Object =========== -.. class:: Task(coro, \*, loop=None, name=None) +.. class:: Task(coro, *, loop=None, name=None) A :class:`Future-like ` object that runs a Python :ref:`coroutine `. Not thread-safe. @@ -842,7 +916,7 @@ Task Object See the documentation of :meth:`Future.remove_done_callback` for more details. - .. method:: get_stack(\*, limit=None) + .. method:: get_stack(*, limit=None) Return the list of stack frames for this Task. @@ -863,7 +937,7 @@ Task Object stack are returned, but the oldest frames of a traceback are returned. (This matches the behavior of the traceback module.) - .. method:: print_stack(\*, limit=None, file=None) + .. method:: print_stack(*, limit=None, file=None) Print the stack or traceback for this Task. @@ -903,31 +977,6 @@ Task Object .. versionadded:: 3.8 - .. classmethod:: all_tasks(loop=None) - - Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - If *loop* is ``None``, the :func:`get_event_loop` function - is used to get the current loop. - - .. deprecated-removed:: 3.7 3.9 - - Do not call this as a task method. Use the :func:`asyncio.all_tasks` - function instead. - - .. classmethod:: current_task(loop=None) - - Return the currently running task or ``None``. - - If *loop* is ``None``, the :func:`get_event_loop` function - is used to get the current loop. - - .. deprecated-removed:: 3.7 3.9 - - Do not call this as a task method. Use the - :func:`asyncio.current_task` function instead. - .. _asyncio_generator_based_coro: diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst index c2c058e474cbdc..f7f038107d11fe 100644 --- a/Doc/library/atexit.rst +++ b/Doc/library/atexit.rst @@ -39,7 +39,7 @@ internal error is detected, or when :func:`os._exit` is called. If an exception is raised during execution of the exit handlers, a traceback is printed (unless :exc:`SystemExit` is raised) and the exception information is - saved. After all exit handlers have had a chance to run the last exception to + saved. After all exit handlers have had a chance to run, the last exception to be raised is re-raised. This function returns *func*, which makes it possible to use it as a @@ -48,11 +48,12 @@ internal error is detected, or when :func:`os._exit` is called. .. function:: unregister(func) - Remove *func* from the list of functions to be run at interpreter - shutdown. After calling :func:`unregister`, *func* is guaranteed not to be - called when the interpreter shuts down, even if it was registered more than - once. :func:`unregister` silently does nothing if *func* was not previously - registered. + Remove *func* from the list of functions to be run at interpreter shutdown. + :func:`unregister` silently does nothing if *func* was not previously + registered. If *func* has been registered more than once, every occurrence + of that function in the :mod:`atexit` call stack will be removed. Equality + comparisons (``==``) are used internally during unregistration, so function + references do not need to have matching identities. .. seealso:: @@ -73,7 +74,7 @@ automatically when the program terminates without relying on the application making an explicit call into this module at termination. :: try: - with open("counterfile") as infile: + with open('counterfile') as infile: _count = int(infile.read()) except FileNotFoundError: _count = 0 @@ -83,21 +84,22 @@ making an explicit call into this module at termination. :: _count = _count + n def savecounter(): - with open("counterfile", "w") as outfile: - outfile.write("%d" % _count) + with open('counterfile', 'w') as outfile: + outfile.write('%d' % _count) import atexit + atexit.register(savecounter) Positional and keyword arguments may also be passed to :func:`register` to be passed along to the registered function when it is called:: def goodbye(name, adjective): - print('Goodbye, %s, it was %s to meet you.' % (name, adjective)) + print('Goodbye %s, it was %s to meet you.' % (name, adjective)) import atexit - atexit.register(goodbye, 'Donny', 'nice') + atexit.register(goodbye, 'Donny', 'nice') # or: atexit.register(goodbye, adjective='nice', name='Donny') @@ -107,6 +109,6 @@ Usage as a :term:`decorator`:: @atexit.register def goodbye(): - print("You are now leaving the Python sector.") + print('You are now leaving the Python sector.') This only works with functions that can be called without arguments. diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst index 3c68a1515b3eff..8227a7955bef81 100644 --- a/Doc/library/audit_events.rst +++ b/Doc/library/audit_events.rst @@ -7,7 +7,7 @@ Audit events table This table contains all events raised by :func:`sys.audit` or :c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later. +standard library. These calls were added in 3.8.0 or later (see :pep:`578`). See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for information on handling these events. @@ -19,3 +19,29 @@ information on handling these events. specific documentation for actual events raised. .. audit-event-table:: + +The following events are raised internally and do not correspond to any +public API of CPython: + ++--------------------------+-------------------------------------------+ +| Audit event | Arguments | ++==========================+===========================================+ +| _winapi.CreateFile | ``file_name``, ``desired_access``, | +| | ``share_mode``, ``creation_disposition``, | +| | ``flags_and_attributes`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreateJunction | ``src_path``, ``dst_path`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreateNamedPipe | ``name``, ``open_mode``, ``pipe_mode`` | ++--------------------------+-------------------------------------------+ +| _winapi.CreatePipe | | ++--------------------------+-------------------------------------------+ +| _winapi.CreateProcess | ``application_name``, ``command_line``, | +| | ``current_directory`` | ++--------------------------+-------------------------------------------+ +| _winapi.OpenProcess | ``process_id``, ``desired_access`` | ++--------------------------+-------------------------------------------+ +| _winapi.TerminateProcess | ``handle``, ``exit_code`` | ++--------------------------+-------------------------------------------+ +| ctypes.PyObj_FromPtr | ``obj`` | ++--------------------------+-------------------------------------------+ diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst index 85cdc16a7d78d4..637baf49da1fc0 100644 --- a/Doc/library/bz2.rst +++ b/Doc/library/bz2.rst @@ -266,7 +266,6 @@ Below are some examples of typical usage of the :mod:`bz2` module. Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -275,11 +274,9 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> c = bz2.compress(data) >>> len(data) / len(c) # Data compression ratio 1.513595166163142 - >>> d = bz2.decompress(c) >>> data == d # Check equality to original object after round-trip True @@ -287,7 +284,6 @@ Using :func:`compress` and :func:`decompress` to demonstrate round-trip compress Using :class:`BZ2Compressor` for incremental compression: >>> import bz2 - >>> def gen_data(chunks=10, chunksize=1000): ... """Yield incremental blocks of chunksize bytes.""" ... for _ in range(chunks): @@ -310,7 +306,6 @@ while ordered, repetitive data usually yields a high compression ratio. Writing and reading a bzip2-compressed file in binary mode: >>> import bz2 - >>> data = b"""\ ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, @@ -319,14 +314,11 @@ Writing and reading a bzip2-compressed file in binary mode: ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> with bz2.open("myfile.bz2", "wb") as f: ... # Write compressed data to file ... unused = f.write(data) - >>> with bz2.open("myfile.bz2", "rb") as f: ... # Decompress data from file ... content = f.read() - >>> content == data # Check equality to original object after round-trip True diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst index 56b75ef0f850a6..c3c04db853ed2d 100644 --- a/Doc/library/calendar.rst +++ b/Doc/library/calendar.rst @@ -349,7 +349,7 @@ For simple text calendars this module provides the following functions. .. function:: monthcalendar(year, month) Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month a represented by zeros. Each week begins with Monday + days outside of the month are represented by zeros. Each week begins with Monday unless set by :func:`setfirstweekday`. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index 4048592e7361f7..0c985c07040c57 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -277,14 +277,14 @@ These are useful if you want more control, or if you want to employ some of the algorithms implemented in this module in other circumstances. -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False) +.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values* and *strict_parsing* parameters are + ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are passed to :func:`urllib.parse.parse_qs` unchanged. -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace") +.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") Parse input of type :mimetype:`multipart/form-data` (for file uploads). Arguments are *fp* for the input file, *pdict* for a dictionary containing @@ -303,6 +303,9 @@ algorithms implemented in this module in other circumstances. Added the *encoding* and *errors* parameters. For non-file fields, the value is now a list of strings, not bytes. + .. versionchanged:: 3.9.2 + Added the *separator* parameter. + .. function:: parse_header(string) @@ -313,7 +316,7 @@ algorithms implemented in this module in other circumstances. .. function:: test() Robust test CGI script, usable as main program. Writes minimal HTTP headers and - formats all information provided to the script in HTML form. + formats all information provided to the script in HTML format. .. function:: print_environ() @@ -343,8 +346,8 @@ Caring about security .. index:: pair: CGI; security -There's one important rule: if you invoke an external program (via the -:func:`os.system` or :func:`os.popen` functions. or others with similar +There's one important rule: if you invoke an external program (via +:func:`os.system`, :func:`os.popen` or other functions with similar functionality), make very sure you don't pass arbitrary strings received from the client to the shell. This is a well-known security hole whereby clever hackers anywhere on the Web can exploit a gullible CGI script to invoke @@ -421,7 +424,7 @@ above on installing your CGI script carefully can save you a lot of time. If you wonder whether you have understood the installation procedure correctly, try installing a copy of this module file (:file:`cgi.py`) as a CGI script. When invoked as a script, the file will dump its environment and the contents of the -form in HTML form. Give it the right mode etc, and send it a request. If it's +form in HTML format. Give it the right mode etc., and send it a request. If it's installed in the standard :file:`cgi-bin` directory, it should be possible to send it a request by entering a URL into your browser of the form: diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst index f071057293eece..3338545e9267dc 100644 --- a/Doc/library/codecs.rst +++ b/Doc/library/codecs.rst @@ -159,9 +159,13 @@ function: .. function:: register(search_function) Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters, and return a - :class:`CodecInfo` object. In case a search function cannot find - a given encoding, it should return ``None``. + argument, being the encoding name in all lower case letters with hyphens + and spaces converted to underscores, and return a :class:`CodecInfo` object. + In case a search function cannot find a given encoding, it should return + ``None``. + + .. versionchanged:: 3.9 + Hyphens and spaces are converted to underscore. .. note:: @@ -694,7 +698,7 @@ compatible with the Python codec registry. .. method:: reset() - Flushes and resets the codec buffers used for keeping state. + Resets the codec buffers used for keeping internal state. Calling this method should ensure that the data on the output is put into a clean state that allows appending of new fresh data without having to @@ -789,7 +793,7 @@ compatible with the Python codec registry. .. method:: reset() - Resets the codec buffers used for keeping state. + Resets the codec buffers used for keeping internal state. Note that no stream repositioning should take place. This method is primarily intended to be able to recover from decoding errors. @@ -1414,6 +1418,9 @@ Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding and :mod:`stringprep`. +If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the +third-party `idna module _`. + These RFCs together define a protocol to support non-ASCII characters in domain names. A domain name containing non-ASCII characters (such as ``www.Alliancefrançaise.nu``) is converted into an ASCII-compatible encoding diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst index 2a3fb142f7297e..2345e78a17e4f5 100644 --- a/Doc/library/collections.abc.rst +++ b/Doc/library/collections.abc.rst @@ -98,12 +98,20 @@ ABC Inherits from Abstract Methods Mixin .. class:: Container - Hashable - Sized - Callable - ABCs for classes that provide respectively the methods :meth:`__contains__`, - :meth:`__hash__`, :meth:`__len__`, and :meth:`__call__`. + ABC for classes that provide the :meth:`__contains__` method. + +.. class:: Hashable + + ABC for classes that provide the :meth:`__hash__` method. + +.. class:: Sized + + ABC for classes that provide the :meth:`__len__` method. + +.. class:: Callable + + ABC for classes that provide the :meth:`__call__` method. .. class:: Iterable @@ -185,7 +193,7 @@ ABC Inherits from Abstract Methods Mixin expressions. Custom implementations must provide the :meth:`__await__` method. - :term:`Coroutine` objects and instances of the + :term:`Coroutine ` objects and instances of the :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. .. note:: @@ -283,7 +291,7 @@ Notes on using :class:`Set` and :class:`MutableSet` as a mixin: :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. If the :class:`Set` mixin is being used in a class with a different constructor signature, you will need to override :meth:`_from_iterable` - with a classmethod that can construct new instances from + with a classmethod or regular method that can construct new instances from an iterable argument. (2) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 549ac1bccadf5d..5189a2e88a6ce1 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -128,7 +128,7 @@ The class can be used to simulate nested scopes and is useful in templating. writing to any mapping in the chain. * Django's `Context class - `_ + `_ for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the @@ -685,9 +685,9 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` objects ---------------------------- -.. class:: defaultdict([default_factory[, ...]]) +.. class:: defaultdict(default_factory=None, /, [...]) - Returns a new dictionary-like object. :class:`defaultdict` is a subclass of the + Return a new dictionary-like object. :class:`defaultdict` is a subclass of the built-in :class:`dict` class. It overrides one method and adds one writable instance variable. The remaining functionality is the same as for the :class:`dict` class and is not documented here. @@ -849,6 +849,9 @@ they add the ability to access fields by name instead of position index. Named tuple instances do not have per-instance dictionaries, so they are lightweight and require no more memory than regular tuples. + To support pickling, the named tuple class should be assigned to a variable + that matches *typename*. + .. versionchanged:: 3.1 Added support for *rename*. diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst index 9b914b1f0d9c6d..de34664acb84ab 100644 --- a/Doc/library/compileall.rst +++ b/Doc/library/compileall.rst @@ -148,7 +148,7 @@ runtime. Public functions ---------------- -.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) +.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Recursively descend the directory tree named by *dir*, compiling all :file:`.py` files along the way. Return a true value if all the files compiled successfully, @@ -166,9 +166,10 @@ Public functions If *force* is true, modules are re-compiled even if the timestamps are up to date. - If *rx* is given, its search method is called on the complete path to each + If *rx* is given, its ``search`` method is called on the complete path to each file considered for compilation, and if it returns a true value, the file - is skipped. + is skipped. This can be used to exclude files matching a regular expression, + given as a :ref:`re.Pattern ` object. If *quiet* is ``False`` or ``0`` (the default), the filenames and other information are printed to standard out. Set to ``1``, only errors are @@ -231,7 +232,7 @@ Public functions Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. Default value of *maxlevels* was changed from ``10`` to ``sys.getrecursionlimit()`` -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, \*, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) +.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) Compile the file with path *fullname*. Return a true value if the file compiled successfully, and a false value otherwise. @@ -242,9 +243,10 @@ Public functions cases where the source file does not exist at the time the byte-code file is executed. - If *rx* is given, its search method is passed the full path name to the + If *rx* is given, its ``search`` method is passed the full path name to the file being compiled, and if it returns a true value, the file is not - compiled and ``True`` is returned. + compiled and ``True`` is returned. This can be used to exclude files matching + a regular expression, given as a :ref:`re.Pattern ` object. If *quiet* is ``False`` or ``0`` (the default), the filenames and other information are printed to standard out. Set to ``1``, only errors are diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst index b150990b83b75b..5be1a1106b09a0 100644 --- a/Doc/library/concurrency.rst +++ b/Doc/library/concurrency.rst @@ -21,6 +21,7 @@ multitasking). Here's an overview: subprocess.rst sched.rst queue.rst + contextvars.rst The following are support modules for some of the above services: diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index b21d5594c84fa0..d57f8ce23d12c4 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -67,7 +67,7 @@ Executor Objects .. versionchanged:: 3.5 Added the *chunksize* argument. - .. method:: shutdown(wait=True, \*, cancel_futures=False) + .. method:: shutdown(wait=True, *, cancel_futures=False) Signal the executor that it should free any resources that it is using when the currently pending futures are done executing. Calls to @@ -221,7 +221,8 @@ ProcessPoolExecutor The :class:`ProcessPoolExecutor` class is an :class:`Executor` subclass that uses a pool of processes to execute calls asynchronously. :class:`ProcessPoolExecutor` uses the :mod:`multiprocessing` module, which -allows it to side-step the :term:`Global Interpreter Lock` but also means that +allows it to side-step the :term:`Global Interpreter Lock +` but also means that only picklable objects can be executed and returned. The ``__main__`` module must be importable by worker subprocesses. This means @@ -235,9 +236,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` + If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. - On Windows, *max_workers* must be equal or lower than ``61``. If it is not + On Windows, *max_workers* must be less than or equal to ``61``. If it is not then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then the default chosen will be at most ``61``, even if more processors are available. @@ -249,7 +250,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. each worker process; *initargs* is a tuple of arguments passed to the initializer. Should *initializer* raise an exception, all currently pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, - as well any attempt to submit more jobs to the pool. + as well as any attempt to submit more jobs to the pool. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst index 739477f55fddda..730d1df9614289 100644 --- a/Doc/library/configparser.rst +++ b/Doc/library/configparser.rst @@ -674,97 +674,98 @@ be overridden by subclasses or by attribute assignment. .. attribute:: ConfigParser.BOOLEAN_STATES - By default when using :meth:`~ConfigParser.getboolean`, config parsers - consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, - ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, - ``'off'``. You can override this by specifying a custom dictionary of strings - and their Boolean outcomes. For example: - - .. doctest:: - - >>> custom = configparser.ConfigParser() - >>> custom['section1'] = {'funky': 'nope'} - >>> custom['section1'].getboolean('funky') - Traceback (most recent call last): - ... - ValueError: Not a boolean: nope - >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} - >>> custom['section1'].getboolean('funky') - False - - Other typical Boolean pairs include ``accept``/``reject`` or - ``enabled``/``disabled``. + By default when using :meth:`~ConfigParser.getboolean`, config parsers + consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, + ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, + ``'off'``. You can override this by specifying a custom dictionary of strings + and their Boolean outcomes. For example: + + .. doctest:: + + >>> custom = configparser.ConfigParser() + >>> custom['section1'] = {'funky': 'nope'} + >>> custom['section1'].getboolean('funky') + Traceback (most recent call last): + ... + ValueError: Not a boolean: nope + >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} + >>> custom['section1'].getboolean('funky') + False + + Other typical Boolean pairs include ``accept``/``reject`` or + ``enabled``/``disabled``. .. method:: ConfigParser.optionxform(option) + :noindex: - This method transforms option names on every read, get, or set - operation. The default converts the name to lowercase. This also - means that when a configuration file gets written, all keys will be - lowercase. Override this method if that's unsuitable. - For example: + This method transforms option names on every read, get, or set + operation. The default converts the name to lowercase. This also + means that when a configuration file gets written, all keys will be + lowercase. Override this method if that's unsuitable. + For example: - .. doctest:: + .. doctest:: + + >>> config = """ + ... [Section1] + ... Key = Value + ... + ... [Section2] + ... AnotherKey = Value + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> list(typical['Section1'].keys()) + ['key'] + >>> list(typical['Section2'].keys()) + ['anotherkey'] + >>> custom = configparser.RawConfigParser() + >>> custom.optionxform = lambda option: option + >>> custom.read_string(config) + >>> list(custom['Section1'].keys()) + ['Key'] + >>> list(custom['Section2'].keys()) + ['AnotherKey'] - >>> config = """ - ... [Section1] - ... Key = Value - ... - ... [Section2] - ... AnotherKey = Value - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> list(typical['Section1'].keys()) - ['key'] - >>> list(typical['Section2'].keys()) - ['anotherkey'] - >>> custom = configparser.RawConfigParser() - >>> custom.optionxform = lambda option: option - >>> custom.read_string(config) - >>> list(custom['Section1'].keys()) - ['Key'] - >>> list(custom['Section2'].keys()) - ['AnotherKey'] - - .. note:: - The optionxform function transforms option names to a canonical form. - This should be an idempotent function: if the name is already in - canonical form, it should be returned unchanged. + .. note:: + The optionxform function transforms option names to a canonical form. + This should be an idempotent function: if the name is already in + canonical form, it should be returned unchanged. .. attribute:: ConfigParser.SECTCRE - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: + A compiled regular expression used to parse section headers. The default + matches ``[section]`` to the name ``"section"``. Whitespace is considered + part of the section name, thus ``[ larch ]`` will be read as a section of + name ``" larch "``. Override this attribute if that's unsuitable. For + example: + + .. doctest:: + + >>> import re + >>> config = """ + ... [Section 1] + ... option = value + ... + ... [ Section 2 ] + ... another = val + ... """ + >>> typical = configparser.ConfigParser() + >>> typical.read_string(config) + >>> typical.sections() + ['Section 1', ' Section 2 '] + >>> custom = configparser.ConfigParser() + >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") + >>> custom.read_string(config) + >>> custom.sections() + ['Section 1', 'Section 2'] - .. doctest:: - - >>> import re - >>> config = """ - ... [Section 1] - ... option = value - ... - ... [ Section 2 ] - ... another = val - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> typical.sections() - ['Section 1', ' Section 2 '] - >>> custom = configparser.ConfigParser() - >>> custom.SECTCRE = re.compile(r"\[ *(?P
[^]]+?) *\]") - >>> custom.read_string(config) - >>> custom.sections() - ['Section 1', 'Section 2'] - - .. note:: + .. note:: - While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing - option lines, it's not recommended to override it because that would - interfere with constructor options *allow_no_value* and *delimiters*. + While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing + option lines, it's not recommended to override it because that would + interfere with constructor options *allow_no_value* and *delimiters*. Legacy API Examples @@ -1128,6 +1129,13 @@ ConfigParser Objects *space_around_delimiters* is true, delimiters between keys and values are surrounded by spaces. + .. note:: + + Comments in the original configuration file are not preserved when + writing the configuration back. + What is considered a comment, depends on the given values for + *comment_prefix* and *inline_comment_prefix*. + .. method:: remove_option(section, option) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index 0aa4ad76523480..f87ee210550cc6 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -191,8 +191,9 @@ Functions and classes provided: .. function:: suppress(*exceptions) Return a context manager that suppresses any of the specified exceptions - if they occur in the body of a with statement and then resumes execution - with the first statement following the end of the with statement. + if they occur in the body of a :keyword:`!with` statement and then + resumes execution with the first statement following the end of the + :keyword:`!with` statement. As with any other mechanism that completely suppresses exceptions, this context manager should be used only to cover very specific errors where @@ -236,10 +237,11 @@ Functions and classes provided: For example, the output of :func:`help` normally is sent to *sys.stdout*. You can capture that output in a string by redirecting the output to an - :class:`io.StringIO` object:: + :class:`io.StringIO` object. The replacement stream is returned from the + ``__enter__`` method and so is available as the target of the + :keyword:`with` statement:: - f = io.StringIO() - with redirect_stdout(f): + with redirect_stdout(io.StringIO()) as f: help(pow) s = f.getvalue() @@ -463,7 +465,7 @@ Functions and classes provided: The :meth:`close` method is not implemented, :meth:`aclose` must be used instead. - .. method:: enter_async_context(cm) + .. coroutinemethod:: enter_async_context(cm) Similar to :meth:`enter_context` but expects an asynchronous context manager. @@ -477,7 +479,7 @@ Functions and classes provided: Similar to :meth:`callback` but expects a coroutine function. - .. method:: aclose() + .. coroutinemethod:: aclose() Similar to :meth:`close` but properly handles awaitables. @@ -638,7 +640,7 @@ even further by means of a small helper class:: class Callback(ExitStack): def __init__(self, callback, /, *args, **kwds): - super(Callback, self).__init__() + super().__init__() self.callback(callback, *args, **kwds) def cancel(self): diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst index 8805661c456edb..14ac47f4c9eb16 100644 --- a/Doc/library/contextvars.rst +++ b/Doc/library/contextvars.rst @@ -26,7 +26,7 @@ See also :pep:`567` for additional details. Context Variables ----------------- -.. class:: ContextVar(name, [\*, default]) +.. class:: ContextVar(name, [*, default]) This class is used to declare a new Context Variable, e.g.:: @@ -146,7 +146,7 @@ Manual Context Management Context implements the :class:`collections.abc.Mapping` interface. - .. method:: run(callable, \*args, \*\*kwargs) + .. method:: run(callable, *args, **kwargs) Execute ``callable(*args, **kwargs)`` code in the context object the *run* method is called on. Return the result of the execution diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst index a8e8bfb1e832bb..0eb5a793ad953a 100644 --- a/Doc/library/copy.rst +++ b/Doc/library/copy.rst @@ -27,7 +27,7 @@ Interface summary: Return a deep copy of *x*. -.. exception:: error +.. exception:: Error Raised for module specific errors. diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst index 61d39828e0194a..7a72c26d5badeb 100644 --- a/Doc/library/csv.rst +++ b/Doc/library/csv.rst @@ -167,6 +167,9 @@ The :mod:`csv` module defines the following classes: All other optional or keyword arguments are passed to the underlying :class:`reader` instance. + .. versionchanged:: 3.6 + Returned rows are now of type :class:`OrderedDict`. + .. versionchanged:: 3.8 Returned rows are now of type :class:`dict`. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst index 2d6c6d0a1c3c57..fd6422cc8c06c5 100644 --- a/Doc/library/ctypes.rst +++ b/Doc/library/ctypes.rst @@ -919,9 +919,9 @@ Let's try it. We create two instances of ``cell``, and let them point to each other, and finally follow the pointer chain a few times:: >>> c1 = cell() - >>> c1.name = "foo" + >>> c1.name = b"foo" >>> c2 = cell() - >>> c2.name = "bar" + >>> c2.name = b"bar" >>> c1.next = pointer(c2) >>> c2.next = pointer(c1) >>> p = c1 @@ -1326,6 +1326,21 @@ way is to instantiate one of the following classes: libraries use the standard C calling convention, and are assumed to return :c:type:`int`. + On Windows creating a :class:`CDLL` instance may fail even if the DLL name + exists. When a dependent DLL of the loaded DLL is not found, a + :exc:`OSError` error is raised with the message *"[WinError 126] The + specified module could not be found".* This error message does not contain + the name of the missing DLL because the Windows API does not return this + information making this error hard to diagnose. To resolve this error and + determine which DLL is not found, you need to find the list of dependent + DLLs and determine which one is not found using Windows debugging and + tracing tools. + +.. seealso:: + + `Microsoft DUMPBIN tool `_ + -- A tool to find DLL dependents. + .. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=0) @@ -1618,7 +1633,7 @@ They are instances of a private class: ``ctypes.seh_exception`` with argument ``code`` will be raised, allowing an audit hook to replace the exception with its own. -.. audit-event:: ctypes.call_function func_pointer,arguments ctype-foreign-functions +.. audit-event:: ctypes.call_function func_pointer,arguments foreign-functions Some ways to invoke foreign function calls may raise an auditing event ``ctypes.call_function`` with arguments ``function pointer`` and ``arguments``. @@ -2493,7 +2508,7 @@ other data types containing pointer type fields. Arrays and pointers ^^^^^^^^^^^^^^^^^^^ -.. class:: Array(\*args) +.. class:: Array(*args) Abstract base class for arrays. @@ -2545,4 +2560,3 @@ Arrays and pointers Returns the object to which to pointer points. Assigning to this attribute changes the pointer to point to the assigned object. - diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 1201e8972de63c..c72840a07c192e 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -112,14 +112,15 @@ The module :mod:`curses` defines the following functions: .. function:: color_content(color_number) Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and :const:`COLORS`. Return a 3-tuple, + *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, containing the R,G,B values for the given color, which will be between ``0`` (no component) and ``1000`` (maximum amount of component). -.. function:: color_pair(color_number) +.. function:: color_pair(pair_number) - Return the attribute value for displaying text in the specified color. This + Return the attribute value for displaying text in the specified color pair. + Only the first 256 color pairs are supported. This attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, and the other :const:`A_\*` attributes. :func:`pair_number` is the counterpart to this function. @@ -214,7 +215,7 @@ The module :mod:`curses` defines the following functions: .. function:: getmouse() After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be call to retrieve the queued mouse event, represented as a + method should be called to retrieve the queued mouse event, represented as a 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is currently unused.) *bstate* is an integer value whose bits will be set to @@ -278,7 +279,7 @@ The module :mod:`curses` defines the following functions: Change the definition of a color, taking the number of the color to be changed followed by three RGB values (for the amounts of red, green, and blue components). The value of *color_number* must be between ``0`` and - :const:`COLORS`. Each of *r*, *g*, *b*, must be a value between ``0`` and + `COLORS - 1`. Each of *r*, *g*, *b*, must be a value between ``0`` and ``1000``. When :func:`init_color` is used, all occurrences of that color on the screen immediately change to the new definition. This function is a no-op on most terminals; it is active only if :func:`can_change_color` returns ``True``. @@ -291,7 +292,8 @@ The module :mod:`curses` defines the following functions: color number. The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot be changed). The value of *fg* and *bg* arguments must be between ``0`` and - :const:`COLORS`. If the color-pair was previously initialized, the screen is + ``COLORS - 1``, or, after calling :func:`use_default_colors`, ``-1``. + If the color-pair was previously initialized, the screen is refreshed and all occurrences of that color-pair are changed to the new definition. @@ -441,7 +443,7 @@ The module :mod:`curses` defines the following functions: .. function:: pair_content(pair_number) Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. - The value of *pair_number* must be between ``1`` and ``COLOR_PAIRS - 1``. + The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. .. function:: pair_number(attr) @@ -708,7 +710,7 @@ the following methods and attributes: window.addch(y, x, ch[, attr]) Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painter at that location. By default, the character + character previously painted at that location. By default, the character position and attributes are the current settings for the window object. .. note:: diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst index fe63d20671dd74..b226cda46c72be 100644 --- a/Doc/library/dataclasses.rst +++ b/Doc/library/dataclasses.rst @@ -17,13 +17,13 @@ adding generated :term:`special method`\s such as :meth:`__init__` and in :pep:`557`. The member variables to use in these generated methods are defined -using :pep:`526` type annotations. For example this code:: +using :pep:`526` type annotations. For example, this code:: from dataclasses import dataclass @dataclass class InventoryItem: - '''Class for keeping track of an item in inventory.''' + """Class for keeping track of an item in inventory.""" name: str unit_price: float quantity_on_hand: int = 0 @@ -31,9 +31,9 @@ using :pep:`526` type annotations. For example this code:: def total_cost(self) -> float: return self.unit_price * self.quantity_on_hand -Will add, among other things, a :meth:`__init__` that looks like:: +will add, among other things, a :meth:`__init__` that looks like:: - def __init__(self, name: str, unit_price: float, quantity_on_hand: int=0): + def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): self.name = name self.unit_price = unit_price self.quantity_on_hand = quantity_on_hand @@ -52,7 +52,7 @@ Module-level decorators, classes, and functions :term:`special method`\s to classes, as described below. The :func:`dataclass` decorator examines the class to find - ``field``\s. A ``field`` is defined as class variable that has a + ``field``\s. A ``field`` is defined as a class variable that has a :term:`type annotation `. With two exceptions described below, nothing in :func:`dataclass` examines the type specified in the variable annotation. @@ -62,8 +62,8 @@ Module-level decorators, classes, and functions The :func:`dataclass` decorator will add various "dunder" methods to the class, described below. If any of the added methods already - exist on the class, the behavior depends on the parameter, as documented - below. The decorator returns the same class that is called on; no new + exist in the class, the behavior depends on the parameter, as documented + below. The decorator returns the same class that it is called on; no new class is created. If :func:`dataclass` is used just as a simple decorator with no parameters, @@ -136,7 +136,7 @@ Module-level decorators, classes, and functions attribute ``__hash__ = None`` has a specific meaning to Python, as described in the :meth:`__hash__` documentation. - If :meth:`__hash__` is not explicit defined, or if it is set to ``None``, + If :meth:`__hash__` is not explicitly defined, or if it is set to ``None``, then :func:`dataclass` *may* add an implicit :meth:`__hash__` method. Although not recommended, you can force :func:`dataclass` to create a :meth:`__hash__` method with ``unsafe_hash=True``. This might be the case @@ -175,7 +175,7 @@ Module-level decorators, classes, and functions def __init__(self, a: int, b: int = 0): :exc:`TypeError` will be raised if a field without a default value - follows a field with a default value. This is true either when this + follows a field with a default value. This is true whether this occurs in a single class, or as a result of class inheritance. .. function:: field(*, default=MISSING, default_factory=MISSING, repr=True, hash=None, init=True, compare=True, metadata=None) @@ -188,7 +188,7 @@ Module-level decorators, classes, and functions @dataclass class C: - mylist: List[int] = field(default_factory=list) + mylist: list[int] = field(default_factory=list) c = C() c.mylist += [1, 2, 3] @@ -301,7 +301,7 @@ Module-level decorators, classes, and functions @dataclass class C: - mylist: List[Point] + mylist: list[Point] p = Point(10, 20) assert asdict(p) == {'x': 10, 'y': 20} @@ -361,7 +361,7 @@ Module-level decorators, classes, and functions .. function:: replace(instance, /, **changes) - Creates a new object of the same type of ``instance``, replacing + Creates a new object of the same type as ``instance``, replacing fields with values from ``changes``. If ``instance`` is not a Data Class, raises :exc:`TypeError`. If values in ``changes`` do not specify fields, raises :exc:`TypeError`. @@ -422,6 +422,27 @@ depend on one or more other fields. For example:: def __post_init__(self): self.c = self.a + self.b +The :meth:`__init__` method generated by :func:`dataclass` does not call base +class :meth:`__init__` methods. If the base class has an :meth:`__init__` method +that has to be called, it is common to call this method in a +:meth:`__post_init__` method:: + + @dataclass + class Rectangle: + height: float + width: float + + @dataclass + class Square(Rectangle): + side: float + + def __post_init__(self): + super().__init__(self.side, self.side) + +Note, however, that in general the dataclass-generated :meth:`__init__` methods +don't need to be called, since the derived dataclass will take care of +initializing all fields of any base class that is a dataclass itself. + See the section below on init-only variables for ways to pass parameters to :meth:`__post_init__`. Also see the warning about how :func:`replace` handles ``init=False`` fields. @@ -592,4 +613,4 @@ Exceptions Raised when an implicitly defined :meth:`__setattr__` or :meth:`__delattr__` is called on a dataclass which was defined with - ``frozen=True``. + ``frozen=True``. It is a subclass of :exc:`AttributeError`. diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst index 675bbb6fafdca4..ff51b2779e5fa3 100644 --- a/Doc/library/datatypes.rst +++ b/Doc/library/datatypes.rst @@ -33,3 +33,4 @@ The following modules are documented in this chapter: pprint.rst reprlib.rst enum.rst + graphlib.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index 800361c54ba717..dae0dd7aa55898 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -35,7 +35,8 @@ on efficient attribute extraction for output formatting and manipulation. Aware and Naive Objects ----------------------- -Date and time objects may be categorized as "aware" or "naive". +Date and time objects may be categorized as "aware" or "naive" depending on +whether or not they include timezone information. With sufficient knowledge of applicable algorithmic and political time adjustments, such as time zone and daylight saving time information, @@ -1218,7 +1219,7 @@ Instance methods: .. method:: datetime.replace(year=self.year, month=self.month, day=self.day, \ hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ - tzinfo=self.tzinfo, * fold=0) + tzinfo=self.tzinfo, *, fold=0) Return a datetime with the same attributes, except for those attributes given new values by whichever keyword arguments are specified. Note that @@ -1782,7 +1783,7 @@ Other constructor: Instance methods: .. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ - microsecond=self.microsecond, tzinfo=self.tzinfo, * fold=0) + microsecond=self.microsecond, tzinfo=self.tzinfo, *, fold=0) Return a :class:`.time` with the same value, except for those attributes given new values by whichever keyword arguments are specified. Note that diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst index 69a20fca17898a..e194649e30d85c 100644 --- a/Doc/library/decimal.rst +++ b/Doc/library/decimal.rst @@ -1357,6 +1357,9 @@ In addition to the three supplied contexts, new contexts can be created with the The rounding mode of the context is used. Results are always correctly-rounded in the Python version. + ``Decimal(0) ** Decimal(0)`` results in ``InvalidOperation``, and if ``InvalidOperation`` + is not trapped, then results in ``Decimal('NaN')``. + .. versionchanged:: 3.3 The C module computes :meth:`power` in terms of the correctly-rounded :meth:`exp` and :meth:`ln` functions. The result is well-defined but @@ -2193,4 +2196,3 @@ are expected to be exact. .. [#] .. versionchanged:: 3.9 This approach now works for all exact results except for non-integer powers. - Also backported to 3.7 and 3.8. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst index dc82a974ce095d..53f98c1018988f 100644 --- a/Doc/library/dialog.rst +++ b/Doc/library/dialog.rst @@ -198,7 +198,7 @@ These do not emulate the native look-and-feel of the platform. A subclass of FileDialog that creates a dialog window for selecting a destination file. - .. method:: ok_command() + .. method:: ok_command() Test whether or not the selection points to a valid file that is not a directory. Confirmation is required if an already existing file is diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst index 7a898c21b52e03..aa08988c8b36f7 100644 --- a/Doc/library/difflib.rst +++ b/Doc/library/difflib.rst @@ -18,12 +18,13 @@ -------------- This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce difference -information in various formats, including HTML and context and unified +can be used for example, for comparing files, and can produce information +about file differences in various formats, including HTML and context and unified diffs. For comparing directories and files, see also, the :mod:`filecmp` module. .. class:: SequenceMatcher + :noindex: This is a flexible class for comparing pairs of sequences of any type, so long as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a @@ -651,6 +652,7 @@ The :class:`Differ` class has this constructor: .. class:: Differ(linejunk=None, charjunk=None) + :noindex: Optional keyword parameters *linejunk* and *charjunk* are for filter functions (or ``None``): diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index f871ec4f13def8..08730c4d993517 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -346,7 +346,7 @@ The Python compiler currently generates the following bytecode instructions. .. opcode:: ROT_FOUR - Lifts second, third and forth stack items one position up, moves top down + Lifts second, third and fourth stack items one position up, moves top down to position four. .. versionadded:: 3.8 @@ -640,7 +640,7 @@ the original TOS1. .. opcode:: LIST_APPEND (i) - Calls ``list.append(TOS[-i], TOS)``. Used to implement list comprehensions. + Calls ``list.append(TOS1[-i], TOS)``. Used to implement list comprehensions. .. opcode:: MAP_ADD (i) @@ -861,7 +861,7 @@ All of the following opcodes use their arguments. .. opcode:: LIST_TO_TUPLE - Pops a list from the stack and pushes a tuple containing the same values. + Pops a list from the stack and pushes a tuple containing the same values. .. versionadded:: 3.9 @@ -889,7 +889,7 @@ All of the following opcodes use their arguments. .. opcode:: DICT_MERGE - Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. + Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. .. versionadded:: 3.9 @@ -907,14 +907,14 @@ All of the following opcodes use their arguments. .. opcode:: IS_OP (invert) - Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. + Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. .. versionadded:: 3.9 .. opcode:: CONTAINS_OP (invert) - Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. + Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. .. versionadded:: 3.9 @@ -955,8 +955,8 @@ All of the following opcodes use their arguments. .. opcode:: JUMP_IF_NOT_EXC_MATCH (target) - Tests whether the second value on the stack is an exception matching TOS, - and jumps if it is not. Pops two values from the stack. + Tests whether the second value on the stack is an exception matching TOS, + and jumps if it is not. Pops two values from the stack. .. versionadded:: 3.9 diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst index 745b3a6a3ad42a..c68e773b1688aa 100644 --- a/Doc/library/email.compat32-message.rst +++ b/Doc/library/email.compat32-message.rst @@ -23,7 +23,7 @@ policy :attr:`~email.policy.Compat32`. If you are going to use another policy, you should be using the :class:`~email.message.EmailMessage` class instead. An email message consists of *headers* and a *payload*. Headers must be -:rfc:`5233` style names and values, where the field name and value are +:rfc:`5322` style names and values, where the field name and value are separated by a colon. The colon is not part of either the field name or the field value. The payload may be a simple text message, or a binary object, or a structured sequence of sub-messages each with their own set of headers and diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst index e09c7c0e402bbc..918fc55677e723 100644 --- a/Doc/library/email.contentmanager.rst +++ b/Doc/library/email.contentmanager.rst @@ -116,7 +116,7 @@ Currently the email package provides only one concrete content manager, decoding the payload to unicode. The default error handler is ``replace``. - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8' \ + .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8', \ cte=None, \ disposition=None, filename=None, cid=None, \ params=None, headers=None) diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst index 9376da2b8d39ce..3e1d97a03264b2 100644 --- a/Doc/library/email.headerregistry.rst +++ b/Doc/library/email.headerregistry.rst @@ -289,7 +289,7 @@ variant, :attr:`~.BaseHeader.max_count` is set to 1. A :class:`ParameterizedMIMEHeader` class that handles the :mailheader:`Content-Disposition` header. - .. attribute:: content-disposition + .. attribute:: content_disposition ``inline`` and ``attachment`` are the only valid values in common use. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst index 8e7076259810f5..bf53b9520fc723 100644 --- a/Doc/library/email.policy.rst +++ b/Doc/library/email.policy.rst @@ -210,7 +210,7 @@ added matters. To illustrate:: :meth:`register_defect` method. - .. attribute:: mangle_from\_ + .. attribute:: mangle_from_ If :const:`True`, lines starting with *"From "* in the body are escaped by putting a ``>`` in front of them. This parameter is used when diff --git a/Doc/library/email.rst b/Doc/library/email.rst index fae99cf3e6abbe..5eebcd9e896d93 100644 --- a/Doc/library/email.rst +++ b/Doc/library/email.rst @@ -16,7 +16,7 @@ The :mod:`email` package is a library for managing email messages. It is specifically *not* designed to do any sending of email messages to SMTP (:rfc:`2821`), NNTP, or other servers; those are functions of modules such as :mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as -RFC-compliant as possible, supporting :rfc:`5233` and :rfc:`6532`, as well as +RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, and :rfc:`2231`. diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index 4b4f5eb1944cc5..bbe8bdc82b4091 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -19,6 +19,12 @@ An enumeration is a set of symbolic names (members) bound to unique, constant values. Within an enumeration, the members can be compared by identity, and the enumeration itself can be iterated over. +.. note:: Case of Enum Members + + Because Enums are used to represent constants we recommend using + UPPER_CASE names for enum members, and will be using that style + in our examples. + Module Contents --------------- @@ -50,6 +56,7 @@ helper, :class:`auto`. the bitwise operations without losing their :class:`Flag` membership. .. function:: unique + :noindex: Enum class decorator that ensures only one name is bound to any one value. @@ -269,7 +276,7 @@ overridden:: .. note:: - The goal of the default :meth:`_generate_next_value_` methods is to provide + The goal of the default :meth:`_generate_next_value_` method is to provide the next :class:`int` in sequence with the last :class:`int` provided, but the way it does this is an implementation detail and may change. @@ -880,6 +887,32 @@ Using an auto-numbering :meth:`__new__` would look like:: >>> Color.GREEN.value 2 +To make a more general purpose ``AutoNumber``, add ``*args`` to the signature:: + + >>> class AutoNumber(NoValue): + ... def __new__(cls, *args): # this is the only change from above + ... value = len(cls.__members__) + 1 + ... obj = object.__new__(cls) + ... obj._value_ = value + ... return obj + ... + +Then when you inherit from ``AutoNumber`` you can write your own ``__init__`` +to handle any extra arguments:: + + >>> class Swatch(AutoNumber): + ... def __init__(self, pantone='unknown'): + ... self.pantone = pantone + ... AUBURN = '3497' + ... SEA_GREEN = '1246' + ... BLEACHED_CORAL = () # New color, no Pantone code yet! + ... + >>> Swatch.SEA_GREEN + + >>> Swatch.SEA_GREEN.pantone + '1246' + >>> Swatch.BLEACHED_CORAL.pantone + 'unknown' .. note:: @@ -1088,6 +1121,15 @@ and raise an error if the two do not match:: In Python 2 code the :attr:`_order_` attribute is necessary as definition order is lost before it can be recorded. + +_Private__names +""""""""""""""" + +Private names will be normal attributes in Python 3.10 instead of either an error +or a member (depending on if the name ends with an underscore). Using these names +in 3.9 will issue a :exc:`DeprecationWarning`. + + ``Enum`` member type """""""""""""""""""" diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index df2cda9d67ad15..6bed5c70f0ad04 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -42,12 +42,12 @@ include the originating exception(s) and the final exception. When raising a new exception (rather than using a bare ``raise`` to re-raise the exception currently being handled), the implicit exception context can be -supplemented with an explicit cause by using :keyword:`from` with +supplemented with an explicit cause by using :keyword:`from` with :keyword:`raise`:: raise new_exc from original_exc -The expression following :keyword:`from` must be an exception or ``None``. It +The expression following :keyword:`from` must be an exception or ``None``. It will be set as :attr:`__cause__` on the raised exception. Setting :attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` attribute to ``True``, so that using ``raise new_exc from None`` @@ -390,16 +390,39 @@ The following exceptions are the exceptions that are usually raised. .. versionadded:: 3.5 -.. exception:: SyntaxError +.. exception:: SyntaxError(message, details) Raised when the parser encounters a syntax error. This may occur in an - :keyword:`import` statement, in a call to the built-in functions :func:`exec` + :keyword:`import` statement, in a call to the built-in functions + :func:`compile`, :func:`exec`, or :func:`eval`, or when reading the initial script or standard input (also interactively). - Instances of this class have attributes :attr:`filename`, :attr:`lineno`, - :attr:`offset` and :attr:`text` for easier access to the details. :func:`str` - of the exception instance returns only the message. + The :func:`str` of the exception instance returns only the error message. + Details is a tuple whose members are also available as separate attributes. + + .. attribute:: filename + + The name of the file the syntax error occurred in. + + .. attribute:: lineno + + Which line number in the file the error occurred in. This is + 1-indexed: the first line in the file has a ``lineno`` of 1. + + .. attribute:: offset + + The column in the line where the error occurred. This is + 1-indexed: the first character in the line has an ``offset`` of 1. + + .. attribute:: text + + The source code text involved in the error. + + For errors in f-string fields, the message is prefixed by "f-string: " + and the offsets are offsets in a text constructed from the replacement + expression. For example, compiling f'Bad {a b} field' results in this + args attribute: ('f-string: ...', ('', 1, 4, '(a b)\n')). .. exception:: IndentationError diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst index ce07d326b395d8..925f08e914685e 100644 --- a/Doc/library/fnmatch.rst +++ b/Doc/library/fnmatch.rst @@ -75,7 +75,7 @@ patterns. .. function:: filter(names, pattern) - Return the subset of the list of *names* that match *pattern*. It is the same as + Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst index f4d4cdf9ada9d9..3a9165ac920519 100644 --- a/Doc/library/ftplib.rst +++ b/Doc/library/ftplib.rst @@ -28,6 +28,7 @@ Here's a sample session using the :mod:`ftplib` module:: >>> ftp.login() # user anonymous, passwd anonymous@ '230 Login successful.' >>> ftp.cwd('debian') # change into "debian" directory + '250 Directory successfully changed.' >>> ftp.retrlines('LIST') # list directory contents -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README ... @@ -39,6 +40,7 @@ Here's a sample session using the :mod:`ftplib` module:: >>> ftp.retrbinary('RETR README', fp.write) '226 Transfer complete.' >>> ftp.quit() + '221 Goodbye.' The module defines the following items: diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index 90a2370c1793bb..784bb62e64a167 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -43,9 +43,8 @@ are always available. They are listed here in alphabetical order. .. function:: abs(x) Return the absolute value of a number. The argument may be an - integer or a floating point number. If the argument is a complex number, its - magnitude is returned. If *x* defines :meth:`__abs__`, - ``abs(x)`` returns ``x.__abs__()``. + integer, a floating point number, or an object implementing :meth:`__abs__`. + If the argument is a complex number, its magnitude is returned. .. function:: all(iterable) @@ -151,8 +150,8 @@ are always available. They are listed here in alphabetical order. * If it is an *integer*, the array will have that size and will be initialized with null bytes. - * If it is an object conforming to the *buffer* interface, a read-only buffer - of the object will be used to initialize the bytes array. + * If it is an object conforming to the :ref:`buffer interface `, + a read-only buffer of the object will be used to initialize the bytes array. * If it is an *iterable*, it must be an iterable of integers in the range ``0 <= x < 256``, which are used as the initial contents of the array. @@ -246,26 +245,24 @@ are always available. They are listed here in alphabetical order. interactive statement (in the latter case, expression statements that evaluate to something other than ``None`` will be printed). - The optional arguments *flags* and *dont_inherit* control which :ref:`future - statements ` affect the compilation of *source*. If neither - is present (or both are zero) the code is compiled with those future - statements that are in effect in the code that is calling :func:`compile`. If the - *flags* argument is given and *dont_inherit* is not (or is zero) then the - future statements specified by the *flags* argument are used in addition to - those that would be used anyway. If *dont_inherit* is a non-zero integer then - the *flags* argument is it -- the future statements in effect around the call - to compile are ignored. - - Future statements are specified by bits which can be bitwise ORed together to - specify multiple statements. The bitfield required to specify a given feature - can be found as the :attr:`~__future__._Feature.compiler_flag` attribute on - the :class:`~__future__._Feature` instance in the :mod:`__future__` module. - - The optional argument *flags* also controls whether the compiled source is - allowed to contain top-level ``await``, ``async for`` and ``async with``. - When the bit ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` is set, the return code - object has ``CO_COROUTINE`` set in ``co_code``, and can be interactively - executed via ``await eval(code_object)``. + The optional arguments *flags* and *dont_inherit* control which + :ref:`compiler options ` should be activated + and which :ref:`future features ` should be allowed. If neither + is present (or both are zero) the code is compiled with the same flags that + affect the code that is calling :func:`compile`. If the *flags* + argument is given and *dont_inherit* is not (or is zero) then the compiler + options and the future statements specified by the *flags* argument are used + in addition to those that would be used anyway. If *dont_inherit* is a + non-zero integer then the *flags* argument is it -- the flags (future + features and compiler options) in the surrounding code are ignored. + + Compiler options and future statements are specified by bits which can be + bitwise ORed together to specify multiple options. The bitfield required to + specify a given future feature can be found as the + :attr:`~__future__._Feature.compiler_flag` attribute on the + :class:`~__future__._Feature` instance in the :mod:`__future__` module. + :ref:`Compiler flags ` can be found in :mod:`ast` + module, with ``PyCF_`` prefix. The argument *optimize* specifies the optimization level of the compiler; the default value of ``-1`` selects the optimization level of the interpreter as @@ -511,7 +508,8 @@ are always available. They are listed here in alphabetical order. occurs). [#]_ If it is a code object, it is simply executed. In all cases, the code that's executed is expected to be valid as file input (see the section "File input" in the Reference Manual). Be aware that the - :keyword:`return` and :keyword:`yield` statements may not be used outside of + :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` + statements may not be used outside of function definitions even within the context of code passed to the :func:`exec` function. The return value is ``None``. @@ -583,7 +581,7 @@ are always available. They are listed here in alphabetical order. input must conform to the following grammar after leading and trailing whitespace characters are removed: - .. productionlist:: + .. productionlist:: float sign: "+" | "-" infinity: "Infinity" | "inf" nan: "nan" @@ -678,6 +676,13 @@ are always available. They are listed here in alphabetical order. ``x.foobar``. If the named attribute does not exist, *default* is returned if provided, otherwise :exc:`AttributeError` is raised. + .. note:: + + Since :ref:`private name mangling ` happens at + compilation time, one must manually mangle a private attribute's + (attributes with two leading underscores) name in order to retrieve it with + :func:`getattr`. + .. function:: globals() @@ -769,6 +774,8 @@ are always available. They are listed here in alphabetical order. .. impl-detail:: This is the address of the object in memory. + .. audit-event:: builtins.id id id + .. function:: input([prompt]) @@ -957,7 +964,7 @@ are always available. They are listed here in alphabetical order. .. _func-memoryview: -.. class:: memoryview(obj) +.. class:: memoryview(object) :noindex: Return a "memory view" object created from the given argument. See @@ -1042,7 +1049,8 @@ are always available. They are listed here in alphabetical order. .. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. + cannot be opened, an :exc:`OSError` is raised. See + :ref:`tut-files` for more examples of how to use this function. *file* is a :term:`path-like object` giving the pathname (absolute or relative to the current working directory) of the file to be opened or an @@ -1494,6 +1502,13 @@ are always available. They are listed here in alphabetical order. object allows it. For example, ``setattr(x, 'foobar', 123)`` is equivalent to ``x.foobar = 123``. + .. note:: + + Since :ref:`private name mangling ` happens at + compilation time, one must manually mangle a private attribute's + (attributes with two leading underscores) name in order to set it with + :func:`setattr`. + .. class:: slice(stop) slice(start, stop[, step]) @@ -1630,7 +1645,7 @@ are always available. They are listed here in alphabetical order. not found in statically compiled languages or languages that only support single inheritance. This makes it possible to implement "diamond diagrams" where multiple base classes implement the same method. Good design dictates - that this method have the same calling signature in every case (because the + that such implementations have the same calling signature in every case (because the order of calls is determined at runtime, because that order adapts to changes in the class hierarchy, and because that order can include sibling classes that are unknown prior to runtime). @@ -1674,7 +1689,7 @@ are always available. They are listed here in alphabetical order. .. class:: type(object) - type(name, bases, dict) + type(name, bases, dict, **kwds) .. index:: object: type @@ -1687,21 +1702,29 @@ are always available. They are listed here in alphabetical order. With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is the - class name and becomes the :attr:`~definition.__name__` attribute; the *bases* - tuple itemizes the base classes and becomes the :attr:`~class.__bases__` - attribute; and the *dict* dictionary is the namespace containing definitions - for class body and is copied to a standard dictionary to become the - :attr:`~object.__dict__` attribute. For example, the following two - statements create identical :class:`type` objects: + dynamic form of the :keyword:`class` statement. The *name* string is + the class name and becomes the :attr:`~definition.__name__` attribute. + The *bases* tuple contains the base classes and becomes the + :attr:`~class.__bases__` attribute; if empty, :class:`object`, the + ultimate base of all classes, is added. The *dict* dictionary contains + attribute and method definitions for the class body; it may be copied + or wrapped before becoming the :attr:`~object.__dict__` attribute. + The following two statements create identical :class:`type` objects: >>> class X: ... a = 1 ... - >>> X = type('X', (object,), dict(a=1)) + >>> X = type('X', (), dict(a=1)) See also :ref:`bltin-type-objects`. + Keyword arguments provided to the three argument form are passed to the + appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) + in the same way that keywords in a class + definition (besides *metaclass*) would. + + See also :ref:`class-customization`. + .. versionchanged:: 3.6 Subclasses of :class:`type` which don't override ``type.__new__`` may no longer use the one-argument form to get the type of an object. @@ -1720,6 +1743,9 @@ are always available. They are listed here in alphabetical order. locals dictionary is only useful for reads since updates to the locals dictionary are ignored. + A :exc:`TypeError` exception is raised if an object is specified but + it doesn't have a :attr:`~object.__dict__` attribute (for example, if + its class defines the :attr:`~object.__slots__` attribute). .. function:: zip(*iterables) diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst index a44eb85b27dbab..85d4e74698d2f1 100644 --- a/Doc/library/functools.rst +++ b/Doc/library/functools.rst @@ -62,27 +62,52 @@ The :mod:`functools` module defines the following functions: Example:: class DataSet: + def __init__(self, sequence_of_numbers): - self._data = sequence_of_numbers + self._data = tuple(sequence_of_numbers) @cached_property def stdev(self): return statistics.stdev(self._data) - @cached_property - def variance(self): - return statistics.variance(self._data) + The mechanics of :func:`cached_property` are somewhat different from + :func:`property`. A regular property blocks attribute writes unless a + setter is defined. In contrast, a *cached_property* allows writes. - .. versionadded:: 3.8 + The *cached_property* decorator only runs on lookups and only when an + attribute of the same name doesn't exist. When it does run, the + *cached_property* writes to the attribute with the same name. Subsequent + attribute reads and writes take precedence over the *cached_property* + method and it works like a normal attribute. - .. note:: + The cached value can be cleared by deleting the attribute. This + allows the *cached_property* method to run again. - This decorator requires that the ``__dict__`` attribute on each instance - be a mutable mapping. This means it will not work with some types, such as - metaclasses (since the ``__dict__`` attributes on type instances are - read-only proxies for the class namespace), and those that specify - ``__slots__`` without including ``__dict__`` as one of the defined slots - (as such classes don't provide a ``__dict__`` attribute at all). + Note, this decorator interferes with the operation of :pep:`412` + key-sharing dictionaries. This means that instance dictionaries + can take more space than usual. + + Also, this decorator requires that the ``__dict__`` attribute on each instance + be a mutable mapping. This means it will not work with some types, such as + metaclasses (since the ``__dict__`` attributes on type instances are + read-only proxies for the class namespace), and those that specify + ``__slots__`` without including ``__dict__`` as one of the defined slots + (as such classes don't provide a ``__dict__`` attribute at all). + + If a mutable mapping is not available or if space-efficient key sharing + is desired, an effect similar to :func:`cached_property` can be achieved + by a stacking :func:`property` on top of :func:`cache`:: + + class DataSet: + def __init__(self, sequence_of_numbers): + self._data = sequence_of_numbers + + @property + @cache + def stdev(self): + return statistics.stdev(self._data) + + .. versionadded:: 3.8 .. function:: cmp_to_key(func) @@ -543,184 +568,6 @@ The :mod:`functools` module defines the following functions: .. versionadded:: 3.8 -.. class:: TopologicalSorter(graph=None) - - Provides functionality to topologically sort a graph of hashable nodes. - - A topological order is a linear ordering of the vertices in a graph such that - for every directed edge u -> v from vertex u to vertex v, vertex u comes - before vertex v in the ordering. For instance, the vertices of the graph may - represent tasks to be performed, and the edges may represent constraints that - one task must be performed before another; in this example, a topological - ordering is just a valid sequence for the tasks. A complete topological - ordering is possible if and only if the graph has no directed cycles, that - is, if it is a directed acyclic graph. - - If the optional *graph* argument is provided it must be a dictionary - representing a directed acyclic graph where the keys are nodes and the values - are iterables of all predecessors of that node in the graph (the nodes that - have edges that point to the value in the key). Additional nodes can be added - to the graph using the :meth:`~TopologicalSorter.add` method. - - In the general case, the steps required to perform the sorting of a given - graph are as follows: - - * Create an instance of the :class:`TopologicalSorter` with an optional - initial graph. - * Add additional nodes to the graph. - * Call :meth:`~TopologicalSorter.prepare` on the graph. - * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over - the nodes returned by :meth:`~TopologicalSorter.get_ready` and - process them. Call :meth:`~TopologicalSorter.done` on each node as it - finishes processing. - - In case just an immediate sorting of the nodes in the graph is required and - no parallelism is involved, the convenience method - :meth:`TopologicalSorter.static_order` can be used directly: - - .. doctest:: - - >>> graph = {"D": {"B", "C"}, "C": {"A"}, "B": {"A"}} - >>> ts = TopologicalSorter(graph) - >>> tuple(ts.static_order()) - ('A', 'C', 'B', 'D') - - The class is designed to easily support parallel processing of the nodes as - they become ready. For instance:: - - topological_sorter = TopologicalSorter() - - # Add nodes to 'topological_sorter'... - - topological_sorter.prepare() - while topological_sorter.is_active(): - for node in topological_sorter.get_ready(): - # Worker threads or processes take nodes to work on off the - # 'task_queue' queue. - task_queue.put(node) - - # When the work for a node is done, workers put the node in - # 'finalized_tasks_queue' so we can get more nodes to work on. - # The definition of 'is_active()' guarantees that, at this point, at - # least one node has been placed on 'task_queue' that hasn't yet - # been passed to 'done()', so this blocking 'get()' must (eventually) - # succeed. After calling 'done()', we loop back to call 'get_ready()' - # again, so put newly freed nodes on 'task_queue' as soon as - # logically possible. - node = finalized_tasks_queue.get() - topological_sorter.done(node) - - .. method:: add(node, *predecessors) - - Add a new node and its predecessors to the graph. Both the *node* and all - elements in *predecessors* must be hashable. - - If called multiple times with the same node argument, the set of - dependencies will be the union of all dependencies passed in. - - It is possible to add a node with no dependencies (*predecessors* is not - provided) or to provide a dependency twice. If a node that has not been - provided before is included among *predecessors* it will be automatically - added to the graph with no predecessors of its own. - - Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. - - .. method:: prepare() - - Mark the graph as finished and check for cycles in the graph. If any cycle - is detected, :exc:`CycleError` will be raised, but - :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many - nodes as possible until cycles block more progress. After a call to this - function, the graph cannot be modified, and therefore no more nodes can be - added using :meth:`~TopologicalSorter.add`. - - .. method:: is_active() - - Returns ``True`` if more progress can be made and ``False`` otherwise. - Progress can be made if cycles do not block the resolution and either - there are still nodes ready that haven't yet been returned by - :meth:`TopologicalSorter.get_ready` or the number of nodes marked - :meth:`TopologicalSorter.done` is less than the number that have been - returned by :meth:`TopologicalSorter.get_ready`. - - The :meth:`~TopologicalSorter.__bool__` method of this class defers to - this function, so instead of:: - - if ts.is_active(): - ... - - if possible to simply do:: - - if ts: - ... - - Raises :exc:`ValueError` if called without calling - :meth:`~TopologicalSorter.prepare` previously. - - .. method:: done(*nodes) - - Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as - processed, unblocking any successor of each node in *nodes* for being - returned in the future by a call to :meth:`TopologicalSorter.get_ready`. - - Raises :exc:`ValueError` if any node in *nodes* has already been marked as - processed by a previous call to this method or if a node was not added to - the graph by using :meth:`TopologicalSorter.add`, if called without - calling :meth:`~TopologicalSorter.prepare` or if node has not yet been - returned by :meth:`~TopologicalSorter.get_ready`. - - .. method:: get_ready() - - Returns a ``tuple`` with all the nodes that are ready. Initially it - returns all nodes with no predecessors, and once those are marked as - processed by calling :meth:`TopologicalSorter.done`, further calls will - return all new nodes that have all their predecessors already processed. - Once no more progress can be made, empty tuples are returned. - - Raises :exc:`ValueError` if called without calling - :meth:`~TopologicalSorter.prepare` previously. - - .. method:: static_order() - - Returns an iterable of nodes in a topological order. Using this method - does not require to call :meth:`TopologicalSorter.prepare` or - :meth:`TopologicalSorter.done`. This method is equivalent to:: - - def static_order(self): - self.prepare() - while self.is_active(): - node_group = self.get_ready() - yield from node_group - self.done(*node_group) - - The particular order that is returned may depend on the specific order in - which the items were inserted in the graph. For example: - - .. doctest:: - - >>> ts = TopologicalSorter() - >>> ts.add(3, 2, 1) - >>> ts.add(1, 0) - >>> print([*ts.static_order()]) - [2, 0, 1, 3] - - >>> ts2 = TopologicalSorter() - >>> ts2.add(1, 0) - >>> ts2.add(3, 2, 1) - >>> print([*ts2.static_order()]) - [0, 2, 1, 3] - - This is due to the fact that "0" and "2" are in the same level in the - graph (they would have been returned in the same call to - :meth:`~TopologicalSorter.get_ready`) and the order between them is - determined by the order of insertion. - - - If any cycle is detected, :exc:`CycleError` will be raised. - - .. versionadded:: 3.9 - - .. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) Update a *wrapper* function to look like the *wrapped* function. The optional @@ -830,19 +677,3 @@ differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` are not created automatically. Also, :class:`partial` objects defined in classes behave like static methods and do not transform into bound methods during instance attribute look-up. - - -Exceptions ----------- -The :mod:`functools` module defines the following exception classes: - -.. exception:: CycleError - - Subclass of :exc:`ValueError` raised by :meth:`TopologicalSorter.prepare` if cycles exist - in the working graph. If multiple cycles exist, only one undefined choice among them will - be reported and included in the exception. - - The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` - attribute of the exception instance and consists in a list of nodes, such that each node is, - in the graph, an immediate predecessor of the next node in the list. In the reported list, - the first and the last node will be the same, to make it clear that it is cyclic. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 0c33c865304591..69a1a8313b7593 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -72,6 +72,8 @@ The :mod:`gc` module provides the following functions: .. versionchanged:: 3.8 New *generation* parameter. + .. audit-event:: gc.get_objects generation gc.get_objects + .. function:: get_stats() Return a list of three per-generation dictionaries containing collection @@ -106,9 +108,9 @@ The :mod:`gc` module provides the following functions: allocations minus the number of deallocations exceeds *threshold0*, collection starts. Initially only generation ``0`` is examined. If generation ``0`` has been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. Similarly, *threshold2* - controls the number of collections of generation ``1`` before collecting - generation ``2``. + examined, then generation ``1`` is examined as well. + With the third generation, things are a bit more complicated, + see `Collecting the oldest generation `_ for more information. .. function:: get_count() @@ -135,10 +137,13 @@ The :mod:`gc` module provides the following functions: resulting referrers. To get only currently live objects, call :func:`collect` before calling :func:`get_referrers`. - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. + .. warning:: + Care must be taken when using objects returned by :func:`get_referrers` because + some of them could still be under construction and hence in a temporarily + invalid state. Avoid using :func:`get_referrers` for any purpose other than + debugging. + + .. audit-event:: gc.get_referrers objs gc.get_referrers .. function:: get_referents(*objs) @@ -151,6 +156,7 @@ The :mod:`gc` module provides the following functions: be involved in a cycle. So, for example, if an integer is directly reachable from an argument, that integer object may or may not appear in the result list. + .. audit-event:: gc.get_referents objs gc.get_referents .. function:: is_tracked(obj) diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index 92a8c4d1eb871c..3c468ebf73769f 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -43,7 +43,9 @@ For example, ``'[?]'`` matches the character ``'?'``. (like :file:`/usr/src/Python-1.5/Makefile`) or relative (like :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken symlinks are included in the results (as in the shell). Whether or not the - results are sorted depends on the file system. + results are sorted depends on the file system. If a file that satisfies + conditions is removed or added during the call of this function, whether + a path name for that file be included is unspecified. .. index:: single: **; in glob-style wildcards diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst new file mode 100644 index 00000000000000..2bc80da4ead2a2 --- /dev/null +++ b/Doc/library/graphlib.rst @@ -0,0 +1,210 @@ +:mod:`graphlib` --- Functionality to operate with graph-like structures +========================================================================= + +.. module:: graphlib + :synopsis: Functionality to operate with graph-like structures + + +**Source code:** :source:`Lib/graphlib.py` + +.. testsetup:: default + + import graphlib + from graphlib import * + +-------------- + + +.. class:: TopologicalSorter(graph=None) + + Provides functionality to topologically sort a graph of hashable nodes. + + A topological order is a linear ordering of the vertices in a graph such that + for every directed edge u -> v from vertex u to vertex v, vertex u comes + before vertex v in the ordering. For instance, the vertices of the graph may + represent tasks to be performed, and the edges may represent constraints that + one task must be performed before another; in this example, a topological + ordering is just a valid sequence for the tasks. A complete topological + ordering is possible if and only if the graph has no directed cycles, that + is, if it is a directed acyclic graph. + + If the optional *graph* argument is provided it must be a dictionary + representing a directed acyclic graph where the keys are nodes and the values + are iterables of all predecessors of that node in the graph (the nodes that + have edges that point to the value in the key). Additional nodes can be added + to the graph using the :meth:`~TopologicalSorter.add` method. + + In the general case, the steps required to perform the sorting of a given + graph are as follows: + + * Create an instance of the :class:`TopologicalSorter` with an optional + initial graph. + * Add additional nodes to the graph. + * Call :meth:`~TopologicalSorter.prepare` on the graph. + * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over + the nodes returned by :meth:`~TopologicalSorter.get_ready` and + process them. Call :meth:`~TopologicalSorter.done` on each node as it + finishes processing. + + In case just an immediate sorting of the nodes in the graph is required and + no parallelism is involved, the convenience method + :meth:`TopologicalSorter.static_order` can be used directly: + + .. doctest:: + + >>> graph = {"D": {"B", "C"}, "C": {"A"}, "B": {"A"}} + >>> ts = TopologicalSorter(graph) + >>> tuple(ts.static_order()) + ('A', 'C', 'B', 'D') + + The class is designed to easily support parallel processing of the nodes as + they become ready. For instance:: + + topological_sorter = TopologicalSorter() + + # Add nodes to 'topological_sorter'... + + topological_sorter.prepare() + while topological_sorter.is_active(): + for node in topological_sorter.get_ready(): + # Worker threads or processes take nodes to work on off the + # 'task_queue' queue. + task_queue.put(node) + + # When the work for a node is done, workers put the node in + # 'finalized_tasks_queue' so we can get more nodes to work on. + # The definition of 'is_active()' guarantees that, at this point, at + # least one node has been placed on 'task_queue' that hasn't yet + # been passed to 'done()', so this blocking 'get()' must (eventually) + # succeed. After calling 'done()', we loop back to call 'get_ready()' + # again, so put newly freed nodes on 'task_queue' as soon as + # logically possible. + node = finalized_tasks_queue.get() + topological_sorter.done(node) + + .. method:: add(node, *predecessors) + + Add a new node and its predecessors to the graph. Both the *node* and all + elements in *predecessors* must be hashable. + + If called multiple times with the same node argument, the set of + dependencies will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not + provided) or to provide a dependency twice. If a node that has not been + provided before is included among *predecessors* it will be automatically + added to the graph with no predecessors of its own. + + Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. + + .. method:: prepare() + + Mark the graph as finished and check for cycles in the graph. If any cycle + is detected, :exc:`CycleError` will be raised, but + :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many + nodes as possible until cycles block more progress. After a call to this + function, the graph cannot be modified, and therefore no more nodes can be + added using :meth:`~TopologicalSorter.add`. + + .. method:: is_active() + + Returns ``True`` if more progress can be made and ``False`` otherwise. + Progress can be made if cycles do not block the resolution and either + there are still nodes ready that haven't yet been returned by + :meth:`TopologicalSorter.get_ready` or the number of nodes marked + :meth:`TopologicalSorter.done` is less than the number that have been + returned by :meth:`TopologicalSorter.get_ready`. + + The :meth:`~TopologicalSorter.__bool__` method of this class defers to + this function, so instead of:: + + if ts.is_active(): + ... + + it is possible to simply do:: + + if ts: + ... + + Raises :exc:`ValueError` if called without calling + :meth:`~TopologicalSorter.prepare` previously. + + .. method:: done(*nodes) + + Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as + processed, unblocking any successor of each node in *nodes* for being + returned in the future by a call to :meth:`TopologicalSorter.get_ready`. + + Raises :exc:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method or if a node was not added to + the graph by using :meth:`TopologicalSorter.add`, if called without + calling :meth:`~TopologicalSorter.prepare` or if node has not yet been + returned by :meth:`~TopologicalSorter.get_ready`. + + .. method:: get_ready() + + Returns a ``tuple`` with all the nodes that are ready. Initially it + returns all nodes with no predecessors, and once those are marked as + processed by calling :meth:`TopologicalSorter.done`, further calls will + return all new nodes that have all their predecessors already processed. + Once no more progress can be made, empty tuples are returned. + + Raises :exc:`ValueError` if called without calling + :meth:`~TopologicalSorter.prepare` previously. + + .. method:: static_order() + + Returns an iterator object which will iterate over nodes in a topological + order. When using this method, :meth:`~TopologicalSorter.prepare` and + :meth:`~TopologicalSorter.done` should not be called. This method is + equivalent to:: + + def static_order(self): + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) + + The particular order that is returned may depend on the specific order in + which the items were inserted in the graph. For example: + + .. doctest:: + + >>> ts = TopologicalSorter() + >>> ts.add(3, 2, 1) + >>> ts.add(1, 0) + >>> print([*ts.static_order()]) + [2, 0, 1, 3] + + >>> ts2 = TopologicalSorter() + >>> ts2.add(1, 0) + >>> ts2.add(3, 2, 1) + >>> print([*ts2.static_order()]) + [0, 2, 1, 3] + + This is due to the fact that "0" and "2" are in the same level in the + graph (they would have been returned in the same call to + :meth:`~TopologicalSorter.get_ready`) and the order between them is + determined by the order of insertion. + + + If any cycle is detected, :exc:`CycleError` will be raised. + + .. versionadded:: 3.9 + + +Exceptions +---------- +The :mod:`graphlib` module defines the following exception classes: + +.. exception:: CycleError + + Subclass of :exc:`ValueError` raised by :meth:`TopologicalSorter.prepare` if cycles exist + in the working graph. If multiple cycles exist, only one undefined choice among them will + be reported and included in the exception. + + The detected cycle can be accessed via the second element in the :attr:`~CycleError.args` + attribute of the exception instance and consists in a list of nodes, such that each node is, + in the graph, an immediate predecessor of the next node in the list. In the reported list, + the first and the last node will be the same, to make it clear that it is cyclic. diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst index d644974e660984..5a507c12d7f12f 100644 --- a/Doc/library/hashlib.rst +++ b/Doc/library/hashlib.rst @@ -118,10 +118,10 @@ More condensed: Using :func:`new` with an algorithm provided by OpenSSL: - >>> h = hashlib.new('ripemd160') + >>> h = hashlib.new('sha512_256') >>> h.update(b"Nobody inspects the spammish repetition") >>> h.hexdigest() - 'cc4a5ce1b3df48aec5d22d1f16b894a0b894eccc' + '19197dc4d03829df858011c6c87600f994a858103bbc19005f20987aa19a97e2' Hashlib provides the following constant attributes: diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst index 5ad348490eaf60..897edc2d19517c 100644 --- a/Doc/library/hmac.rst +++ b/Doc/library/hmac.rst @@ -138,6 +138,11 @@ This module also provides the following helper function: .. versionadded:: 3.3 + .. versionchanged:: 3.9 + + The function uses OpenSSL's ``CRYPTO_memcmp()`` internally when + available. + .. seealso:: diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index 478a5b31475cfd..029e9ec5401e08 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -98,7 +98,9 @@ provides three different variants: .. attribute:: path - Contains the request path. + Contains the request path. If query component of the URL is present, + then ``path`` includes the query. Using the terminology of :rfc:`3986`, + ``path`` here includes ``hier-part`` and the ``query``. .. attribute:: request_version @@ -318,9 +320,16 @@ provides three different variants: .. class:: SimpleHTTPRequestHandler(request, client_address, server, directory=None) - This class serves files from the current directory and below, directly + This class serves files from the directory *directory* and below, + or the current directory if *directory* is not provided, directly mapping the directory structure to HTTP requests. + .. versionadded:: 3.7 + The *directory* parameter. + + .. versionchanged:: 3.9 + The *directory* parameter accepts a :term:`path-like object`. + A lot of the work, such as parsing the request, is done by the base class :class:`BaseHTTPRequestHandler`. This class implements the :func:`do_GET` and :func:`do_HEAD` functions. @@ -343,13 +352,6 @@ provides three different variants: This dictionary is no longer filled with the default system mappings, but only contains overrides. - .. attribute:: directory - - If not specified, the directory to serve is the current working directory. - - .. versionchanged:: 3.9 - Accepts a :term:`path-like object`. - The :class:`SimpleHTTPRequestHandler` class defines the following methods: .. method:: do_HEAD() diff --git a/Doc/library/idle.rst b/Doc/library/idle.rst index b1192e7bb46552..3c302115b5f408 100644 --- a/Doc/library/idle.rst +++ b/Doc/library/idle.rst @@ -147,7 +147,7 @@ Go to Line Clear any selection and update the line and column status. Show Completions - Open a scrollable list allowing selection of keywords and attributes. See + Open a scrollable list allowing selection of existing names. See :ref:`Completions ` in the Editing and navigation section below. Expand Word @@ -250,7 +250,7 @@ View Last Restart Scroll the shell window to the last Shell restart. Restart Shell - Restart the shell to clean the environment. + Restart the shell to clean the environment and reset display and exception handling. Previous History Cycle through earlier commands in history which match the current entry. @@ -469,82 +469,91 @@ are restricted to four spaces due to Tcl/Tk limitations. See also the indent/dedent region commands on the :ref:`Format menu `. - .. _completions: Completions ^^^^^^^^^^^ -Completions are supplied for functions, classes, and attributes of classes, -both built-in and user-defined. Completions are also provided for -filenames. - -The AutoCompleteWindow (ACW) will open after a predefined delay (default is -two seconds) after a '.' or (in a string) an os.sep is typed. If after one -of those characters (plus zero or more other characters) a tab is typed -the ACW will open immediately if a possible continuation is found. - -If there is only one possible completion for the characters entered, a -:kbd:`Tab` will supply that completion without opening the ACW. - -'Show Completions' will force open a completions window, by default the -:kbd:`C-space` will open a completions window. In an empty -string, this will contain the files in the current directory. On a -blank line, it will contain the built-in and user-defined functions and -classes in the current namespaces, plus any modules imported. If some -characters have been entered, the ACW will attempt to be more specific. - -If a string of characters is typed, the ACW selection will jump to the -entry most closely matching those characters. Entering a :kbd:`tab` will -cause the longest non-ambiguous match to be entered in the Editor window or -Shell. Two :kbd:`tab` in a row will supply the current ACW selection, as -will return or a double click. Cursor keys, Page Up/Down, mouse selection, -and the scroll wheel all operate on the ACW. - -"Hidden" attributes can be accessed by typing the beginning of hidden -name after a '.', e.g. '_'. This allows access to modules with -``__all__`` set, or to class-private attributes. - -Completions and the 'Expand Word' facility can save a lot of typing! - -Completions are currently limited to those in the namespaces. Names in -an Editor window which are not via ``__main__`` and :data:`sys.modules` will -not be found. Run the module once with your imports to correct this situation. -Note that IDLE itself places quite a few modules in sys.modules, so -much can be found by default, e.g. the re module. - -If you don't like the ACW popping up unbidden, simply make the delay -longer or disable the extension. +Completions are supplied, when requested and available, for module +names, attributes of classes or functions, or filenames. Each request +method displays a completion box with existing names. (See tab +completions below for an exception.) For any box, change the name +being completed and the item highlighted in the box by +typing and deleting characters; by hitting :kbd:`Up`, :kbd:`Down`, +:kbd:`PageUp`, :kbd:`PageDown`, :kbd:`Home`, and :kbd:`End` keys; +and by a single click within the box. Close the box with :kbd:`Escape`, +:kbd:`Enter`, and double :kbd:`Tab` keys or clicks outside the box. +A double click within the box selects and closes. + +One way to open a box is to type a key character and wait for a +predefined interval. This defaults to 2 seconds; customize it +in the settings dialog. (To prevent auto popups, set the delay to a +large number of milliseconds, such as 100000000.) For imported module +names or class or function attributes, type '.'. +For filenames in the root directory, type :data:`os.sep` or +:data:`os.altsep` immediately after an opening quote. (On Windows, +one can specify a drive first.) Move into subdirectories by typing a +directory name and a separator. + +Instead of waiting, or after a box is closed, open a completion box +immediately with Show Completions on the Edit menu. The default hot +key is :kbd:`C-space`. If one types a prefix for the desired name +before opening the box, the first match or near miss is made visible. +The result is the same as if one enters a prefix +after the box is displayed. Show Completions after a quote completes +filenames in the current directory instead of a root directory. + +Hitting :kbd:`Tab` after a prefix usually has the same effect as Show +Completions. (With no prefix, it indents.) However, if there is only +one match to the prefix, that match is immediately added to the editor +text without opening a box. + +Invoking 'Show Completions', or hitting :kbd:`Tab` after a prefix, +outside of a string and without a preceding '.' opens a box with +keywords, builtin names, and available module-level names. + +When editing code in an editor (as oppose to Shell), increase the +available module-level names by running your code +and not restarting the Shell thereafter. This is especially useful +after adding imports at the top of a file. This also increases +possible attribute completions. + +Completion boxes intially exclude names beginning with '_' or, for +modules, not included in '__all__'. The hidden names can be accessed +by typing '_' after '.', either before or after the box is opened. .. _calltips: Calltips ^^^^^^^^ -A calltip is shown when one types :kbd:`(` after the name of an *accessible* -function. A name expression may include dots and subscripts. A calltip -remains until it is clicked, the cursor is moved out of the argument area, -or :kbd:`)` is typed. When the cursor is in the argument part of a definition, -the menu or shortcut display a calltip. +A calltip is shown automatically when one types :kbd:`(` after the name +of an *accessible* function. A function name expression may include +dots and subscripts. A calltip remains until it is clicked, the cursor +is moved out of the argument area, or :kbd:`)` is typed. Whenever the +cursor is in the argument part of a definition, select Edit and "Show +Call Tip" on the menu or enter its shortcut to display a calltip. -A calltip consists of the function signature and the first line of the -docstring. For builtins without an accessible signature, the calltip -consists of all lines up the fifth line or the first blank line. These -details may change. +The calltip consists of the function's signature and docstring up to +the latter's first blank line or the fifth non-blank line. (Some builtin +functions lack an accessible signature.) A '/' or '*' in the signature +indicates that the preceding or following arguments are passed by +position or name (keyword) only. Details are subject to change. -The set of *accessible* functions depends on what modules have been imported -into the user process, including those imported by Idle itself, -and what definitions have been run, all since the last restart. +In Shell, the accessible functions depends on what modules have been +imported into the user process, including those imported by Idle itself, +and which definitions have been run, all since the last restart. For example, restart the Shell and enter ``itertools.count(``. A calltip -appears because Idle imports itertools into the user process for its own use. -(This could change.) Enter ``turtle.write(`` and nothing appears. Idle does -not import turtle. The menu or shortcut do nothing either. Enter -``import turtle`` and then ``turtle.write(`` will work. +appears because Idle imports itertools into the user process for its own +use. (This could change.) Enter ``turtle.write(`` and nothing appears. +Idle does not itself import turtle. The menu entry and shortcut also do +nothing. Enter ``import turtle``. Thereafter, ``turtle.write(`` +will display a calltip. -In an editor, import statements have no effect until one runs the file. One -might want to run a file after writing the import statements at the top, -or immediately run an existing file before editing. +In an editor, import statements have no effect until one runs the file. +One might want to run a file after writing import statements, after +adding function definitions, or after opening an existing file. .. _code-context: @@ -661,8 +670,16 @@ IDLE uses a socket to communicate between the IDLE GUI process and the user code execution process. A connection must be established whenever the Shell starts or restarts. (The latter is indicated by a divider line that says 'RESTART'). If the user process fails to connect to the GUI process, it -displays a ``Tk`` error box with a 'cannot connect' message that directs the -user here. It then exits. +usually displays a ``Tk`` error box with a 'cannot connect' message +that directs the user here. It then exits. + +One specific connection failure on Unix systems results from +misconfigured masquerading rules somewhere in a system's network setup. +When IDLE is started from a terminal, one will see a message starting +with ``** Invalid host:``. +The valid value is ``127.0.0.1 (idlelib.rpc.LOCALHOST)``. +One can diagnose with ``tcpconnect -irv 127.0.0.1 6543`` in one +terminal window and ``tcplisten `` in another. A common cause of failure is a user-written file with the same name as a standard library module, such as *random.py* and *tkinter.py*. When such a @@ -700,6 +717,13 @@ If IDLE quits with no message, and it was not started from a console, try starting it from a console or terminal (``python -m idlelib``) and see if this results in an error message. +On Unix-based systems with tcl/tk older than ``8.6.11`` (see +``About IDLE``) certain characters of certain fonts can cause +a tk failure with a message to the terminal. This can happen either +if one starts IDLE to edit a file with such a character or later +when entering such a character. If one cannot upgrade tcl/tk, +then re-configure IDLE to use a font that works better. + Running user code ^^^^^^^^^^^^^^^^^ @@ -708,7 +732,7 @@ intended to be the same as executing the same code by the default method, directly with Python in a text-mode system console or terminal window. However, the different interface and operation occasionally affect visible results. For instance, ``sys.modules`` starts with more entries, -and ``threading.activeCount()`` returns 2 instead of 1. +and ``threading.active_count()`` returns 2 instead of 1. By default, IDLE runs user code in a separate OS process rather than in the user interface process that runs the shell and editor. In the execution @@ -717,28 +741,38 @@ with objects that get input from and send output to the Shell window. The original values stored in ``sys.__stdin__``, ``sys.__stdout__``, and ``sys.__stderr__`` are not touched, but may be ``None``. -When Shell has the focus, it controls the keyboard and screen. This is -normally transparent, but functions that directly access the keyboard -and screen will not work. These include system-specific functions that -determine whether a key has been pressed and if so, which. +Sending print output from one process to a text widget in another is +slower than printing to a system terminal in the same process. +This has the most effect when printing multiple arguments, as the string +for each argument, each separator, the newline are sent separately. +For development, this is usually not a problem, but if one wants to +print faster in IDLE, format and join together everything one wants +displayed together and then print a single string. Both format strings +and :meth:`str.join` can help combine fields and lines. IDLE's standard stream replacements are not inherited by subprocesses -created in the execution process, whether directly by user code or by modules -such as multiprocessing. If such subprocess use ``input`` from sys.stdin -or ``print`` or ``write`` to sys.stdout or sys.stderr, +created in the execution process, whether directly by user code or by +modules such as multiprocessing. If such subprocess use ``input`` from +sys.stdin or ``print`` or ``write`` to sys.stdout or sys.stderr, IDLE should be started in a command line window. The secondary subprocess will then be attached to that window for input and output. -The IDLE code running in the execution process adds frames to the call stack -that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and -``sys.setrecursionlimit`` to reduce the effect of the additional stack frames. - If ``sys`` is reset by user code, such as with ``importlib.reload(sys)``, IDLE's changes are lost and input from the keyboard and output to the screen will not work correctly. -When user code raises SystemExit either directly or by calling sys.exit, IDLE -returns to a Shell prompt instead of exiting. +When Shell has the focus, it controls the keyboard and screen. This is +normally transparent, but functions that directly access the keyboard +and screen will not work. These include system-specific functions that +determine whether a key has been pressed and if so, which. + +The IDLE code running in the execution process adds frames to the call stack +that would not be there otherwise. IDLE wraps ``sys.getrecursionlimit`` and +``sys.setrecursionlimit`` to reduce the effect of the additional stack +frames. + +When user code raises SystemExit either directly or by calling sys.exit, +IDLE returns to a Shell prompt instead of exiting. User output in Shell ^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 7c5b0750161598..65681ec093598c 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -143,7 +143,7 @@ The following utility functions are defined: .. function:: Int2AP(num) - Converts an integer into a string representation using characters from the set + Converts an integer into a bytes representation using characters from the set [``A`` .. ``P``]. @@ -174,9 +174,9 @@ example of usage. .. seealso:: - Documents describing the protocol, and sources and binaries for servers - implementing it, can all be found at the University of Washington's *IMAP - Information Center* (https://www.washington.edu/imap/). + Documents describing the protocol, sources for servers + implementing it, by the University of Washington's IMAP Information Center + can all be found at (**Source Code**) https://github.com/uw-imap/imap (**Not Maintained**). .. _imap4-objects: @@ -197,7 +197,7 @@ you want to avoid having an argument string quoted (eg: the *flags* argument to Each command returns a tuple: ``(type, [data, ...])`` where *type* is usually ``'OK'`` or ``'NO'``, and *data* is either the text from the command response, -or mandated results from the command. Each *data* is either a string, or a +or mandated results from the command. Each *data* is either a ``bytes``, or a tuple. If a tuple, then the first part is the header of the response, and the second part contains the data (ie: 'literal' value). diff --git a/Doc/library/imp.rst b/Doc/library/imp.rst index f5ad8c7229644f..121a730e0c9b4a 100644 --- a/Doc/library/imp.rst +++ b/Doc/library/imp.rst @@ -8,7 +8,7 @@ **Source code:** :source:`Lib/imp.py` .. deprecated:: 3.4 - The :mod:`imp` package is pending deprecation in favor of :mod:`importlib`. + The :mod:`imp` module is deprecated in favor of :mod:`importlib`. .. index:: statement: import diff --git a/Doc/library/importlib.metadata.rst b/Doc/library/importlib.metadata.rst index 15e58b860d97d0..9bf34047ddaecb 100644 --- a/Doc/library/importlib.metadata.rst +++ b/Doc/library/importlib.metadata.rst @@ -4,6 +4,13 @@ Using :mod:`!importlib.metadata` ================================= +.. module:: importlib.metadata + :synopsis: The implementation of the importlib metadata. + +**Source code:** :source:`Lib/importlib/metadata.py` + +.. versionadded:: 3.8 + .. note:: This functionality is provisional and may deviate from the usual version semantics of the standard library. @@ -77,7 +84,9 @@ Entry points The ``entry_points()`` function returns a dictionary of all entry points, keyed by group. Entry points are represented by ``EntryPoint`` instances; each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and -a ``.load()`` method to resolve the value. +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: >>> eps = entry_points() # doctest: +SKIP >>> list(eps) # doctest: +SKIP @@ -86,6 +95,12 @@ a ``.load()`` method to resolve the value. >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] # doctest: +SKIP >>> wheel # doctest: +SKIP EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module # doctest: +SKIP + 'wheel.cli' + >>> wheel.attr # doctest: +SKIP + 'main' + >>> wheel.extras # doctest: +SKIP + [] >>> main = wheel.load() # doctest: +SKIP >>> main # doctest: +SKIP @@ -94,7 +109,7 @@ The ``group`` and ``name`` are arbitrary values defined by the package author and usually a client will wish to resolve all entry points for a particular group. Read `the setuptools docs `_ -for more information on entrypoints, their definition, and usage. +for more information on entry points, their definition, and usage. .. _metadata: @@ -198,9 +213,9 @@ Thus, an alternative way to get the version number is through the There are all kinds of additional metadata available on the ``Distribution`` instance:: - >>> d.metadata['Requires-Python'] # doctest: +SKIP + >>> dist.metadata['Requires-Python'] # doctest: +SKIP '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' - >>> d.metadata['License'] # doctest: +SKIP + >>> dist.metadata['License'] # doctest: +SKIP 'MIT' The full set of available metadata is not described here. See :pep:`566` @@ -235,7 +250,7 @@ method:: """ The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` -properties indicating the path to search and names to match and may +properties indicating the path to search and name to match and may supply other relevant context. What this means in practice is that to support finding distribution package diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 99bfeacbbc7407..736c43d96558c2 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -438,8 +438,9 @@ ABC hierarchy:: package. This attribute is not set on modules. - :attr:`__package__` - The parent package for the module/package. If the module is - top-level then it has a value of the empty string. The + The fully-qualified name of the package under which the module was + loaded as a submodule (or the empty string for top-level modules). + For packages, it is the same as :attr:`__name__`. The :func:`importlib.util.module_for_loader` decorator can handle the details for :attr:`__package__`. @@ -480,7 +481,7 @@ ABC hierarchy:: .. class:: ResourceReader - *Superseded by TraversableReader* + *Superseded by TraversableResources* An :term:`abstract base class` to provide the ability to read *resources*. @@ -805,7 +806,7 @@ ABC hierarchy:: .. versionadded:: 3.9 -.. class:: TraversableReader +.. class:: TraversableResources An abstract base class for resource readers capable of serving the ``files`` interface. Subclasses ResourceReader and provides @@ -813,9 +814,6 @@ ABC hierarchy:: methods. Therefore, any loader supplying TraversableReader also supplies ResourceReader. - Loaders that wish to support resource reading are expected to - implement this interface. - .. versionadded:: 3.9 @@ -890,6 +888,22 @@ The following functions are available. .. versionadded:: 3.9 +.. function:: as_file(traversable) + + Given a :class:`importlib.resources.abc.Traversable` object representing + a file, typically from :func:`importlib.resources.files`, return + a context manager for use in a :keyword:`with` statement. + The context manager provides a :class:`pathlib.Path` object. + + Exiting the context manager cleans up any temporary file created when the + resource was extracted from e.g. a zip file. + + Use ``as_file`` when the Traversable methods + (``read_text``, etc) are insufficient and an actual file on + the file system is required. + + .. versionadded:: 3.9 + .. function:: open_binary(package, resource) Open for binary reading the *resource* within *package*. @@ -1073,7 +1087,7 @@ find and load modules. .. class:: WindowsRegistryFinder - :term:`Finder` for modules declared in the Windows registry. This class + :term:`Finder ` for modules declared in the Windows registry. This class implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for @@ -1088,7 +1102,7 @@ find and load modules. .. class:: PathFinder - A :term:`Finder` for :data:`sys.path` and package ``__path__`` attributes. + A :term:`Finder ` for :data:`sys.path` and package ``__path__`` attributes. This class implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for @@ -1137,7 +1151,7 @@ find and load modules. directory for ``''`` (i.e. the empty string). -.. class:: FileFinder(path, \*loader_details) +.. class:: FileFinder(path, *loader_details) A concrete implementation of :class:`importlib.abc.PathEntryFinder` which caches results from the file system. @@ -1180,7 +1194,7 @@ find and load modules. Clear out the internal cache. - .. classmethod:: path_hook(\*loader_details) + .. classmethod:: path_hook(*loader_details) A class method which returns a closure for use on :attr:`sys.path_hooks`. An instance of :class:`FileFinder` is returned by the closure using the @@ -1347,8 +1361,8 @@ find and load modules. (``__loader__``) - The loader to use for loading. For namespace packages this should be - set to ``None``. + The :term:`Loader ` that should be used when loading + the module. :term:`Finders ` should always set this. .. attribute:: origin @@ -1381,8 +1395,9 @@ find and load modules. (``__package__``) - (Read-only) Fully-qualified name of the package to which the module - belongs as a submodule (or ``None``). + (Read-only) The fully-qualified name of the package under which the module + should be loaded as a submodule (or the empty string for top-level modules). + For packages, it is the same as :attr:`__name__`. .. attribute:: has_location @@ -1474,7 +1489,7 @@ an :term:`importer`. If **name** has no leading dots, then **name** is simply returned. This allows for usage such as - ``importlib.util.resolve_name('sys', __package__)`` without doing a + ``importlib.util.resolve_name('sys', __spec__.parent)`` without doing a check to see if the **package** argument is needed. :exc:`ImportError` is raised if **name** is a relative module name but diff --git a/Doc/library/index.rst b/Doc/library/index.rst index bebf7429b0e63e..2cddb417da82e3 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -56,7 +56,6 @@ the `Python Package Index `_. crypto.rst allos.rst concurrency.rst - contextvars.rst ipc.rst netdata.rst markup.rst diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index d00a30ff004063..d1e4a9b5f30f3d 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -556,7 +556,7 @@ The Signature object represents the call signature of a callable object and its return annotation. To retrieve a Signature object, use the :func:`signature` function. -.. function:: signature(callable, \*, follow_wrapped=True) +.. function:: signature(callable, *, follow_wrapped=True) Return a :class:`Signature` object for the given ``callable``:: @@ -597,7 +597,7 @@ function. C provide no metadata about their arguments. -.. class:: Signature(parameters=None, \*, return_annotation=Signature.empty) +.. class:: Signature(parameters=None, *, return_annotation=Signature.empty) A Signature object represents the call signature of a function and its return annotation. For each parameter accepted by the function it stores a @@ -668,7 +668,7 @@ function. >>> str(new_sig) "(a, b) -> 'new return anno'" - .. classmethod:: Signature.from_callable(obj, \*, follow_wrapped=True) + .. classmethod:: Signature.from_callable(obj, *, follow_wrapped=True) Return a :class:`Signature` (or its subclass) object for a given callable ``obj``. Pass ``follow_wrapped=False`` to get a signature of ``obj`` @@ -684,7 +684,7 @@ function. .. versionadded:: 3.5 -.. class:: Parameter(name, kind, \*, default=Parameter.empty, annotation=Parameter.empty) +.. class:: Parameter(name, kind, *, default=Parameter.empty, annotation=Parameter.empty) Parameter objects are *immutable*. Instead of modifying a Parameter object, you can use :meth:`Parameter.replace` to create a modified copy. @@ -809,10 +809,10 @@ function. >>> str(param.replace(default=Parameter.empty, annotation='spam')) "foo:'spam'" - .. versionchanged:: 3.4 - In Python 3.3 Parameter objects were allowed to have ``name`` set - to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. - This is no longer permitted. + .. versionchanged:: 3.4 + In Python 3.3 Parameter objects were allowed to have ``name`` set + to ``None`` if their ``kind`` was set to ``POSITIONAL_ONLY``. + This is no longer permitted. .. class:: BoundArguments diff --git a/Doc/library/io.rst b/Doc/library/io.rst index aecbec56866d73..96e02e839ae653 100644 --- a/Doc/library/io.rst +++ b/Doc/library/io.rst @@ -155,16 +155,6 @@ High-level Module Interface when an unsupported operation is called on a stream. -In-memory streams -^^^^^^^^^^^^^^^^^ - -It is also possible to use a :class:`str` or :term:`bytes-like object` as a -file for both reading and writing. For strings :class:`StringIO` can be used -like a file opened in text mode. :class:`BytesIO` can be used like a file -opened in binary mode. Both provide full read-write capabilities with random -access. - - .. seealso:: :mod:`sys` diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 5f5e66412da477..1c2263b128a8fe 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -104,8 +104,7 @@ write code that handles both IP versions correctly. Address objects are 1. A string in decimal-dot notation, consisting of four decimal integers in the inclusive range 0--255, separated by dots (e.g. ``192.168.0.1``). Each integer represents an octet (byte) in the address. Leading zeroes are - tolerated only for values less than 8 (as there is no ambiguity - between the decimal and octal interpretations of such strings). + not tolerated to prevent confusion with octal notation. 2. An integer that fits into 32 bits. 3. An integer packed into a :class:`bytes` object of length 4 (most significant octet first). @@ -117,6 +116,22 @@ write code that handles both IP versions correctly. Address objects are >>> ipaddress.IPv4Address(b'\xC0\xA8\x00\x01') IPv4Address('192.168.0.1') + .. versionchanged:: 3.8 + + Leading zeros are tolerated, even in ambiguous cases that look like + octal notation. + + .. versionchanged:: 3.10 + + Leading zeros are no longer tolerated and are treated as an error. + IPv4 address strings are now parsed as strict as glibc + :func:`~socket.inet_pton`. + + .. versionchanged:: 3.9.5 + + The above change was also included in Python 3.9 starting with + version 3.9.5. + .. attribute:: version The appropriate version number: ``4`` for IPv4, ``6`` for IPv6. @@ -202,6 +217,32 @@ write code that handles both IP versions correctly. Address objects are .. _iana-ipv4-special-registry: https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml .. _iana-ipv6-special-registry: https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml +.. method:: IPv4Address.__format__(fmt) + + Returns a string representation of the IP address, controlled by + an explicit format string. + *fmt* can be one of the following: ``'s'``, the default option, + equivalent to :func:`str`, ``'b'`` for a zero-padded binary string, + ``'X'`` or ``'x'`` for an uppercase or lowercase hexadecimal + representation, or ``'n'``, which is equivalent to ``'b'`` for IPv4 + addresses and ``'x'`` for IPv6. For binary and hexadecimal + representations, the form specifier ``'#'`` and the grouping option + ``'_'`` are available. ``__format__`` is used by ``format``, ``str.format`` + and f-strings. + + >>> format(ipaddress.IPv4Address('192.168.0.1')) + '192.168.0.1' + >>> '{:#b}'.format(ipaddress.IPv4Address('192.168.0.1')) + '0b11000000101010000000000000000001' + >>> f'{ipaddress.IPv6Address("2001:db8::1000"):s}' + '2001:db8::1000' + >>> format(ipaddress.IPv6Address('2001:db8::1000'), '_X') + '2001_0DB8_0000_0000_0000_0000_0000_1000' + >>> '{:#_n}'.format(ipaddress.IPv6Address('2001:db8::1000')) + '0x2001_0db8_0000_0000_0000_0000_0000_1000' + + .. versionadded:: 3.9 + .. class:: IPv6Address(address) @@ -246,8 +287,8 @@ write code that handles both IP versions correctly. Address objects are groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documentation of the - :class:`IPv4Address` class: + For the following attributes and methods, see the corresponding + documentation of the :class:`IPv4Address` class: .. attribute:: packed .. attribute:: reverse_pointer @@ -297,6 +338,12 @@ write code that handles both IP versions correctly. Address objects are the embedded ``(server, client)`` IP address pair. For any other address, this property will be ``None``. +.. method:: IPv6Address.__format__(fmt) + + Refer to the corresponding method documentation in + :class:`IPv4Address`. + + .. versionadded:: 3.9 Conversion to Strings and Integers ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 3e5a673898106c..6da55f8a3f49cf 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -199,9 +199,9 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable*. - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. + The combination tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, there will be no repeat @@ -248,9 +248,9 @@ loops that truncate the stream. Return *r* length subsequences of elements from the input *iterable* allowing individual elements to be repeated more than once. - Combinations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the combination tuples will be produced - in sorted order. + The combination tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, the generated combinations @@ -484,9 +484,9 @@ loops that truncate the stream. of the *iterable* and all possible full-length permutations are generated. - Permutations are emitted in lexicographic sort order. So, if the - input *iterable* is sorted, the permutation tuples will be produced - in sorted order. + The permutation tuples are emitted in lexicographic ordering according to + the order of the input *iterable*. So, if the input *iterable* is sorted, + the combination tuples will be produced in sorted order. Elements are treated as unique based on their position, not on their value. So if the input elements are unique, there will be no repeat @@ -563,6 +563,9 @@ loops that truncate the stream. for prod in result: yield tuple(prod) + Before :func:`product` runs, it completely consumes the input iterables, + keeping pools of values in memory to generate the products. Accordingly, + it is only useful with finite inputs. .. function:: repeat(object[, times]) @@ -752,7 +755,7 @@ which incur interpreter overhead. "Count how many times the predicate is true" return sum(map(pred, iterable)) - def padnone(iterable): + def pad_none(iterable): """Returns the sequence elements and then returns None indefinitely. Useful for emulating the behavior of the built-in map() function. @@ -766,6 +769,18 @@ which incur interpreter overhead. def dotproduct(vec1, vec2): return sum(map(operator.mul, vec1, vec2)) + def convolve(signal, kernel): + # See: https://betterexplained.com/articles/intuitive-convolution/ + # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) + # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) + # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) + kernel = tuple(kernel)[::-1] + n = len(kernel) + window = collections.deque([0], maxlen=n) * n + for x in chain(signal, repeat(0, n-1)): + window.append(x) + yield sum(map(operator.mul, kernel, window)) + def flatten(list_of_lists): "Flatten one level of nesting" return chain.from_iterable(list_of_lists) @@ -806,7 +821,7 @@ which incur interpreter overhead. nexts = cycle(islice(nexts, num_active)) def partition(pred, iterable): - 'Use a predicate to partition entries into false entries and true entries' + "Use a predicate to partition entries into false entries and true entries" # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 t1, t2 = tee(iterable) return filterfalse(pred, t1), filter(pred, t2) @@ -878,7 +893,7 @@ which incur interpreter overhead. def random_product(*args, repeat=1): "Random selection from itertools.product(*args, **kwds)" pools = [tuple(pool) for pool in args] * repeat - return tuple(random.choice(pool) for pool in pools) + return tuple(map(random.choice, pools)) def random_permutation(iterable, r=None): "Random selection from itertools.permutations(iterable, r)" @@ -897,11 +912,11 @@ which incur interpreter overhead. "Random selection from itertools.combinations_with_replacement(iterable, r)" pool = tuple(iterable) n = len(pool) - indices = sorted(random.randrange(n) for i in range(r)) + indices = sorted(random.choices(range(n), k=r)) return tuple(pool[i] for i in indices) def nth_combination(iterable, r, index): - 'Equivalent to list(combinations(iterable, r))[index]' + "Equivalent to list(combinations(iterable, r))[index]" pool = tuple(iterable) n = len(pool) if r < 0 or r > n: diff --git a/Doc/library/json.rst b/Doc/library/json.rst index b923c8e1670529..c8184da80fe43e 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -333,7 +333,7 @@ Encoders and Decoders *object_hook*, if specified, will be called with the result of every JSON object decoded and its return value will be used in place of the given :class:`dict`. This can be used to provide custom deserializations (e.g. to - support JSON-RPC class hinting). + support `JSON-RPC `_ class hinting). *object_pairs_hook*, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of @@ -422,10 +422,9 @@ Encoders and Decoders for ``o`` if possible, otherwise it should call the superclass implementation (to raise :exc:`TypeError`). - If *skipkeys* is false (the default), then it is a :exc:`TypeError` to - attempt encoding of keys that are not :class:`str`, :class:`int`, - :class:`float` or ``None``. If *skipkeys* is true, such items are simply - skipped. + If *skipkeys* is false (the default), a :exc:`TypeError` will be raised when + trying to encode keys that are not :class:`str`, :class:`int`, :class:`float` + or ``None``. If *skipkeys* is true, such items are simply skipped. If *ensure_ascii* is true (the default), the output is guaranteed to have all incoming non-ASCII characters escaped. If *ensure_ascii* is @@ -479,8 +478,8 @@ Encoders and Decoders object for *o*, or calls the base implementation (to raise a :exc:`TypeError`). - For example, to support arbitrary iterators, you could implement default - like this:: + For example, to support arbitrary iterators, you could implement + :meth:`default` like this:: def default(self, o): try: @@ -745,7 +744,7 @@ Command line options .. cmdoption:: --indent, --tab, --no-indent, --compact - Mutually exclusive options for whitespace control + Mutually exclusive options for whitespace control. .. versionadded:: 3.9 diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst index acec45cdcd586e..5cae79f5dc9dbc 100644 --- a/Doc/library/keyword.rst +++ b/Doc/library/keyword.rst @@ -22,3 +22,19 @@ This module allows a Python program to determine if a string is a Sequence containing all the :ref:`keywords ` defined for the interpreter. If any keywords are defined to only be active when particular :mod:`__future__` statements are in effect, these will be included as well. + + +.. function:: issoftkeyword(s) + + Return ``True`` if *s* is a Python soft :ref:`keyword `. + + .. versionadded:: 3.9 + + +.. data:: softkwlist + + Sequence containing all the soft :ref:`keywords ` defined for the + interpreter. If any soft keywords are defined to only be active when particular + :mod:`__future__` statements are in effect, these will be included as well. + + .. versionadded:: 3.9 diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 683d6ed5e8ba52..0b5e2fc2a658d5 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -35,45 +35,45 @@ in :mod:`logging` itself) and defining handlers which are declared either in .. function:: dictConfig(config) - Takes the logging configuration from a dictionary. The contents of - this dictionary are described in :ref:`logging-config-dictschema` - below. - - If an error is encountered during configuration, this function will - raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` - or :exc:`ImportError` with a suitably descriptive message. The - following is a (possibly incomplete) list of conditions which will - raise an error: - - * A ``level`` which is not a string or which is a string not - corresponding to an actual logging level. - * A ``propagate`` value which is not a boolean. - * An id which does not have a corresponding destination. - * A non-existent handler id found during an incremental call. - * An invalid logger name. - * Inability to resolve to an internal or external object. - - Parsing is performed by the :class:`DictConfigurator` class, whose - constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module - has a callable attribute :attr:`dictConfigClass` - which is initially set to :class:`DictConfigurator`. - You can replace the value of :attr:`dictConfigClass` with a - suitable implementation of your own. - - :func:`dictConfig` calls :attr:`dictConfigClass` passing - the specified dictionary, and then calls the :meth:`configure` method on - the returned object to put the configuration into effect:: - - def dictConfig(config): - dictConfigClass(config).configure() - - For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then - set up custom prefixes which would be usable in the subsequent - :meth:`configure` call. :attr:`dictConfigClass` would be bound to - this new subclass, and then :func:`dictConfig` could be called exactly as - in the default, uncustomized state. + Takes the logging configuration from a dictionary. The contents of + this dictionary are described in :ref:`logging-config-dictschema` + below. + + If an error is encountered during configuration, this function will + raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` + or :exc:`ImportError` with a suitably descriptive message. The + following is a (possibly incomplete) list of conditions which will + raise an error: + + * A ``level`` which is not a string or which is a string not + corresponding to an actual logging level. + * A ``propagate`` value which is not a boolean. + * An id which does not have a corresponding destination. + * A non-existent handler id found during an incremental call. + * An invalid logger name. + * Inability to resolve to an internal or external object. + + Parsing is performed by the :class:`DictConfigurator` class, whose + constructor is passed the dictionary used for configuration, and + has a :meth:`configure` method. The :mod:`logging.config` module + has a callable attribute :attr:`dictConfigClass` + which is initially set to :class:`DictConfigurator`. + You can replace the value of :attr:`dictConfigClass` with a + suitable implementation of your own. + + :func:`dictConfig` calls :attr:`dictConfigClass` passing + the specified dictionary, and then calls the :meth:`configure` method on + the returned object to put the configuration into effect:: + + def dictConfig(config): + dictConfigClass(config).configure() + + For example, a subclass of :class:`DictConfigurator` could call + ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then + set up custom prefixes which would be usable in the subsequent + :meth:`configure` call. :attr:`dictConfigClass` would be bound to + this new subclass, and then :func:`dictConfig` could be called exactly as + in the default, uncustomized state. .. versionadded:: 3.2 diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 7267f812cc1925..a7f815e9983e8d 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -1089,8 +1089,8 @@ functions. suitable value. .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See Issue - #28524 for more information about this change. + The *level* parameter was defaulted to level ``CRITICAL``. See + :issue:`28524` for more information about this change. .. function:: addLevelName(level, levelName) @@ -1106,18 +1106,27 @@ functions. .. function:: getLevelName(level) - Returns the textual representation of logging level *level*. If the level is one - of the predefined levels :const:`CRITICAL`, :const:`ERROR`, :const:`WARNING`, - :const:`INFO` or :const:`DEBUG` then you get the corresponding string. If you - have associated levels with names using :func:`addLevelName` then the name you - have associated with *level* is returned. If a numeric value corresponding to one - of the defined levels is passed in, the corresponding string representation is - returned. Otherwise, the string 'Level %s' % level is returned. + Returns the textual or numeric representation of logging level *level*. + + If *level* is one of the predefined levels :const:`CRITICAL`, :const:`ERROR`, + :const:`WARNING`, :const:`INFO` or :const:`DEBUG` then you get the + corresponding string. If you have associated levels with names using + :func:`addLevelName` then the name you have associated with *level* is + returned. If a numeric value corresponding to one of the defined levels is + passed in, the corresponding string representation is returned. + + The *level* parameter also accepts a string representation of the level such + as 'INFO'. In such cases, this functions returns the corresponding numeric + value of the level. + + If no matching numeric or string value is passed in, the string + 'Level %s' % level is returned. .. note:: Levels are internally integers (as they need to be compared in the logging logic). This function is used to convert between an integer level and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`). + ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`), and + vice versa. .. versionchanged:: 3.4 In Python versions earlier than 3.4, this function could also be passed a @@ -1167,7 +1176,9 @@ functions. | | to ``'a'``. | +--------------+---------------------------------------------+ | *format* | Use the specified format string for the | - | | handler. | + | | handler. Defaults to attributes | + | | ``levelname``, ``name`` and ``message`` | + | | separated by colons. | +--------------+---------------------------------------------+ | *datefmt* | Use the specified date/time format, as | | | accepted by :func:`time.strftime`. | diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst index 4bfff9c6147ed4..633c87873cd8ce 100644 --- a/Doc/library/lzma.rst +++ b/Doc/library/lzma.rst @@ -33,7 +33,7 @@ from multiple threads, it is necessary to protect it with a lock. Reading and writing compressed files ------------------------------------ -.. function:: open(filename, mode="rb", \*, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) +.. function:: open(filename, mode="rb", *, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) Open an LZMA-compressed file in binary or text mode, returning a :term:`file object`. @@ -69,7 +69,7 @@ Reading and writing compressed files Accepts a :term:`path-like object`. -.. class:: LZMAFile(filename=None, mode="r", \*, format=None, check=-1, preset=None, filters=None) +.. class:: LZMAFile(filename=None, mode="r", *, format=None, check=-1, preset=None, filters=None) Open an LZMA-compressed file in binary mode. diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst index f82a3b200deb7c..94d95d10290b00 100644 --- a/Doc/library/mailbox.rst +++ b/Doc/library/mailbox.rst @@ -426,17 +426,14 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. seealso:: - `maildir man page from qmail `_ - The original specification of the format. + `maildir man page from Courier `_ + A specification of the format. Describes a common extension for + supporting folders. `Using maildir format `_ Notes on Maildir by its inventor. Includes an updated name-creation scheme and details on "info" semantics. - `maildir man page from Courier `_ - Another specification of the format. Describes a common extension for supporting - folders. - .. _mailbox-mbox: @@ -485,11 +482,8 @@ Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. .. seealso:: - `mbox man page from qmail `_ - A specification of the format and its variations. - `mbox man page from tin `_ - Another specification of the format, with details on locking. + A specification of the format, with details on locking. `Configuring Netscape Mail on Unix: Why The Content-Length Format is Bad `_ An argument for using the original mbox format rather than a variation. diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst index d65afc20041133..b38ba54b3c3bc6 100644 --- a/Doc/library/marshal.rst +++ b/Doc/library/marshal.rst @@ -66,6 +66,8 @@ The module defines these functions: The *version* argument indicates the data format that ``dump`` should use (see below). + .. audit-event:: marshal.dumps value,version marshal.dump + .. function:: load(file) @@ -74,11 +76,18 @@ The module defines these functions: format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The file must be a readable :term:`binary file`. + .. audit-event:: marshal.load "" marshal.load + .. note:: If an object containing an unsupported type was marshalled with :func:`dump`, :func:`load` will substitute ``None`` for the unmarshallable type. + .. versionchanged:: 3.9.7 + + This call used to raise a ``code.__new__`` audit event for each code object. Now + it raises a single ``marshal.load`` event for the entire load operation. + .. function:: dumps(value[, version]) @@ -89,6 +98,8 @@ The module defines these functions: The *version* argument indicates the data format that ``dumps`` should use (see below). + .. audit-event:: marshal.dumps value,version marshal.dump + .. function:: loads(bytes) @@ -96,6 +107,13 @@ The module defines these functions: :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the input are ignored. + .. audit-event:: marshal.loads bytes marshal.load + + .. versionchanged:: 3.9.7 + + This call used to raise a ``code.__new__`` audit event for each code object. Now + it raises a single ``marshal.loads`` event for the entire load operation. + In addition, the following constants are defined: diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 6ec1feee35a6dc..b20e557b5c6109 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -130,7 +130,7 @@ Number-theoretic and representation functions Return the greatest common divisor of the specified integer arguments. If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor af all arguments. If all arguments + positive integer that is a divisor of all arguments. If all arguments are zero, then the returned value is ``0``. ``gcd()`` without arguments returns ``0``. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst index 50b90031ab5a58..def27bf07a03e4 100644 --- a/Doc/library/multiprocessing.rst +++ b/Doc/library/multiprocessing.rst @@ -14,7 +14,8 @@ Introduction :mod:`multiprocessing` is a package that supports spawning processes using an API similar to the :mod:`threading` module. The :mod:`multiprocessing` package offers both local and remote concurrency, effectively side-stepping the -:term:`Global Interpreter Lock` by using subprocesses instead of threads. Due +:term:`Global Interpreter Lock ` by using +subprocesses instead of threads. Due to this, the :mod:`multiprocessing` module allows the programmer to fully leverage multiple processors on a given machine. It runs on both Unix and Windows. @@ -97,7 +98,7 @@ to start a process. These *start methods* are *spawn* The parent process starts a fresh python interpreter process. The child process will only inherit those resources necessary to run - the process objects :meth:`~Process.run` method. In particular, + the process object's :meth:`~Process.run` method. In particular, unnecessary file descriptors and handles from the parent process will not be inherited. Starting a process using this method is rather slow compared to using *fork* or *forkserver*. @@ -1925,7 +1926,7 @@ client to access it remotely:: >>> class Worker(Process): ... def __init__(self, q): ... self.q = q - ... super(Worker, self).__init__() + ... super().__init__() ... def run(self): ... self.q.put('local hello') ... @@ -2173,7 +2174,8 @@ with the :class:`Pool` class. .. method:: apply_async(func[, args[, kwds[, callback[, error_callback]]]]) - A variant of the :meth:`apply` method which returns a result object. + A variant of the :meth:`apply` method which returns a + :class:`~multiprocessing.pool.AsyncResult` object. If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to @@ -2203,7 +2205,8 @@ with the :class:`Pool` class. .. method:: map_async(func, iterable[, chunksize[, callback[, error_callback]]]) - A variant of the :meth:`.map` method which returns a result object. + A variant of the :meth:`.map` method which returns a + :class:`~multiprocessing.pool.AsyncResult` object. If *callback* is specified then it should be a callable which accepts a single argument. When the result becomes ready *callback* is applied to @@ -2566,9 +2569,9 @@ Address Formats filesystem. * An ``'AF_PIPE'`` address is a string of the form - :samp:`r'\\\\.\\pipe\\{PipeName}'`. To use :func:`Client` to connect to a named - pipe on a remote computer called *ServerName* one should use an address of the - form :samp:`r'\\\\{ServerName}\\pipe\\{PipeName}'` instead. + :samp:`r'\\\\.\\pipe\\{PipeName}'`. To use :func:`Client` to connect to a named + pipe on a remote computer called *ServerName* one should use an address of the + form :samp:`r'\\\\{ServerName}\\pipe\\{PipeName}'` instead. Note that any string beginning with two backslashes is assumed by default to be an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. @@ -2658,6 +2661,46 @@ The :mod:`multiprocessing.dummy` module :mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is no more than a wrapper around the :mod:`threading` module. +.. currentmodule:: multiprocessing.pool + +In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` +returns an instance of :class:`ThreadPool`, which is a subclass of +:class:`Pool` that supports all the same method calls but uses a pool of +worker threads rather than worker processes. + + +.. class:: ThreadPool([processes[, initializer[, initargs]]]) + + A thread pool object which controls a pool of worker threads to which jobs + can be submitted. :class:`ThreadPool` instances are fully interface + compatible with :class:`Pool` instances, and their resources must also be + properly managed, either by using the pool as a context manager or by + calling :meth:`~multiprocessing.pool.Pool.close` and + :meth:`~multiprocessing.pool.Pool.terminate` manually. + + *processes* is the number of worker threads to use. If *processes* is + ``None`` then the number returned by :func:`os.cpu_count` is used. + + If *initializer* is not ``None`` then each worker process will call + ``initializer(*initargs)`` when it starts. + + Unlike :class:`Pool`, *maxtasksperchild* and *context* cannot be provided. + + .. note:: + + A :class:`ThreadPool` shares the same interface as :class:`Pool`, which + is designed around a pool of processes and predates the introduction of + the :class:`concurrent.futures` module. As such, it inherits some + operations that don't make sense for a pool backed by threads, and it + has its own type for representing the status of asynchronous jobs, + :class:`AsyncResult`, that is not understood by any other libraries. + + Users should generally prefer to use + :class:`concurrent.futures.ThreadPoolExecutor`, which has a simpler + interface that was designed around threads from the start, and which + returns :class:`concurrent.futures.Future` instances that are + compatible with many other libraries, including :mod:`asyncio`. + .. _multiprocessing-programming: diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst index 1b594952ead724..b77845ed0dee92 100644 --- a/Doc/library/numbers.rst +++ b/Doc/library/numbers.rst @@ -10,7 +10,7 @@ The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric :term:`abstract base classes ` which progressively define -more operations. None of the types defined in this module can be instantiated. +more operations. None of the types defined in this module are intended to be instantiated. .. class:: Number @@ -27,8 +27,8 @@ The numeric tower Subclasses of this type describe complex numbers and include the operations that work on the built-in :class:`complex` type. These are: conversions to :class:`complex` and :class:`bool`, :attr:`.real`, :attr:`.imag`, ``+``, - ``-``, ``*``, ``/``, :func:`abs`, :meth:`conjugate`, ``==``, and ``!=``. All - except ``-`` and ``!=`` are abstract. + ``-``, ``*``, ``/``, ``**``, :func:`abs`, :meth:`conjugate`, ``==``, and + ``!=``. All except ``-`` and ``!=`` are abstract. .. attribute:: real @@ -76,8 +76,9 @@ The numeric tower Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`. Adds abstract methods for ``**`` and - bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, ``~``. + :attr:`~Rational.denominator`. Adds abstract methods for :func:`pow` with + modulus and bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, + ``~``. Notes for type implementors diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst index fa02bde84650e1..f2fb084ac17202 100644 --- a/Doc/library/operator.rst +++ b/Doc/library/operator.rst @@ -315,15 +315,12 @@ expect a function argument. method. Dictionaries accept any hashable value. Lists, tuples, and strings accept an index or a slice: - >>> itemgetter('name')({'name': 'tu', 'age': 18}) - 'tu' >>> itemgetter(1)('ABCDEFG') 'B' - >>> itemgetter(1,3,5)('ABCDEFG') + >>> itemgetter(1, 3, 5)('ABCDEFG') ('B', 'D', 'F') - >>> itemgetter(slice(2,None))('ABCDEFG') + >>> itemgetter(slice(2, None))('ABCDEFG') 'CDEFG' - >>> soldier = dict(rank='captain', name='dotterbart') >>> itemgetter('rank')(soldier) 'captain' diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index c1a18e01474319..b1094198f4c844 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -55,7 +55,7 @@ equivalent to the above example:: -q -foutfile -qfoutfile -Additionally, users can run one of :: +Additionally, users can run one of the following :: -h --help diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst index a5abacf02144a7..d2fe4943524533 100644 --- a/Doc/library/os.path.rst +++ b/Doc/library/os.path.rst @@ -306,11 +306,10 @@ the :mod:`glob` module.) Join one or more path components intelligently. The return value is the concatenation of *path* and any members of *\*paths* with exactly one - directory separator (``os.sep``) following each non-empty part except the - last, meaning that the result will only end in a separator if the last - part is empty. If a component is an absolute path, all previous - components are thrown away and joining continues from the absolute path - component. + directory separator following each non-empty part except the last, meaning + that the result will only end in a separator if the last part is empty. If + a component is an absolute path, all previous components are thrown away + and joining continues from the absolute path component. On Windows, the drive letter is not reset when an absolute path component (e.g., ``r'\foo'``) is encountered. If a component contains a drive @@ -367,7 +366,8 @@ the :mod:`glob` module.) Return a relative filepath to *path* either from the current directory or from an optional *start* directory. This is a path computation: the filesystem is not accessed to confirm the existence or nature of *path* or - *start*. + *start*. On Windows, :exc:`ValueError` is raised when *path* and *start* + are on different drives. *start* defaults to :attr:`os.curdir`. diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 6d5fb314a8e39a..2295ffc82f1737 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -1837,6 +1837,8 @@ features: Return a list containing the names of the entries in the directory given by *path*. The list is in arbitrary order, and does not include the special entries ``'.'`` and ``'..'`` even if they are present in the directory. + If a file is removed from or added to the directory during the call of + this function, whether a name for that file be included is unspecified. *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` (directly or indirectly through the :class:`PathLike` interface), @@ -1867,7 +1869,7 @@ features: Accepts a :term:`path-like object`. -.. function:: lstat(path, \*, dir_fd=None) +.. function:: lstat(path, *, dir_fd=None) Perform the equivalent of an :c:func:`lstat` system call on the given path. Similar to :func:`~os.stat`, but does not follow symbolic links. Return a @@ -1893,7 +1895,7 @@ features: Added the *dir_fd* parameter. .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. + Accepts a :term:`path-like object`. .. versionchanged:: 3.8 On Windows, now opens reparse points that represent another path @@ -2113,6 +2115,7 @@ features: Remove (delete) the file *path*. If *path* is a directory, an :exc:`IsADirectoryError` is raised. Use :func:`rmdir` to remove directories. + If the file does not exist, a :exc:`FileNotFoundError` is raised. This function can support :ref:`paths relative to directory descriptors `. @@ -2242,7 +2245,9 @@ features: Return an iterator of :class:`os.DirEntry` objects corresponding to the entries in the directory given by *path*. The entries are yielded in arbitrary order, and the special entries ``'.'`` and ``'..'`` are not - included. + included. If a file is removed from or added to the directory after + creating the iterator, whether an entry for that file be included is + unspecified. Using :func:`scandir` instead of :func:`listdir` can significantly increase the performance of code that also needs file type or file @@ -2372,7 +2377,7 @@ features: On the first, uncached call, a system call is required on Windows but not on Unix. - .. method:: is_dir(\*, follow_symlinks=True) + .. method:: is_dir(*, follow_symlinks=True) Return ``True`` if this entry is a directory or a symbolic link pointing to a directory; return ``False`` if the entry is or points to any other @@ -2396,7 +2401,7 @@ features: This method can raise :exc:`OSError`, such as :exc:`PermissionError`, but :exc:`FileNotFoundError` is caught and not raised. - .. method:: is_file(\*, follow_symlinks=True) + .. method:: is_file(*, follow_symlinks=True) Return ``True`` if this entry is a file or a symbolic link pointing to a file; return ``False`` if the entry is or points to a directory or other @@ -2426,7 +2431,7 @@ features: This method can raise :exc:`OSError`, such as :exc:`PermissionError`, but :exc:`FileNotFoundError` is caught and not raised. - .. method:: stat(\*, follow_symlinks=True) + .. method:: stat(*, follow_symlinks=True) Return a :class:`stat_result` object for this entry. This method follows symbolic links by default; to stat a symbolic link add the @@ -2458,7 +2463,7 @@ features: for :class:`bytes` paths on Windows. -.. function:: stat(path, \*, dir_fd=None, follow_symlinks=True) +.. function:: stat(path, *, dir_fd=None, follow_symlinks=True) Get the status of a file or a file descriptor. Perform the equivalent of a :c:func:`stat` system call on the given path. *path* may be specified as @@ -2992,7 +2997,10 @@ features: *filenames* is a list of the names of the non-directory files in *dirpath*. Note that the names in the lists contain no path components. To get a full path (which begins with *top*) to a file or directory in *dirpath*, do - ``os.path.join(dirpath, name)``. + ``os.path.join(dirpath, name)``. Whether or not the lists are sorted + depends on the file system. If a file is removed from or added to the + *dirpath* directory during generating the lists, whether a name for that + file be included is unspecified. If optional argument *topdown* is ``True`` or not specified, the triple for a directory is generated before the triples for any of its subdirectories @@ -3242,9 +3250,9 @@ These functions are all available on Linux only. indirectly through the :class:`PathLike` interface). If it is a str, it is encoded with the filesystem encoding. *flags* may be :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is - given and the attribute does not exist, ``EEXISTS`` will be raised. + given and the attribute does not exist, ``ENODATA`` will be raised. If :data:`XATTR_CREATE` is given and the attribute already exists, the - attribute will not be created and ``ENODATA`` will be raised. + attribute will not be created and ``EEXISTS`` will be raised. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. @@ -3688,8 +3696,8 @@ written in Python, such as a mail server's external command delivery program. The positional-only arguments *path*, *args*, and *env* are similar to :func:`execve`. - The *path* parameter is the path to the executable file.The *path* should - contain a directory.Use :func:`posix_spawnp` to pass an executable file + The *path* parameter is the path to the executable file. The *path* should + contain a directory. Use :func:`posix_spawnp` to pass an executable file without directory. The *file_actions* argument may be a sequence of tuples describing actions @@ -3957,12 +3965,12 @@ written in Python, such as a mail server's external command delivery program. the Standard C function :c:func:`system`, and has the same limitations. Changes to :data:`sys.stdin`, etc. are not reflected in the environment of the executed command. If *command* generates any output, it will be sent to - the interpreter standard output stream. + the interpreter standard output stream. The C standard does not + specify the meaning of the return value of the C function, so the return + value of the Python function is system-dependent. On Unix, the return value is the exit status of the process encoded in the - format specified for :func:`wait`. Note that POSIX does not specify the - meaning of the return value of the C :c:func:`system` function, so the return - value of the Python function is system-dependent. + format specified for :func:`wait`. On Windows, the return value is that returned by the system shell after running *command*. The shell is given by the Windows environment variable @@ -4381,7 +4389,7 @@ operating system. .. function:: sched_setparam(pid, param) - Set a scheduling parameters for the process with PID *pid*. A *pid* of 0 means + Set the scheduling parameters for the process with PID *pid*. A *pid* of 0 means the calling process. *param* is a :class:`sched_param` instance. diff --git a/Doc/library/othergui.rst b/Doc/library/othergui.rst index 4548459f8e261d..48c1f2754111aa 100644 --- a/Doc/library/othergui.rst +++ b/Doc/library/othergui.rst @@ -30,10 +30,11 @@ available for Python: for generating bindings for C++ libraries as Python classes, and is specifically designed for Python. - `PySide `_ - PySide is a newer binding to the Qt toolkit, provided by Nokia. - Compared to PyQt, its licensing scheme is friendlier to non-open source - applications. + `PySide2 `_ + Also known as the Qt for Python project, PySide2 is a newer binding to the + Qt toolkit. It is provided by The Qt Company and aims to provide a + complete port of PySide to Qt 5. Compared to PyQt, its licensing scheme is + friendlier to non-open source applications. `wxPython `_ wxPython is a cross-platform GUI toolkit for Python that is built around @@ -47,7 +48,7 @@ available for Python: an XML-based resource format and more, including an ever growing library of user-contributed modules. -PyGTK, PyQt, and wxPython, all have a modern look and feel and more +PyGTK, PyQt, PySide2, and wxPython, all have a modern look and feel and more widgets than Tkinter. In addition, there are many other GUI toolkits for Python, both cross-platform, and platform-specific. See the `GUI Programming `_ page in the Python Wiki for a diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst index 83f7c836f0e71c..c31ff6543aee52 100644 --- a/Doc/library/pathlib.rst +++ b/Doc/library/pathlib.rst @@ -551,7 +551,9 @@ Pure paths provide the following methods and properties: File "", line 1, in File "pathlib.py", line 694, in relative_to .format(str(self), str(formatted))) - ValueError: '/etc/passwd' does not start with '/usr' + ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other absolute. + + NOTE: This function is part of :class:`PurePath` and works with strings. It does not check or access the underlying file structure. .. method:: PurePath.with_name(name) @@ -888,6 +890,11 @@ call fails (for example because the path doesn't exist). PosixPath('docs/_static') PosixPath('docs/Makefile') + The children are yielded in arbitrary order, and the special entries + ``'.'`` and ``'..'`` are not included. If a file is removed from or added + to the directory after creating the iterator, whether an path object for + that file be included is unspecified. + .. method:: Path.lchmod(mode) Like :meth:`Path.chmod` but, if the path points to a symbolic link, the @@ -1001,6 +1008,10 @@ call fails (for example because the path doesn't exist). >>> target.open().read() 'some text' + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the Path + object. + .. versionchanged:: 3.8 Added return value, return the new Path instance. @@ -1011,6 +1022,10 @@ call fails (for example because the path doesn't exist). instance pointing to *target*. If *target* points to an existing file or directory, it will be unconditionally replaced. + The target path may be absolute or relative. Relative paths are interpreted + relative to the current working directory, *not* the directory of the Path + object. + .. versionchanged:: 3.8 Added return value, return the new Path instance. @@ -1104,6 +1119,20 @@ call fails (for example because the path doesn't exist). of :func:`os.symlink`'s. +.. method:: Path.link_to(target) + + Make *target* a hard link to this path. + + .. warning:: + + This function does not make this path a hard link to *target*, despite + the implication of the function and argument names. The argument order + (target, link) is the reverse of :func:`Path.symlink_to`, but matches + that of :func:`os.link`. + + .. versionadded:: 3.8 + + .. method:: Path.touch(mode=0o666, exist_ok=True) Create a file at this given path. If *mode* is given, it is combined @@ -1128,13 +1157,6 @@ call fails (for example because the path doesn't exist). The *missing_ok* parameter was added. -.. method:: Path.link_to(target) - - Create a hard link pointing to a path named *target*. - - .. versionchanged:: 3.8 - - .. method:: Path.write_bytes(data) Open the file pointed to in bytes mode, write *data* to it, and close the @@ -1194,9 +1216,12 @@ os and os.path pathlib :func:`os.path.exists` :meth:`Path.exists` :func:`os.path.expanduser` :meth:`Path.expanduser` and :meth:`Path.home` +:func:`os.listdir` :meth:`Path.iterdir` :func:`os.path.isdir` :meth:`Path.is_dir` :func:`os.path.isfile` :meth:`Path.is_file` :func:`os.path.islink` :meth:`Path.is_symlink` +:func:`os.link` :meth:`Path.link_to` +:func:`os.symlink` :meth:`Path.symlink_to` :func:`os.readlink` :meth:`Path.readlink` :func:`os.stat` :meth:`Path.stat`, :meth:`Path.owner`, diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst index 606e8e53457706..ed1e9712c0e3de 100644 --- a/Doc/library/pdb.rst +++ b/Doc/library/pdb.rst @@ -538,6 +538,7 @@ by the local file. executed in the current environment). .. pdbcommand:: retval + Print the return value for the last return of a function. .. rubric:: Footnotes diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst index b7c34527719486..be48561ed10ac8 100644 --- a/Doc/library/pickle.rst +++ b/Doc/library/pickle.rst @@ -213,7 +213,7 @@ The :mod:`pickle` module provides the following constants: The :mod:`pickle` module provides the following functions to make the pickling process more convenient: -.. function:: dump(obj, file, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) Write the pickled representation of the object *obj* to the open :term:`file object` *file*. This is equivalent to @@ -225,7 +225,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffer_callback* argument was added. -.. function:: dumps(obj, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. function:: dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None) Return the pickled representation of the object *obj* as a :class:`bytes` object, instead of writing it to a file. @@ -236,7 +236,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffer_callback* argument was added. -.. function:: load(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. function:: load(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Read the pickled representation of an object from the open :term:`file object` *file* and return the reconstituted object hierarchy specified therein. @@ -252,7 +252,7 @@ process more convenient: .. versionchanged:: 3.8 The *buffers* argument was added. -.. function:: loads(data, /, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. function:: loads(data, /, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) Return the reconstituted object hierarchy of the pickled representation *data* of an object. *data* must be a :term:`bytes-like object`. @@ -296,7 +296,7 @@ The :mod:`pickle` module defines three exceptions: The :mod:`pickle` module exports three classes, :class:`Pickler`, :class:`Unpickler` and :class:`PickleBuffer`: -.. class:: Pickler(file, protocol=None, \*, fix_imports=True, buffer_callback=None) +.. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) This takes a binary file for writing a pickle data stream. @@ -391,7 +391,7 @@ The :mod:`pickle` module exports three classes, :class:`Pickler`, Use :func:`pickletools.optimize` if you need more compact pickles. -.. class:: Unpickler(file, \*, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) +.. class:: Unpickler(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) This takes a binary file for reading a pickle data stream. diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst index 2066cbb9fc57ce..3b17b9a6219870 100644 --- a/Doc/library/pkgutil.rst +++ b/Doc/library/pkgutil.rst @@ -68,7 +68,7 @@ support. .. class:: ImpLoader(fullname, file, filename, etc) - :term:`Loader` that wraps Python's "classic" import algorithm. + :term:`Loader ` that wraps Python's "classic" import algorithm. .. deprecated:: 3.3 This emulation is no longer needed, as the standard import mechanism diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 8e8e3775aaff4a..b293adf48e6e33 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -209,13 +209,6 @@ Windows Platform which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. - .. note:: - - This function works best with Mark Hammond's - :mod:`win32all` package installed, but also on Python 2.3 and - later (support for this was added in Python 2.6). It obviously - only runs on Win32 compatible platforms. - .. function:: win32_edition() Returns a string representing the current Windows edition. Possible diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst index 6def72b3736b91..ce6d4a85bf5e9d 100644 --- a/Doc/library/plistlib.rst +++ b/Doc/library/plistlib.rst @@ -52,7 +52,7 @@ or :class:`datetime.datetime` objects. This module defines the following functions: -.. function:: load(fp, \*, fmt=None, dict_type=dict) +.. function:: load(fp, *, fmt=None, dict_type=dict) Read a plist file. *fp* should be a readable and binary file object. Return the unpacked root object (which usually is a @@ -80,7 +80,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: loads(data, \*, fmt=None, dict_type=dict) +.. function:: loads(data, *, fmt=None, dict_type=dict) Load a plist from a bytes object. See :func:`load` for an explanation of the keyword arguments. @@ -88,7 +88,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: dump(value, fp, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) Write *value* to a plist file. *Fp* should be a writable, binary file object. @@ -116,7 +116,7 @@ This module defines the following functions: .. versionadded:: 3.4 -.. function:: dumps(value, \*, fmt=FMT_XML, sort_keys=True, skipkeys=False) +.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) Return *value* as a plist-formatted bytes object. See the documentation for :func:`dump` for an explanation of the keyword diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst index 2f349b35b7e0ff..9bf9212d917a55 100644 --- a/Doc/library/poplib.rst +++ b/Doc/library/poplib.rst @@ -67,7 +67,7 @@ The :mod:`poplib` module provides two classes: .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL - .. audit-event:: poplib.putline self,line popplib.POP3_SSL + .. audit-event:: poplib.putline self,line poplib.POP3_SSL All commands will raise an :ref:`auditing event ` ``poplib.putline`` with arguments ``self`` and ``line``, diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst index 34525a96f55c43..774d46d0e96247 100644 --- a/Doc/library/profile.rst +++ b/Doc/library/profile.rst @@ -525,16 +525,16 @@ Analysis of the profiler data is done using the :class:`~pstats.Stats` class. ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - .. method:: get_stats_profile() + .. method:: get_stats_profile() This method returns an instance of StatsProfile, which contains a mapping of function names to instances of FunctionProfile. Each FunctionProfile instance holds information related to the function's profile such as how long the function took to run, how many times it was called, etc... - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. + .. versionadded:: 3.9 + Added the following dataclasses: StatsProfile, FunctionProfile. + Added the following function: get_stats_profile. .. _deterministic-profiling: diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst index a12a5bb0b1aa2c..cac6dcb63e7f3e 100644 --- a/Doc/library/py_compile.rst +++ b/Doc/library/py_compile.rst @@ -35,7 +35,7 @@ byte-code cache files in the directory containing the source code. in ``.pyc``. For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is - specified, it is used as the name of the source file in error messages when + specified, it is used as the name of the source file in error messages instead of *file*. If *doraise* is true, a :exc:`PyCompileError` is raised when an error is encountered while compiling *file*. If *doraise* is false (the default), an error string is written to ``sys.stderr``, but no exception diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst index e43b9aecd86835..034e579315de00 100644 --- a/Doc/library/pyexpat.rst +++ b/Doc/library/pyexpat.rst @@ -665,14 +665,14 @@ The ``errors`` module has the following attributes: .. data:: codes - A dictionary mapping numeric error codes to their string descriptions. + A dictionary mapping string descriptions to their error codes. .. versionadded:: 3.2 .. data:: messages - A dictionary mapping string descriptions to their error codes. + A dictionary mapping numeric error codes to their string descriptions. .. versionadded:: 3.2 diff --git a/Doc/library/random.rst b/Doc/library/random.rst index 90366f499cae6a..f2d6749422a15a 100644 --- a/Doc/library/random.rst +++ b/Doc/library/random.rst @@ -142,10 +142,11 @@ Functions for integers .. function:: getrandbits(k) - Returns a Python integer with *k* random bits. This method is supplied with - the MersenneTwister generator and some other generators may also provide it - as an optional part of the API. When available, :meth:`getrandbits` enables - :meth:`randrange` to handle arbitrarily large ranges. + Returns a non-negative Python integer with *k* random bits. This method + is supplied with the MersenneTwister generator and some other generators + may also provide it as an optional part of the API. When available, + :meth:`getrandbits` enables :meth:`randrange` to handle arbitrarily large + ranges. .. versionchanged:: 3.9 This method now accepts zero for *k*. @@ -253,6 +254,8 @@ Functions for sequences order so that the sample is reproducible. +.. _real-valued-distributions: + Real-valued distributions ------------------------- @@ -317,6 +320,13 @@ be found in any statistics text. deviation. This is slightly faster than the :func:`normalvariate` function defined below. + Multithreading note: When two threads call this function + simultaneously, it is possible that they will receive the + same return value. This can be avoided in three ways. + 1) Have each thread use a different instance of the random + number generator. 2) Put locks around all calls. 3) Use the + slower, but thread-safe :func:`normalvariate` function instead. + .. function:: lognormvariate(mu, sigma) @@ -391,15 +401,15 @@ change across Python versions, but two aspects are guaranteed not to change: .. _random-examples: -Examples and Recipes --------------------- +Examples +-------- Basic examples:: >>> random() # Random float: 0.0 <= x < 1.0 0.37444887175646646 - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x < 10.0 + >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 3.1800146073117523 >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds @@ -536,3 +546,58 @@ Simulation of arrival times and service deliveries for a multiserver queue:: a tutorial by `Peter Norvig `_ covering the basics of probability theory, how to write simulations, and how to perform data analysis using Python. + + +Recipes +------- + +The default :func:`.random` returns multiples of 2â»âµÂ³ in the range +*0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly +representable as Python floats. However, many other representable +floats in that interval are not possible selections. For example, +``0.05954861408025609`` isn't an integer multiple of 2â»âµÂ³. + +The following recipe takes a different approach. All floats in the +interval are possible selections. The mantissa comes from a uniform +distribution of integers in the range *2âµÂ² ≤ mantissa < 2âµÂ³*. The +exponent comes from a geometric distribution where exponents smaller +than *-53* occur half as often as the next larger exponent. + +:: + + from random import Random + from math import ldexp + + class FullRandom(Random): + + def random(self): + mantissa = 0x10_0000_0000_0000 | self.getrandbits(52) + exponent = -53 + x = 0 + while not x: + x = self.getrandbits(32) + exponent += x.bit_length() - 32 + return ldexp(mantissa, exponent) + +All :ref:`real valued distributions ` +in the class will use the new method:: + + >>> fr = FullRandom() + >>> fr.random() + 0.05954861408025609 + >>> fr.expovariate(0.25) + 8.87925541791544 + +The recipe is conceptually equivalent to an algorithm that chooses from +all the multiples of 2â»Â¹â°â·â´ in the range *0.0 ≤ x < 1.0*. All such +numbers are evenly spaced, but most have to be rounded down to the +nearest representable Python float. (The value 2â»Â¹â°â·â´ is the smallest +positive unnormalized float and is equal to ``math.ulp(0.0)``.) + + +.. seealso:: + + `Generating Pseudo-random Floating-Point Values + `_ a + paper by Allen B. Downey describing ways to generate more + fine-grained floats than normally generated by :func:`.random`. diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst index e4eac43642d14d..c8c1348b9ba77e 100644 --- a/Doc/library/resource.rst +++ b/Doc/library/resource.rst @@ -241,7 +241,9 @@ platform. The maximum size (in bytes) of the swap space that may be reserved or used by all of this user id's processes. This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. - Please see :manpage:`tuning(7)` for a complete description of this sysctl. + Please see + `tuning(7) `__ + for a complete description of this sysctl. .. availability:: FreeBSD 9 or later. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst index bc4766da2785b3..afa8e2d385fa46 100644 --- a/Doc/library/secrets.rst +++ b/Doc/library/secrets.rst @@ -21,7 +21,7 @@ The :mod:`secrets` module is used for generating cryptographically strong random numbers suitable for managing data such as passwords, account authentication, security tokens, and related secrets. -In particularly, :mod:`secrets` should be used in preference to the +In particular, :mod:`secrets` should be used in preference to the default pseudo-random number generator in the :mod:`random` module, which is designed for modelling and simulation, not security or cryptography. diff --git a/Doc/library/select.rst b/Doc/library/select.rst index bb2809580d0401..a354187c266c7f 100644 --- a/Doc/library/select.rst +++ b/Doc/library/select.rst @@ -117,7 +117,7 @@ The module defines the following: .. function:: select(rlist, wlist, xlist[, timeout]) This is a straightforward interface to the Unix :c:func:`select` system call. - The first three arguments are sequences of 'waitable objects': either + The first three arguments are iterables of 'waitable objects': either integers representing file descriptors or objects with a parameterless method named :meth:`~io.IOBase.fileno` returning such an integer: @@ -126,7 +126,7 @@ The module defines the following: * *xlist*: wait for an "exceptional condition" (see the manual page for what your system considers such a condition) - Empty sequences are allowed, but acceptance of three empty sequences is + Empty iterables are allowed, but acceptance of three empty iterables is platform-dependent. (It is known to work on Unix but not on Windows.) The optional *timeout* argument specifies a time-out as a floating point number in seconds. When the *timeout* argument is omitted the function blocks until @@ -141,7 +141,7 @@ The module defines the following: single: socket() (in module socket) single: popen() (in module os) - Among the acceptable object types in the sequences are Python :term:`file + Among the acceptable object types in the iterables are Python :term:`file objects ` (e.g. ``sys.stdin``, or objects returned by :func:`open` or :func:`os.popen`), socket objects returned by :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index c7c63e6f80844a..d5080da15bba41 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -158,9 +158,9 @@ Directory and files operations .. function:: copy(src, dst, *, follow_symlinks=True) Copies the file *src* to the file or directory *dst*. *src* and *dst* - should be strings. If *dst* specifies a directory, the file will be - copied into *dst* using the base filename from *src*. Returns the - path to the newly created file. + should be :term:`path-like objects ` or strings. If + *dst* specifies a directory, the file will be copied into *dst* using the + base filename from *src*. Returns the path to the newly created file. If *follow_symlinks* is false, and *src* is a symbolic link, *dst* will be created as a symbolic link. If *follow_symlinks* @@ -218,7 +218,7 @@ Directory and files operations copy the file more efficiently. See :ref:`shutil-platform-dependent-efficient-copy-operations` section. -.. function:: ignore_patterns(\*patterns) +.. function:: ignore_patterns(*patterns) This factory function creates a function that can be used as a callable for :func:`copytree`\'s *ignore* argument, ignoring files and directories that @@ -349,7 +349,7 @@ Directory and files operations will be created in or as *dst* and *src* will be removed. If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dest* if + *src* and *dst*, and will be used to copy *src* to *dst* if :func:`os.rename` cannot be used. If the source is a directory, :func:`copytree` is called, passing it the :func:`copy_function`. The default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the @@ -443,8 +443,9 @@ Directory and files operations Platform-dependent efficient copy operations ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Starting from Python 3.8 all functions involving a file copy (:func:`copyfile`, -:func:`copy`, :func:`copy2`, :func:`copytree`, and :func:`move`) may use +Starting from Python 3.8, all functions involving a file copy +(:func:`copyfile`, :func:`~shutil.copy`, :func:`copy2`, +:func:`copytree`, and :func:`move`) may use platform-specific "fast-copy" syscalls in order to copy the file more efficiently (see :issue:`33671`). "fast-copy" means that the copying operation occurs within the kernel, avoiding @@ -570,12 +571,14 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. available), or "xztar" (if the :mod:`lzma` module is available). *root_dir* is a directory that will be the root directory of the - archive; for example, we typically chdir into *root_dir* before creating the - archive. + archive, all paths in the archive will be relative to it; for example, + we typically chdir into *root_dir* before creating the archive. *base_dir* is the directory where we start archiving from; i.e. *base_dir* will be the common prefix of all files and - directories in the archive. + directories in the archive. *base_dir* must be given relative + to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to + use *base_dir* and *root_dir* together. *root_dir* and *base_dir* both default to the current directory. @@ -727,6 +730,48 @@ The resulting archive contains: -rw-r--r-- tarek/staff 37192 2010-02-06 18:23:10 ./known_hosts +.. _shutil-archiving-example-with-basedir: + +Archiving example with *base_dir* +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In this example, similar to the `one above `_, +we show how to use :func:`make_archive`, but this time with the usage of +*base_dir*. We now have the following directory structure: + +.. code-block:: shell-session + + $ tree tmp + tmp + └── root + └── structure + ├── content + └── please_add.txt + └── do_not_add.txt + +In the final archive, :file:`please_add.txt` should be included, but +:file:`do_not_add.txt` should not. Therefore we use the following:: + + >>> from shutil import make_archive + >>> import os + >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) + >>> make_archive( + ... archive_name, + ... 'tar', + ... root_dir='tmp/root', + ... base_dir='structure/content', + ... ) + '/Users/tarek/my_archive.tar' + +Listing the files in the resulting archive gives us: + +.. code-block:: shell-session + + $ python -m tarfile -l /Users/tarek/myarchive.tar + structure/content/ + structure/content/please_add.txt + + Querying the size of the output terminal ---------------------------------------- diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst index 05b285ed110ec4..e1daeff48ad8fc 100644 --- a/Doc/library/signal.rst +++ b/Doc/library/signal.rst @@ -117,7 +117,7 @@ The variables defined in the :mod:`signal` module are: Child process stopped or terminated. - .. availability:: Windows. + .. availability:: Unix. .. data:: SIGCLD diff --git a/Doc/library/site.rst b/Doc/library/site.rst index b424e1ba348d87..2e3646f6a74f80 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -231,7 +231,9 @@ Module contents Return the path of the user-specific site-packages directory, :data:`USER_SITE`. If it is not initialized yet, this function will also set - it, respecting :envvar:`PYTHONNOUSERSITE` and :data:`USER_BASE`. + it, respecting :data:`USER_BASE`. To determine if the user-specific + site-packages was added to ``sys.path`` :data:`ENABLE_USER_SITE` should be + used. .. versionadded:: 3.2 diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst index a88e358eae5fde..c1a20fee6cf024 100644 --- a/Doc/library/smtplib.rst +++ b/Doc/library/smtplib.rst @@ -115,7 +115,7 @@ Protocol) and :rfc:`1869` (SMTP Service Extensions). If the *timeout* parameter is set to be zero, it will raise a :class:`ValueError` to prevent the creation of a non-blocking socket -.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, +.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ source_address=None[, timeout]) The LMTP protocol, which is very similar to ESMTP, is heavily based on the diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index d798c1a9d10a05..26892c99268ba9 100755 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -56,12 +56,12 @@ created. Socket addresses are represented as follows: bytes-like object can be used for either type of address when passing it as an argument. - .. versionchanged:: 3.3 - Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 - encoding. + .. versionchanged:: 3.3 + Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 + encoding. - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. + .. versionchanged:: 3.5 + Writable :term:`bytes-like object` is now accepted. .. _host_port: @@ -1091,6 +1091,19 @@ The :mod:`socket` module also offers various network-related services: .. versionchanged:: 3.8 Windows support was added. + .. note:: + + On Windows network interfaces have different names in different contexts + (all names are examples): + + * UUID: ``{FB605B73-AAC2-49A6-9A2F-25416AEA0573}`` + * name: ``ethernet_32770`` + * friendly name: ``vEthernet (nat)`` + * description: ``Hyper-V Virtual Ethernet Adapter`` + + This function returns names of the second form from the list, ``ethernet_32770`` + in this example case. + .. function:: if_nametoindex(if_name) @@ -1105,6 +1118,9 @@ The :mod:`socket` module also offers various network-related services: .. versionchanged:: 3.8 Windows support was added. + .. seealso:: + "Interface name" is a name as documented in :func:`if_nameindex`. + .. function:: if_indextoname(if_index) @@ -1119,6 +1135,35 @@ The :mod:`socket` module also offers various network-related services: .. versionchanged:: 3.8 Windows support was added. + .. seealso:: + "Interface name" is a name as documented in :func:`if_nameindex`. + + +.. function:: send_fds(sock, buffers, fds[, flags[, address]]) + + Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. + The *fds* parameter is a sequence of file descriptors. + Consult :meth:`sendmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + + +.. function:: recv_fds(sock, bufsize, maxfds[, flags]) + + Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. + Return ``(msg, list(fds), flags, addr)``. + Consult :meth:`recvmsg` for the documentation of these parameters. + + .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. + + .. versionadded:: 3.9 + + .. note:: + + Any truncated integers at the end of the list of file descriptors. + .. _socket-objects: @@ -1614,29 +1659,6 @@ to sockets. .. versionadded:: 3.6 -.. method:: socket.send_fds(sock, buffers, fds[, flags[, address]]) - - Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket. - The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. - - .. availability:: Unix supporting :meth:`~socket.sendmsg` and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - -.. method:: socket.recv_fds(sock, bufsize, maxfds[, flags]) - - Receive up to *maxfds* file descriptors. Return ``(msg, list(fds), flags, addr)``. Consult - :meth:`recvmsg` for the documentation of these parameters. - - .. availability:: Unix supporting :meth:`~socket.recvmsg` and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - - .. note:: - - Any truncated integers at the end of the list of file descriptors. - .. method:: socket.sendfile(file, offset=0, count=None) Send a file until EOF is reached by using high-performance @@ -1695,7 +1717,9 @@ to sockets. .. method:: socket.setsockopt(level, optname, value: int) .. method:: socket.setsockopt(level, optname, value: buffer) + :noindex: .. method:: socket.setsockopt(level, optname, None, optlen: int) + :noindex: .. index:: module: struct diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index ccb82278bdaa13..4ae4b4dcff6767 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -25,61 +25,34 @@ represents the database. Here the data will be stored in the :file:`example.db` file:: import sqlite3 - conn = sqlite3.connect('example.db') + con = sqlite3.connect('example.db') You can also supply the special name ``:memory:`` to create a database in RAM. Once you have a :class:`Connection`, you can create a :class:`Cursor` object and call its :meth:`~Cursor.execute` method to perform SQL commands:: - c = conn.cursor() + cur = con.cursor() # Create table - c.execute('''CREATE TABLE stocks - (date text, trans text, symbol text, qty real, price real)''') + cur.execute('''CREATE TABLE stocks + (date text, trans text, symbol text, qty real, price real)''') # Insert a row of data - c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") + cur.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)") # Save (commit) the changes - conn.commit() + con.commit() # We can also close the connection if we are done with it. # Just be sure any changes have been committed or they will be lost. - conn.close() + con.close() The data you've saved is persistent and is available in subsequent sessions:: import sqlite3 - conn = sqlite3.connect('example.db') - c = conn.cursor() - -Usually your SQL operations will need to use values from Python variables. You -shouldn't assemble your query using Python's string operations because doing so -is insecure; it makes your program vulnerable to an SQL injection attack -(see https://xkcd.com/327/ for humorous example of what can go wrong). - -Instead, use the DB-API's parameter substitution. Put ``?`` as a placeholder -wherever you want to use a value, and then provide a tuple of values as the -second argument to the cursor's :meth:`~Cursor.execute` method. (Other database -modules may use a different placeholder, such as ``%s`` or ``:1``.) For -example:: - - # Never do this -- insecure! - symbol = 'RHAT' - c.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) - - # Do this instead - t = ('RHAT',) - c.execute('SELECT * FROM stocks WHERE symbol=?', t) - print(c.fetchone()) - - # Larger example that inserts many records at a time - purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00), - ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00), - ('2006-04-06', 'SELL', 'IBM', 500, 53.00), - ] - c.executemany('INSERT INTO stocks VALUES (?,?,?,?,?)', purchases) + con = sqlite3.connect('example.db') + cur = con.cursor() To retrieve data after executing a SELECT statement, you can either treat the cursor as an :term:`iterator`, call the cursor's :meth:`~Cursor.fetchone` method to @@ -88,7 +61,7 @@ matching rows. This example uses the iterator form:: - >>> for row in c.execute('SELECT * FROM stocks ORDER BY price'): + >>> for row in cur.execute('SELECT * FROM stocks ORDER BY price'): print(row) ('2006-01-05', 'BUY', 'RHAT', 100, 35.14) @@ -97,11 +70,34 @@ This example uses the iterator form:: ('2006-04-05', 'BUY', 'MSFT', 1000, 72.0) -.. seealso:: +.. _sqlite3-placeholders: - https://github.com/ghaering/pysqlite - The pysqlite web page -- sqlite3 is developed externally under the name - "pysqlite". +Usually your SQL operations will need to use values from Python variables. You +shouldn't assemble your query using Python's string operations because doing so +is insecure; it makes your program vulnerable to an SQL injection attack +(see the `xkcd webcomic `_ for a humorous example of +what can go wrong):: + + # Never do this -- insecure! + symbol = 'RHAT' + cur.execute("SELECT * FROM stocks WHERE symbol = '%s'" % symbol) + +Instead, use the DB-API's parameter substitution. Put a placeholder wherever +you want to use a value, and then provide a tuple of values as the second +argument to the cursor's :meth:`~Cursor.execute` method. An SQL statement may +use one of two kinds of placeholders: question marks (qmark style) or named +placeholders (named style). For the qmark style, ``parameters`` must be a +:term:`sequence `. For the named style, it can be either a +:term:`sequence ` or :class:`dict` instance. The length of the +:term:`sequence ` must match the number of placeholders, or a +:exc:`ProgrammingError` is raised. If a :class:`dict` is given, it must contain +keys for all named parameters. Any extra items are ignored. Here's an example +of both styles: + +.. literalinclude:: ../includes/sqlite3/execute_1.py + + +.. seealso:: https://www.sqlite.org The SQLite web page; the documentation describes the syntax and the @@ -197,7 +193,9 @@ Module functions and constants *detect_types* defaults to 0 (i. e. off, no type detection), you can set it to any combination of :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` to turn - type detection on. + type detection on. Due to SQLite behaviour, types can't be detected for generated + fields (for example ``max(data)``), even when *detect_types* parameter is set. In + such case, the returned type is :class:`str`. By default, *check_same_thread* is :const:`True` and only the creating thread may use the connection. If set :const:`False`, the returned connection may be shared @@ -543,7 +541,7 @@ Connection Objects con.close() - .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) This method makes a backup of a SQLite database even while it's being accessed by other clients, or concurrently by the same connection. The copy will be @@ -610,14 +608,8 @@ Cursor Objects .. method:: execute(sql[, parameters]) - Executes an SQL statement. The SQL statement may be parameterized (i. e. - placeholders instead of SQL literals). The :mod:`sqlite3` module supports two - kinds of placeholders: question marks (qmark style) and named placeholders - (named style). - - Here's an example of both styles: - - .. literalinclude:: ../includes/sqlite3/execute_1.py + Executes an SQL statement. Values may be bound to the statement using + :ref:`placeholders `. :meth:`execute` will only execute a single SQL statement. If you try to execute more than one statement with it, it will raise a :exc:`.Warning`. Use @@ -627,9 +619,10 @@ Cursor Objects .. method:: executemany(sql, seq_of_parameters) - Executes an SQL command against all parameter sequences or mappings found in - the sequence *seq_of_parameters*. The :mod:`sqlite3` module also allows - using an :term:`iterator` yielding parameters instead of a sequence. + Executes a :ref:`parameterized ` SQL command + against all parameter sequences or mappings found in the sequence + *seq_of_parameters*. The :mod:`sqlite3` module also allows using an + :term:`iterator` yielding parameters instead of a sequence. .. literalinclude:: ../includes/sqlite3/executemany_1.py @@ -642,7 +635,8 @@ Cursor Objects This is a nonstandard convenience method for executing multiple SQL statements at once. It issues a ``COMMIT`` statement first, then executes the SQL script it - gets as a parameter. + gets as a parameter. This method disregards :attr:`isolation_level`; any + transation control must be added to *sql_script*. *sql_script* can be an instance of :class:`str`. @@ -770,23 +764,23 @@ Row Objects Let's assume we initialize a table as in the example given above:: - conn = sqlite3.connect(":memory:") - c = conn.cursor() - c.execute('''create table stocks + con = sqlite3.connect(":memory:") + cur = con.cursor() + cur.execute('''create table stocks (date text, trans text, symbol text, qty real, price real)''') - c.execute("""insert into stocks - values ('2006-01-05','BUY','RHAT',100,35.14)""") - conn.commit() - c.close() + cur.execute("""insert into stocks + values ('2006-01-05','BUY','RHAT',100,35.14)""") + con.commit() + cur.close() Now we plug :class:`Row` in:: - >>> conn.row_factory = sqlite3.Row - >>> c = conn.cursor() - >>> c.execute('select * from stocks') + >>> con.row_factory = sqlite3.Row + >>> cur = con.cursor() + >>> cur.execute('select * from stocks') - >>> r = c.fetchone() + >>> r = cur.fetchone() >>> type(r) >>> tuple(r) @@ -1045,6 +1039,9 @@ setting :attr:`isolation_level` to ``None``. This will leave the underlying control the transaction state by explicitly issuing ``BEGIN``, ``ROLLBACK``, ``SAVEPOINT``, and ``RELEASE`` statements in your code. +Note that :meth:`~Cursor.executescript` disregards +:attr:`isolation_level`; any transaction control must be added explicitly. + .. versionchanged:: 3.6 :mod:`sqlite3` used to implicitly commit an open transaction before DDL statements. This is no longer the case. @@ -1091,23 +1088,10 @@ committed: .. literalinclude:: ../includes/sqlite3/ctx_manager.py -Common issues -------------- - -Multithreading -^^^^^^^^^^^^^^ - -Older SQLite versions had issues with sharing connections between threads. -That's why the Python module disallows sharing connections and cursors between -threads. If you still try to do so, you will get an exception at runtime. - -The only exception is calling the :meth:`~Connection.interrupt` method, which -only makes sense to call from a different thread. - .. rubric:: Footnotes .. [#f1] The sqlite3 module is not built with loadable extension support by default, because some platforms (notably Mac OS X) have SQLite libraries which are compiled without this feature. To get loadable - extension support, you must pass --enable-loadable-sqlite-extensions to + extension support, you must pass ``--enable-loadable-sqlite-extensions`` to configure. diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 952ee1653b9248..d673effc4b82c7 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -615,7 +615,7 @@ Constants Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the peer cert is checked but none of the intermediate CA certificates. The mode requires a valid CRL that is signed by the peer cert's issuer (its direct - ancestor CA). If no proper CRL has has been loaded with + ancestor CA). If no proper CRL has been loaded with :attr:`SSLContext.load_verify_locations`, validation will fail. .. versionadded:: 3.4 @@ -886,6 +886,14 @@ Constants .. versionadded:: 3.6 +.. data:: OP_IGNORE_UNEXPECTED_EOF + + Ignore unexpected shutdown of TLS connections. + + This option is only available with OpenSSL 3.0.0 and later. + + .. versionadded:: 3.10 + .. data:: HAS_ALPN Whether the OpenSSL library has built-in support for the *Application-Layer @@ -1878,7 +1886,7 @@ to speed up repeated connections from the same clients. .. attribute:: SSLContext.check_hostname - Whether to match the peer cert's hostname with :func:`match_hostname` in + Whether to match the peer cert's hostname in :meth:`SSLSocket.do_handshake`. The context's :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`, and you must pass *server_hostname* to @@ -2032,6 +2040,11 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.7 + .. versionchanged:: 3.9.3 + + The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, + 3.9.3, and 3.10 include workarounds for previous versions. + .. attribute:: SSLContext.verify_flags The flags for certificate verification operations. You can set flags like diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 4e7729c83f49a4..acaf99b7c81015 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -434,12 +434,10 @@ Notes: Negative shift counts are illegal and cause a :exc:`ValueError` to be raised. (2) - A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)`` - without overflow check. + A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)``. (3) - A right shift by *n* bits is equivalent to division by ``pow(2, n)`` without - overflow check. + A right shift by *n* bits is equivalent to floor division by ``pow(2, n)``. (4) Performing these calculations with at least one extra sign extension bit in @@ -480,7 +478,7 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.1 -.. method:: int.to_bytes(length, byteorder, \*, signed=False) +.. method:: int.to_bytes(length, byteorder, *, signed=False) Return an array of bytes representing an integer. @@ -512,7 +510,7 @@ class`. In addition, it provides a few more methods: .. versionadded:: 3.2 -.. classmethod:: int.from_bytes(bytes, byteorder, \*, signed=False) +.. classmethod:: int.from_bytes(bytes, byteorder, *, signed=False) Return the integer represented by the given array of bytes. @@ -1549,33 +1547,6 @@ expression support in the :mod:`re` module). interpreted as in slice notation. -.. method:: str.removeprefix(prefix, /) - - If the string starts with the *prefix* string, return - ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: - - >>> 'TestHook'.removeprefix('Test') - 'Hook' - >>> 'BaseTestCase'.removeprefix('Test') - 'BaseTestCase' - - .. versionadded:: 3.9 - -.. method:: str.removesuffix(suffix, /) - - If the string ends with the *suffix* string and that *suffix* is not empty, - return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: - - >>> 'MiscTests'.removesuffix('Tests') - 'Misc' - >>> 'TmpDirMixin'.removesuffix('Tests') - 'TmpDirMixin' - - .. versionadded:: 3.9 - - .. method:: str.encode(encoding="utf-8", errors="strict") Return an encoded version of the string as a bytes object. Default encoding @@ -1890,6 +1861,34 @@ expression support in the :mod:`re` module). the string itself, followed by two empty strings. +.. method:: str.removeprefix(prefix, /) + + If the string starts with the *prefix* string, return + ``string[len(prefix):]``. Otherwise, return a copy of the original + string:: + + >>> 'TestHook'.removeprefix('Test') + 'Hook' + >>> 'BaseTestCase'.removeprefix('Test') + 'BaseTestCase' + + .. versionadded:: 3.9 + + +.. method:: str.removesuffix(suffix, /) + + If the string ends with the *suffix* string and that *suffix* is not empty, + return ``string[:-len(suffix)]``. Otherwise, return a copy of the + original string:: + + >>> 'MiscTests'.removesuffix('Tests') + 'Misc' + >>> 'TmpDirMixin'.removesuffix('Tests') + 'TmpDirMixin' + + .. versionadded:: 3.9 + + .. method:: str.replace(old, new[, count]) Return a copy of the string with all occurrences of substring *old* replaced by @@ -3610,17 +3609,16 @@ Memory Views of an object that supports the :ref:`buffer protocol ` without copying. -.. class:: memoryview(obj) +.. class:: memoryview(object) - Create a :class:`memoryview` that references *obj*. *obj* must support the - buffer protocol. Built-in objects that support the buffer protocol include - :class:`bytes` and :class:`bytearray`. + Create a :class:`memoryview` that references *object*. *object* must + support the buffer protocol. Built-in objects that support the buffer + protocol include :class:`bytes` and :class:`bytearray`. A :class:`memoryview` has the notion of an *element*, which is the - atomic memory unit handled by the originating object *obj*. For many - simple types such as :class:`bytes` and :class:`bytearray`, an element - is a single byte, but other types such as :class:`array.array` may have - bigger elements. + atomic memory unit handled by the originating *object*. For many simple + types such as :class:`bytes` and :class:`bytearray`, an element is a single + byte, but other types such as :class:`array.array` may have bigger elements. ``len(view)`` is equal to the length of :class:`~memoryview.tolist`. If ``view.ndim = 0``, the length is 1. If ``view.ndim = 1``, the length @@ -4120,6 +4118,12 @@ The constructors for both classes work the same: objects. If *iterable* is not specified, a new empty set is returned. + Sets can be created by several means: + + * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` + * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` + * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` + Instances of :class:`set` and :class:`frozenset` provide the following operations: @@ -4312,6 +4316,14 @@ pairs within braces, for example: ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: Return a new dictionary initialized from an optional positional argument and a possibly empty set of keyword arguments. + Dictionaries can be created by several means: + + * Use a comma-separated list of ``key: value`` pairs within braces: + ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: 'jack', 4127: 'sjoerd'}`` + * Use a dict comprehension: ``{}``, ``{x: x ** 2 for x in range(10)}`` + * Use the type constructor: ``dict()``, + ``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)`` + If no positional argument is given, an empty dictionary is created. If a positional argument is given and it is a mapping object, a dictionary is created with the same key-value pairs as the mapping object. Otherwise, @@ -4718,6 +4730,200 @@ Compared to the overhead of setting up the runtime context, the overhead of a single class dictionary lookup is negligible. +.. _types-genericalias: + +Generic Alias Type +================== + +.. index:: + object: GenericAlias + pair: Generic; Alias + +``GenericAlias`` objects are created by subscripting a class (usually a +container), such as ``list[int]``. They are intended primarily for +:term:`type annotations `. + +Usually, the :ref:`subscription ` of container objects calls the +method :meth:`__getitem__` of the object. However, the subscription of some +containers' classes may call the classmethod :meth:`__class_getitem__` of the +class instead. The classmethod :meth:`__class_getitem__` should return a +``GenericAlias`` object. + +.. note:: + If the :meth:`__getitem__` of the class' metaclass is present, it will take + precedence over the :meth:`__class_getitem__` defined in the class (see + :pep:`560` for more details). + +The ``GenericAlias`` object acts as a proxy for :term:`generic types +`, implementing *parameterized generics* - a specific instance +of a generic which provides the types for container elements. + +The user-exposed type for the ``GenericAlias`` object can be accessed from +:class:`types.GenericAlias` and used for :func:`isinstance` checks. It can +also be used to create ``GenericAlias`` objects directly. + +.. describe:: T[X, Y, ...] + + Creates a ``GenericAlias`` representing a type ``T`` containing elements + of types *X*, *Y*, and more depending on the ``T`` used. + For example, a function expecting a :class:`list` containing + :class:`float` elements:: + + def average(values: list[float]) -> float: + return sum(values) / len(values) + + Another example for :term:`mapping` objects, using a :class:`dict`, which + is a generic type expecting two type parameters representing the key type + and the value type. In this example, the function expects a ``dict`` with + keys of type :class:`str` and values of type :class:`int`:: + + def send_post_request(url: str, body: dict[str, int]) -> None: + ... + +The builtin functions :func:`isinstance` and :func:`issubclass` do not accept +``GenericAlias`` types for their second argument:: + + >>> isinstance([1, 2], list[str]) + Traceback (most recent call last): + File "", line 1, in + TypeError: isinstance() argument 2 cannot be a parameterized generic + +The Python runtime does not enforce :term:`type annotations `. +This extends to generic types and their type parameters. When creating +an object from a ``GenericAlias``, container elements are not checked +against their type. For example, the following code is discouraged, but will +run without errors:: + + >>> t = list[str] + >>> t([1, 2, 3]) + [1, 2, 3] + +Furthermore, parameterized generics erase type parameters during object +creation:: + + >>> t = list[str] + >>> type(t) + + + >>> l = t() + >>> type(l) + + +Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: + + >>> repr(list[int]) + 'list[int]' + + >>> str(list[int]) + 'list[int]' + +The :meth:`__getitem__` method of generics will raise an exception to disallow +mistakes like ``dict[str][str]``:: + + >>> dict[str][str] + Traceback (most recent call last): + File "", line 1, in + TypeError: There are no type variables left in dict[str] + +However, such expressions are valid when :ref:`type variables ` are +used. The index must have as many elements as there are type variable items +in the ``GenericAlias`` object's :attr:`__args__ `. :: + + >>> from typing import TypeVar + >>> Y = TypeVar('Y') + >>> dict[str, Y][int] + dict[str, int] + + +Standard Generic Collections +---------------------------- + +These standard library collections support parameterized generics. + +* :class:`tuple` +* :class:`list` +* :class:`dict` +* :class:`set` +* :class:`frozenset` +* :class:`type` +* :class:`collections.deque` +* :class:`collections.defaultdict` +* :class:`collections.OrderedDict` +* :class:`collections.Counter` +* :class:`collections.ChainMap` +* :class:`collections.abc.Awaitable` +* :class:`collections.abc.Coroutine` +* :class:`collections.abc.AsyncIterable` +* :class:`collections.abc.AsyncIterator` +* :class:`collections.abc.AsyncGenerator` +* :class:`collections.abc.Iterable` +* :class:`collections.abc.Iterator` +* :class:`collections.abc.Generator` +* :class:`collections.abc.Reversible` +* :class:`collections.abc.Container` +* :class:`collections.abc.Collection` +* :class:`collections.abc.Callable` +* :class:`collections.abc.Set` +* :class:`collections.abc.MutableSet` +* :class:`collections.abc.Mapping` +* :class:`collections.abc.MutableMapping` +* :class:`collections.abc.Sequence` +* :class:`collections.abc.MutableSequence` +* :class:`collections.abc.ByteString` +* :class:`collections.abc.MappingView` +* :class:`collections.abc.KeysView` +* :class:`collections.abc.ItemsView` +* :class:`collections.abc.ValuesView` +* :class:`contextlib.AbstractContextManager` +* :class:`contextlib.AbstractAsyncContextManager` +* :ref:`re.Pattern ` +* :ref:`re.Match ` + + +Special Attributes of Generic Alias +----------------------------------- + +All parameterized generics implement special read-only attributes. + +.. attribute:: genericalias.__origin__ + + This attribute points at the non-parameterized generic class:: + + >>> list[int].__origin__ + + + +.. attribute:: genericalias.__args__ + + This attribute is a :class:`tuple` (possibly of length 1) of generic + types passed to the original :meth:`__class_getitem__` + of the generic container:: + + >>> dict[str, list[int]].__args__ + (, list[int]) + + +.. attribute:: genericalias.__parameters__ + + This attribute is a lazily computed tuple (possibly empty) of unique type + variables found in ``__args__``:: + + >>> from typing import TypeVar + + >>> T = TypeVar('T') + >>> list[T].__parameters__ + (~T,) + + +.. seealso:: + + * :pep:`585` -- "Type Hinting Generics In Standard Collections" + * :meth:`__class_getitem__` -- Used to implement parameterized generics. + * :ref:`generics` -- Generics in the :mod:`typing` module. + +.. versionadded:: 3.9 + + .. _typesother: Other Built-in Types @@ -4837,6 +5043,9 @@ environment. Code objects are returned by the built-in :func:`compile` function and can be extracted from function objects through their :attr:`__code__` attribute. See also the :mod:`code` module. +Accessing ``__code__`` raises an :ref:`auditing event ` +``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. + .. index:: builtin: exec builtin: eval @@ -4989,8 +5198,8 @@ types, where they are relevant. Some of these are not reported by the .. method:: class.__subclasses__ Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. - Example:: + method returns a list of all those references still alive. The list is in + definition order. Example:: >>> int.__subclasses__() [] diff --git a/Doc/library/string.rst b/Doc/library/string.rst index fa906f799c1082..d1bdf4347de192 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -188,8 +188,8 @@ Format String Syntax The :meth:`str.format` method and the :class:`Formatter` class share the same syntax for format strings (although in the case of :class:`Formatter`, subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but -there are differences. +related to that of :ref:`formatted string literals `, but it is +less sophisticated and, in particular, does not support arbitrary expressions. .. index:: single: {} (curly brackets); in string formatting @@ -205,7 +205,7 @@ literal text, it can be escaped by doubling: ``{{`` and ``}}``. The grammar for a replacement field is as follows: - .. productionlist:: sf + .. productionlist:: format-string replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* arg_name: [`identifier` | `digit`+] @@ -308,7 +308,7 @@ non-empty format specification typically modifies the result. The general form of a *standard format specifier* is: -.. productionlist:: sf +.. productionlist:: format-spec format_spec: [[`fill`]`align`][`sign`][#][0][`width`][`grouping_option`][.`precision`][`type`] fill: align: "<" | ">" | "=" | "^" @@ -384,10 +384,10 @@ following: The ``'#'`` option causes the "alternate form" to be used for the conversion. The alternate form is defined differently for different -types. This option is only valid for integer, float, complex and -Decimal types. For integers, when binary, octal, or hexadecimal output -is used, this option adds the prefix respective ``'0b'``, ``'0o'``, or -``'0x'`` to the output value. For floats, complex and Decimal the +types. This option is only valid for integer, float and complex +types. For integers, when binary, octal, or hexadecimal output +is used, this option adds the respective prefix ``'0b'``, ``'0o'``, +``'0x'``, or ``'0X'`` to the output value. For float and complex the alternate form causes the result of the conversion to always contain a decimal-point character, even if no digits follow it. Normally, a decimal-point character appears in the result of these conversions @@ -463,6 +463,8 @@ The available integer presentation types are: +---------+----------------------------------------------------------+ | ``'X'`` | Hex format. Outputs the number in base 16, using | | | upper-case letters for the digits above 9. | + | | In case ``'#'`` is specified, the prefix ``'0x'`` will | + | | be upper-cased to ``'0X'`` as well. | +---------+----------------------------------------------------------+ | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | | | the current locale setting to insert the appropriate | @@ -476,20 +478,36 @@ with the floating point presentation types listed below (except ``'n'`` and ``None``). When doing so, :func:`float` is used to convert the integer to a floating point number before formatting. -The available presentation types for floating point and decimal values are: +The available presentation types for :class:`float` and +:class:`~decimal.Decimal` values are: +---------+----------------------------------------------------------+ | Type | Meaning | +=========+==========================================================+ - | ``'e'`` | Exponent notation. Prints the number in scientific | - | | notation using the letter 'e' to indicate the exponent. | - | | The default precision is ``6``. | + | ``'e'`` | Scientific notation. For a given precision ``p``, | + | | formats the number in scientific notation with the | + | | letter 'e' separating the coefficient from the exponent. | + | | The coefficient has one digit before and ``p`` digits | + | | after the decimal point, for a total of ``p + 1`` | + | | significant digits. With no precision given, uses a | + | | precision of ``6`` digits after the decimal point for | + | | :class:`float`, and shows all coefficient digits | + | | for :class:`~decimal.Decimal`. If no digits follow the | + | | decimal point, the decimal point is also removed unless | + | | the ``#`` option is used. | +---------+----------------------------------------------------------+ - | ``'E'`` | Exponent notation. Same as ``'e'`` except it uses an | - | | upper case 'E' as the separator character. | + | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | + | | an upper case 'E' as the separator character. | +---------+----------------------------------------------------------+ - | ``'f'`` | Fixed-point notation. Displays the number as a | - | | fixed-point number. The default precision is ``6``. | + | ``'f'`` | Fixed-point notation. For a given precision ``p``, | + | | formats the number as a decimal number with exactly | + | | ``p`` digits following the decimal point. With no | + | | precision given, uses a precision of ``6`` digits after | + | | the decimal point for :class:`float`, and uses a | + | | precision large enough to show all coefficient digits | + | | for :class:`~decimal.Decimal`. If no digits follow the | + | | decimal point, the decimal point is also removed unless | + | | the ``#`` option is used. | +---------+----------------------------------------------------------+ | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | @@ -498,6 +516,8 @@ The available presentation types for floating point and decimal values are: | | this rounds the number to ``p`` significant digits and | | | then formats the result in either fixed-point format | | | or in scientific notation, depending on its magnitude. | + | | A precision of ``0`` is treated as equivalent to a | + | | precision of ``1``. | | | | | | The precise rules are as follows: suppose that the | | | result formatted with presentation type ``'e'`` and | @@ -512,13 +532,19 @@ The available presentation types for floating point and decimal values are: | | removed if there are no remaining digits following it, | | | unless the ``'#'`` option is used. | | | | + | | With no precision given, uses a precision of ``6`` | + | | significant digits for :class:`float`. For | + | | :class:`~decimal.Decimal`, the coefficient of the result | + | | is formed from the coefficient digits of the value; | + | | scientific notation is used for values smaller than | + | | ``1e-6`` in absolute value and values where the place | + | | value of the least significant digit is larger than 1, | + | | and fixed-point notation is used otherwise. | + | | | | | Positive and negative infinity, positive and negative | | | zero, and nans, are formatted as ``inf``, ``-inf``, | | | ``0``, ``-0`` and ``nan`` respectively, regardless of | | | the precision. | - | | | - | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. The default precision is ``6``. | +---------+----------------------------------------------------------+ | ``'G'`` | General format. Same as ``'g'`` except switches to | | | ``'E'`` if the number gets too large. The | @@ -531,12 +557,18 @@ The available presentation types for floating point and decimal values are: | ``'%'`` | Percentage. Multiplies the number by 100 and displays | | | in fixed (``'f'``) format, followed by a percent sign. | +---------+----------------------------------------------------------+ - | None | Similar to ``'g'``, except that fixed-point notation, | - | | when used, has at least one digit past the decimal point.| - | | The default precision is as high as needed to represent | - | | the particular value. The overall effect is to match the | - | | output of :func:`str` as altered by the other format | - | | modifiers. | + | None | For :class:`float` this is the same as ``'g'``, except | + | | that when fixed-point notation is used to format the | + | | result, it always includes at least one digit past the | + | | decimal point. The precision used is as large as needed | + | | to represent the given value faithfully. | + | | | + | | For :class:`~decimal.Decimal`, this is the same as | + | | either ``'g'`` or ``'G'`` depending on the value of | + | | ``context.capitals`` for the current decimal context. | + | | | + | | The overall effect is to match the output of :func:`str` | + | | as altered by the other format modifiers. | +---------+----------------------------------------------------------+ diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst index 856b6da8bb255d..eccba20fb8fe7e 100644 --- a/Doc/library/struct.rst +++ b/Doc/library/struct.rst @@ -159,8 +159,8 @@ the :ref:`format-characters` section. Note the difference between ``'@'`` and ``'='``: both use native byte order, but the size and alignment of the latter is standardized. -The form ``'!'`` is available for those poor souls who claim they can't remember -whether network byte order is big-endian or little-endian. +The form ``'!'`` represents the network byte order which is always big-endian +as defined in `IETF RFC 1700 `_. There is no way to indicate non-native byte order (force byte-swapping); use the appropriate choice of ``'<'`` or ``'>'``. @@ -467,3 +467,5 @@ The :mod:`struct` module also defines the following type: .. _half precision format: https://en.wikipedia.org/wiki/Half-precision_floating-point_format .. _ieee 754 standard: https://en.wikipedia.org/wiki/IEEE_floating_point#IEEE_754-2008 + +.. _IETF RFC 1700: https://tools.ietf.org/html/rfc1700 diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 5988bd35e72b12..762da9b1b4dca3 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -339,7 +339,7 @@ functions. stderr=None, preexec_fn=None, close_fds=True, shell=False, \ cwd=None, env=None, universal_newlines=None, \ startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=(), \*, group=None, \ + start_new_session=False, pass_fds=(), *, group=None, \ extra_groups=None, user=None, umask=-1, \ encoding=None, errors=None, text=None) @@ -738,10 +738,11 @@ Instances of the :class:`Popen` class have the following methods: .. method:: Popen.communicate(input=None, timeout=None) Interact with process: Send data to stdin. Read data from stdout and stderr, - until end-of-file is reached. Wait for process to terminate. The optional - *input* argument should be data to be sent to the child process, or - ``None``, if no data should be sent to the child. If streams were opened in - text mode, *input* must be a string. Otherwise, it must be bytes. + until end-of-file is reached. Wait for process to terminate and set the + :attr:`~Popen.returncode` attribute. The optional *input* argument should be + data to be sent to the child process, or ``None``, if no data should be sent + to the child. If streams were opened in text mode, *input* must be a string. + Otherwise, it must be bytes. :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. The data will be strings if streams were opened in text mode; otherwise, @@ -1162,8 +1163,9 @@ calls these functions. The arguments shown above are merely some common ones. The full function signature is largely the same as that of :func:`run` - most arguments are passed directly through to that interface. - However, explicitly passing ``input=None`` to inherit the parent's - standard input file handle is not supported. + One API deviation from :func:`run` behavior exists: passing ``input=None`` + will behave the same as ``input=b''`` (or ``input=''``, depending on other + arguments) rather than using the parent's standard input file handle. By default, this function will return the data as encoded bytes. The actual encoding of the output data may depend on the command being invoked, so the @@ -1266,11 +1268,17 @@ Replacing :func:`os.system` sts = os.system("mycmd" + " myarg") # becomes - sts = call("mycmd" + " myarg", shell=True) + retcode = call("mycmd" + " myarg", shell=True) Notes: * Calling the program through the shell is usually not required. +* The :func:`call` return value is encoded differently to that of + :func:`os.system`. + +* The :func:`os.system` function ignores SIGINT and SIGQUIT signals while + the command is running, but the caller must do this separately when + using the :mod:`subprocess` module. A more realistic example would look like this:: diff --git a/Doc/library/symbol.rst b/Doc/library/symbol.rst index 44996936e2d28f..d56600af29d6e2 100644 --- a/Doc/library/symbol.rst +++ b/Doc/library/symbol.rst @@ -17,6 +17,11 @@ the definitions of the names in the context of the language grammar. The specific numeric values which the names map to may change between Python versions. +.. warning:: + + The symbol module is deprecated and will be removed in future versions of + Python. + This module also provides one additional data object: diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst index 3efdecb5af7102..e364232247c200 100644 --- a/Doc/library/symtable.rst +++ b/Doc/library/symtable.rst @@ -156,6 +156,12 @@ Examining Symbol Tables Return ``True`` if the symbol is local to its block. + .. method:: is_annotated() + + Return ``True`` if the symbol is annotated. + + .. versionadded:: 3.6 + .. method:: is_free() Return ``True`` if the symbol is referenced in its block, but not assigned diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 98f63fb83f87a0..6b23b6f0ab74aa 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -26,21 +26,27 @@ always available. .. function:: addaudithook(hook) Append the callable *hook* to the list of active auditing hooks for the - current interpreter. + current (sub)interpreter. When an auditing event is raised through the :func:`sys.audit` function, each hook will be called in the order it was added with the event name and the tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are - called first, followed by hooks added in the current interpreter. + called first, followed by hooks added in the current (sub)interpreter. Hooks + can then log the event, raise an exception to abort the operation, + or terminate the process entirely. .. audit-event:: sys.addaudithook "" sys.addaudithook - Raise an auditing event ``sys.addaudithook`` with no arguments. If any + Calling :func:`sys.addaudithook` will itself raise an auditing event + named ``sys.addaudithook`` with no arguments. If any existing hooks raise an exception derived from :class:`RuntimeError`, the new hook will not be added and the exception suppressed. As a result, callers cannot assume that their hook has been added unless they control all existing hooks. + See the :ref:`audit events table ` for all events raised by + CPython, and :pep:`578` for the original design discussion. + .. versionadded:: 3.8 .. versionchanged:: 3.8.1 @@ -79,14 +85,23 @@ always available. .. index:: single: auditing - Raise an auditing event with any active hooks. The event name is a string - identifying the event and its associated schema, which is the number and - types of arguments. The schema for a given event is considered public and - stable API and should not be modified between releases. + Raise an auditing event and trigger any active auditing hooks. + *event* is a string identifying the event, and *args* may contain + optional arguments with more information about the event. The + number and types of arguments for a given event are considered a + public and stable API and should not be modified between releases. + + For example, one auditing event is named ``os.chdir``. This event has + one argument called *path* that will contain the requested new + working directory. - This function will raise the first exception raised by any hook. In general, - these errors should not be handled and should terminate the process as - quickly as possible. + :func:`sys.audit` will call the existing auditing hooks, passing + the event name and arguments, and will re-raise the first exception + from any hook. In general, if an exception is raised, it should not + be handled and the process should be terminated as quickly as + possible. This allows hook implementations to decide how to respond + to particular events: they can merely log the event or abort the + operation by raising an exception. Hooks are added using the :func:`sys.addaudithook` or :c:func:`PySys_AddAuditHook` functions. @@ -762,11 +777,16 @@ always available. Microsoft documentation on :c:func:`OSVERSIONINFOEX` for more information about these fields. - *platform_version* returns the accurate major version, minor version and + *platform_version* returns the major version, minor version and build number of the current operating system, rather than the version that is being emulated for the process. It is intended for use in logging rather than for feature detection. + .. note:: + *platform_version* derives the version from kernel32.dll which can be of a different + version than the OS version. Please use :mod:`platform` module for achieving accurate + OS version. + .. availability:: Windows. .. versionchanged:: 3.2 @@ -1141,8 +1161,7 @@ always available. .. data:: platlibdir Name of the platform-specific library directory. It is used to build the - path of platform-specific dynamic libraries and the path of the standard - library. + path of standard library and the paths of installed extension modules. It is equal to ``"lib"`` on most platforms. On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit platforms which gives the following ``sys.path`` @@ -1153,8 +1172,10 @@ always available. * ``/usr/lib64/pythonX.Y/lib-dynload/``: C extension modules of the standard library (like the :mod:`errno` module, the exact filename is platform specific) - * ``/usr/lib/pythonX.Y/site-packages`` (always use ``lib``, not + * ``/usr/lib/pythonX.Y/site-packages/`` (always use ``lib``, not :data:`sys.platlibdir`): Third-party modules + * ``/usr/lib64/pythonX.Y/site-packages/``: + C extension modules of third-party packages .. versionadded:: 3.9 diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst index 78a1dfce9ae05c..c9306e9bf9de16 100644 --- a/Doc/library/sysconfig.rst +++ b/Doc/library/sysconfig.rst @@ -32,7 +32,7 @@ can be accessed using :func:`get_config_vars` or :func:`get_config_var`. Notice that on Windows, it's a much smaller set. -.. function:: get_config_vars(\*args) +.. function:: get_config_vars(*args) With no arguments, return a dictionary of all configuration variables relevant for the current platform. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 459e4ad991d9dc..13088a10d77c57 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -37,7 +37,7 @@ Some facts and figures: Added support for :mod:`lzma` compression. -.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, \*\*kwargs) +.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) Return a :class:`TarFile` object for the pathname *name*. For detailed information on :class:`TarFile` objects and the keyword arguments that are @@ -151,6 +151,7 @@ Some facts and figures: .. class:: TarFile + :noindex: Class for reading and writing tar archives. Do not use this class directly: use :func:`tarfile.open` instead. See :ref:`tarfile-objects`. @@ -444,10 +445,11 @@ be finalized; only the internally used file object will be closed. See the .. method:: TarFile.extractfile(member) - Extract a member from the archive as a file object. *member* may be a filename - or a :class:`TarInfo` object. If *member* is a regular file or a link, an - :class:`io.BufferedReader` object is returned. Otherwise, :const:`None` is - returned. + Extract a member from the archive as a file object. *member* may be + a filename or a :class:`TarInfo` object. If *member* is a regular file or + a link, an :class:`io.BufferedReader` object is returned. For all other + existing members, :const:`None` is returned. If *member* does not appear + in the archive, :exc:`KeyError` is raised. .. versionchanged:: 3.3 Return an :class:`io.BufferedReader` object. @@ -787,7 +789,7 @@ How to read a gzip compressed tar archive and display some member information:: import tarfile tar = tarfile.open("sample.tar.gz", "r:gz") for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is", end="") + print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") if tarinfo.isreg(): print("a regular file.") elif tarinfo.isdir(): diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst index a59817c1039210..f9421da5fe7dfa 100644 --- a/Doc/library/tempfile.rst +++ b/Doc/library/tempfile.rst @@ -31,7 +31,7 @@ is recommended to use keyword arguments for clarity. The module defines the following user-callable items: -.. function:: TemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) +.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) Return a :term:`file-like object` that can be used as a temporary storage area. The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon @@ -72,7 +72,7 @@ The module defines the following user-callable items: Added *errors* parameter. -.. function:: NamedTemporaryFile(mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) +.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) This function operates exactly as :func:`TemporaryFile` does, except that the file is guaranteed to have a visible name in the file system (on @@ -93,7 +93,7 @@ The module defines the following user-callable items: Added *errors* parameter. -.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=None, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) +.. function:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) This function operates exactly as :func:`TemporaryFile` does, except that data is spooled in memory until the file size exceeds *max_size*, or @@ -175,9 +175,8 @@ The module defines the following user-callable items: If you want to force a bytes return value with otherwise default behavior, pass ``suffix=b''``. - If *text* is specified, it indicates whether to open the file in binary - mode (the default) or text mode. On some platforms, this makes no - difference. + If *text* is specified and true, the file is opened in text mode. + Otherwise, (the default) the file is opened in binary mode. :func:`mkstemp` returns a tuple containing an OS-level handle to an open file (as would be returned by :func:`os.open`) and the absolute pathname diff --git a/Doc/library/test.rst b/Doc/library/test.rst index f7e6eba0181614..bb1bd29bf698ed 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -546,7 +546,7 @@ The :mod:`test.support` module defines the following functions: Define match test with regular expression *patterns*. -.. function:: run_unittest(\*classes) +.. function:: run_unittest(*classes) Execute :class:`unittest.TestCase` subclasses passed to the function. The function scans the classes for methods starting with the prefix ``test_`` @@ -1609,6 +1609,8 @@ script execution tests. The :mod:`test.support.bytecode_helper` module provides support for testing and inspecting bytecode generation. +.. versionadded:: 3.9 + The module defines the following class: .. class:: BytecodeTestCase(unittest.TestCase) diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 3a446adfac8c5a..894bbb11da9152 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -21,6 +21,19 @@ level :mod:`_thread` module. See also the :mod:`queue` module. supported by this module. +.. impl-detail:: + + In CPython, due to the :term:`Global Interpreter Lock + `, only one thread + can execute Python code at once (even though certain performance-oriented + libraries might overcome this limitation). + If you want your application to make better use of the computational + resources of multi-core machines, you are advised to use + :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. + However, threading is still an appropriate model if you want to run + multiple I/O-bound tasks simultaneously. + + This module defines the following functions: @@ -97,10 +110,11 @@ This module defines the following functions: .. function:: enumerate() - Return a list of all :class:`Thread` objects currently alive. The list - includes daemonic threads, dummy thread objects created by - :func:`current_thread`, and the main thread. It excludes terminated threads - and threads that have not yet been started. + Return a list of all :class:`Thread` objects currently active. The list + includes daemonic threads and dummy thread objects created by + :func:`current_thread`. It excludes terminated threads and threads + that have not yet been started. However, the main thread is always part + of the result, even when terminated. .. function:: main_thread() @@ -393,18 +407,6 @@ since it is impossible to detect the termination of alien threads. property instead. -.. impl-detail:: - - In CPython, due to the :term:`Global Interpreter Lock`, only one thread - can execute Python code at once (even though certain performance-oriented - libraries might overcome this limitation). - If you want your application to make better use of the computational - resources of multi-core machines, you are advised to use - :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. - However, threading is still an appropriate model if you want to run - multiple I/O-bound tasks simultaneously. - - .. _lock-objects: Lock Objects diff --git a/Doc/library/time.rst b/Doc/library/time.rst index cff6320b526db5..11aba5dd842c56 100644 --- a/Doc/library/time.rst +++ b/Doc/library/time.rst @@ -271,7 +271,7 @@ Functions Return the value (in fractional seconds) of a monotonic clock, i.e. a clock that cannot go backwards. The clock is not affected by system clock updates. The reference point of the returned value is undefined, so that only the - difference between the results of consecutive calls is valid. + difference between the results of two calls is valid. .. versionadded:: 3.3 .. versionchanged:: 3.5 @@ -293,7 +293,7 @@ Functions clock with the highest available resolution to measure a short duration. It does include time elapsed during sleep and is system-wide. The reference point of the returned value is undefined, so that only the difference between - the results of consecutive calls is valid. + the results of two calls is valid. .. versionadded:: 3.3 @@ -315,7 +315,7 @@ Functions CPU time of the current process. It does not include time elapsed during sleep. It is process-wide by definition. The reference point of the returned value is undefined, so that only the difference between the results - of consecutive calls is valid. + of two calls is valid. .. versionadded:: 3.3 @@ -593,7 +593,7 @@ Functions CPU time of the current thread. It does not include time elapsed during sleep. It is thread-specific by definition. The reference point of the returned value is undefined, so that only the difference between the results - of consecutive calls in the same thread is valid. + of two calls in the same thread is valid. .. availability:: Windows, Linux, Unix systems supporting ``CLOCK_THREAD_CPUTIME_ID``. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 46fa62c15fc2ef..d4e8b749db4808 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -15,8 +15,8 @@ This module provides a simple way to time small bits of Python code. It has both a :ref:`timeit-command-line-interface` as well as a :ref:`callable ` one. It avoids a number of common traps for measuring execution times. -See also Tim Peters' introduction to the "Algorithms" chapter in the *Python -Cookbook*, published by O'Reilly. +See also Tim Peters' introduction to the "Algorithms" chapter in the second +edition of *Python Cookbook*, published by O'Reilly. Basic Examples @@ -233,7 +233,7 @@ Where the following options are understood: .. cmdoption:: -u, --unit=U - specify a time unit for timer output; can select nsec, usec, msec, or sec + specify a time unit for timer output; can select nsec, usec, msec, or sec .. versionadded:: 3.5 diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst index 30c1e7b5f9eb43..b0f4505e9e3c69 100644 --- a/Doc/library/tkinter.font.rst +++ b/Doc/library/tkinter.font.rst @@ -38,8 +38,8 @@ The different font weights and slants are: | *family* - font family i.e. Courier, Times | *size* - font size | If *size* is positive it is interpreted as size in points. - | If *size* is a negative number its absolute value is treated as - as size in pixels. + | If *size* is a negative number its absolute value is treated + | as size in pixels. | *weight* - font emphasis (NORMAL, BOLD) | *slant* - ROMAN, ITALIC | *underline* - font underlining (0 - none, 1 - underline) diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst index 2dc44ad36a7f73..7739f2f60a7980 100644 --- a/Doc/library/tkinter.rst +++ b/Doc/library/tkinter.rst @@ -31,7 +31,7 @@ installed, so you can read the Tcl/Tk documentation specific to that version. `TKDocs `_ Extensive tutorial plus friendlier widget pages for some of the widgets. - `Tkinter 8.5 reference: a GUI for Python `_ + `Tkinter 8.5 reference: a GUI for Python `_ On-line reference material. `Tkinter docs from effbot `_ @@ -464,12 +464,11 @@ The Packer .. index:: single: packing (widgets) The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of the positioning of widgets -within their container - their mutual *master*. In contrast to the more -cumbersome *placer* (which is used less commonly, and we do not cover here), the -packer takes qualitative relationship specification - *above*, *to the left of*, -*filling*, etc - and works everything out to determine the exact placement -coordinates for you. +are used to specify the relative positioning of widgets within their container - +their mutual *master*. In contrast to the more cumbersome *placer* (which is +used less commonly, and we do not cover here), the packer takes qualitative +relationship specification - *above*, *to the left of*, *filling*, etc - and +works everything out to determine the exact placement coordinates for you. The size of any *master* widget is determined by the size of the "slave widgets" inside. The packer is used to control where slave widgets appear inside the @@ -542,31 +541,35 @@ the variable, with no further intervention on your part. For example:: - class App(Frame): - def __init__(self, master=None): + import tkinter as tk + + class App(tk.Frame): + def __init__(self, master): super().__init__(master) self.pack() - self.entrythingy = Entry() + self.entrythingy = tk.Entry() self.entrythingy.pack() - # here is the application variable - self.contents = StringVar() - # set it to some value + # Create the application variable. + self.contents = tk.StringVar() + # Set it to some value. self.contents.set("this is a variable") - # tell the entry widget to watch this variable + # Tell the entry widget to watch this variable. self.entrythingy["textvariable"] = self.contents - # and here we get a callback when the user hits return. - # we will have the program print out the value of the - # application variable when the user hits return + # Define a callback for when the user hits return. + # It prints the current value of the variable. self.entrythingy.bind('', - self.print_contents) + self.print_contents) def print_contents(self, event): - print("hi. contents of entry is now ---->", + print("Hi. The current entry content is:", self.contents.get()) + root = tk.Tk() + myapp = App(root) + myapp.mainloop() The Window Manager ^^^^^^^^^^^^^^^^^^ @@ -861,4 +864,4 @@ use raw reads or ``os.read(file.fileno(), maxbytecount)``. WRITABLE EXCEPTION - Constants used in the *mask* arguments. \ No newline at end of file + Constants used in the *mask* arguments. diff --git a/Doc/library/token.rst b/Doc/library/token.rst index dab8f0fa9b64fc..7f598cd38d7f8c 100644 --- a/Doc/library/token.rst +++ b/Doc/library/token.rst @@ -70,6 +70,7 @@ the :mod:`tokenize` module. .. data:: TYPE_COMMENT + :noindex: Token value indicating that a type comment was recognized. Such tokens are only produced when :func:`ast.parse()` is invoked with diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst index c2732d900bc138..40cf198f1287d7 100644 --- a/Doc/library/trace.rst +++ b/Doc/library/trace.rst @@ -153,47 +153,47 @@ Programmatic Interface count information. *timing* enables a timestamp relative to when tracing was started to be displayed. - .. method:: run(cmd) + .. method:: run(cmd) - Execute the command and gather statistics from the execution with - the current tracing parameters. *cmd* must be a string or code object, - suitable for passing into :func:`exec`. + Execute the command and gather statistics from the execution with + the current tracing parameters. *cmd* must be a string or code object, + suitable for passing into :func:`exec`. - .. method:: runctx(cmd, globals=None, locals=None) + .. method:: runctx(cmd, globals=None, locals=None) - Execute the command and gather statistics from the execution with the - current tracing parameters, in the defined global and local - environments. If not defined, *globals* and *locals* default to empty - dictionaries. + Execute the command and gather statistics from the execution with the + current tracing parameters, in the defined global and local + environments. If not defined, *globals* and *locals* default to empty + dictionaries. - .. method:: runfunc(func, /, *args, **kwds) + .. method:: runfunc(func, /, *args, **kwds) - Call *func* with the given arguments under control of the :class:`Trace` - object with the current tracing parameters. + Call *func* with the given arguments under control of the :class:`Trace` + object with the current tracing parameters. - .. method:: results() + .. method:: results() - Return a :class:`CoverageResults` object that contains the cumulative - results of all previous calls to ``run``, ``runctx`` and ``runfunc`` - for the given :class:`Trace` instance. Does not reset the accumulated - trace results. + Return a :class:`CoverageResults` object that contains the cumulative + results of all previous calls to ``run``, ``runctx`` and ``runfunc`` + for the given :class:`Trace` instance. Does not reset the accumulated + trace results. .. class:: CoverageResults A container for coverage results, created by :meth:`Trace.results`. Should not be created directly by the user. - .. method:: update(other) + .. method:: update(other) - Merge in data from another :class:`CoverageResults` object. + Merge in data from another :class:`CoverageResults` object. - .. method:: write_results(show_missing=True, summary=False, coverdir=None) + .. method:: write_results(show_missing=True, summary=False, coverdir=None) - Write coverage results. Set *show_missing* to show lines that had no - hits. Set *summary* to include in the output the coverage summary per - module. *coverdir* specifies the directory into which the coverage - result files will be output. If ``None``, the results for each source - file are placed in its directory. + Write coverage results. Set *show_missing* to show lines that had no + hits. Set *summary* to include in the output the coverage summary per + module. *coverdir* specifies the directory into which the coverage + result files will be output. If ``None``, the results for each source + file are placed in its directory. A simple example demonstrating the use of the programmatic interface:: diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst index 3eee9457fb29a8..20f668c7282028 100644 --- a/Doc/library/tracemalloc.rst +++ b/Doc/library/tracemalloc.rst @@ -249,6 +249,47 @@ Example of output of the Python test suite:: See :meth:`Snapshot.statistics` for more options. +Record the current and peak size of all traced memory blocks +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The following code computes two sums like ``0 + 1 + 2 + ...`` inefficiently, by +creating a list of those numbers. This list consumes a lot of memory +temporarily. We can use :func:`get_traced_memory` and :func:`reset_peak` to +observe the small memory usage after the sum is computed as well as the peak +memory usage during the computations:: + + import tracemalloc + + tracemalloc.start() + + # Example code: compute a sum with a large temporary list + large_sum = sum(list(range(100000))) + + first_size, first_peak = tracemalloc.get_traced_memory() + + tracemalloc.reset_peak() + + # Example code: compute a sum with a small temporary list + small_sum = sum(list(range(1000))) + + second_size, second_peak = tracemalloc.get_traced_memory() + + print(f"{first_size=}, {first_peak=}") + print(f"{second_size=}, {second_peak=}") + +Output:: + + first_size=664, first_peak=3592984 + second_size=804, second_peak=29704 + +Using :func:`reset_peak` ensured we could accurately record the peak during the +computation of ``small_sum``, even though it is much smaller than the overall +peak size of memory blocks since the :func:`start` call. Without the call to +:func:`reset_peak`, ``second_peak`` would still be the peak from the +computation ``large_sum`` (that is, equal to ``first_peak``). In this case, +both peaks are much higher than the final memory usage, and which suggests we +could optimise (by removing the unnecessary call to :class:`list`, and writing +``sum(range(...))``). API --- @@ -289,6 +330,24 @@ Functions :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. +.. function:: reset_peak() + + Set the peak size of memory blocks traced by the :mod:`tracemalloc` module + to the current size. + + Do nothing if the :mod:`tracemalloc` module is not tracing memory + allocations. + + This function only modifies the recorded peak size, and does not modify or + clear any traces, unlike :func:`clear_traces`. Snapshots taken with + :func:`take_snapshot` before a call to :func:`reset_peak` can be + meaningfully compared to snapshots taken after the call. + + See also :func:`get_traced_memory`. + + .. versionadded:: 3.9 + + .. function:: get_tracemalloc_memory() Get the memory usage in bytes of the :mod:`tracemalloc` module used to store diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst index fed85045435b1b..6a9d61916ad1a5 100644 --- a/Doc/library/turtle.rst +++ b/Doc/library/turtle.rst @@ -662,7 +662,7 @@ Tell Turtle's state Return the angle between the line from turtle position to position specified by (x,y), the vector or the other turtle. This depends on the turtle's start - orientation which depends on the mode - "standard"/"world" or "logo"). + orientation which depends on the mode - "standard"/"world" or "logo". .. doctest:: :skipif: _tkinter is None @@ -913,8 +913,8 @@ Color control Set pencolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the outline of that polygon is drawn with the - newly set pencolor. + If turtleshape is a polygon, the outline of that polygon is drawn with the + newly set pencolor. .. doctest:: :skipif: _tkinter is None @@ -962,8 +962,8 @@ Color control Set fillcolor to the RGB color represented by *r*, *g*, and *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode. - If turtleshape is a polygon, the interior of that polygon is drawn - with the newly set fillcolor. + If turtleshape is a polygon, the interior of that polygon is drawn + with the newly set fillcolor. .. doctest:: :skipif: _tkinter is None @@ -1001,8 +1001,8 @@ Color control Equivalent to ``pencolor(colorstring1)`` and ``fillcolor(colorstring2)`` and analogously if the other input format is used. - If turtleshape is a polygon, outline and interior of that polygon is drawn - with the newly set colors. + If turtleshape is a polygon, outline and interior of that polygon is drawn + with the newly set colors. .. doctest:: :skipif: _tkinter is None @@ -1069,6 +1069,7 @@ More drawing control ~~~~~~~~~~~~~~~~~~~~ .. function:: reset() + :noindex: Delete the turtle's drawings from the screen, re-center the turtle and set variables to the default values. @@ -1090,6 +1091,7 @@ More drawing control .. function:: clear() + :noindex: Delete the turtle's drawings from the screen. Do not move turtle. State and position of the turtle as well as drawings of other turtles are not affected. @@ -1103,7 +1105,7 @@ More drawing control :param font: a triple (fontname, fontsize, fonttype) Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or right") and with the given + position according to *align* ("left", "center" or "right") and with the given font. If *move* is true, the pen is moved to the bottom-right corner of the text. By default, *move* is ``False``. @@ -1190,7 +1192,7 @@ Appearance :func:`shapesize`. - "noresize": no adaption of the turtle's appearance takes place. - resizemode("user") is called by :func:`shapesize` when used with arguments. + ``resizemode("user")`` is called by :func:`shapesize` when used with arguments. .. doctest:: :skipif: _tkinter is None @@ -1328,7 +1330,7 @@ Appearance matrix as a tuple of 4 elements. Otherwise set the given elements and transform the turtleshape according to the matrix consisting of first row t11, t12 and - second row t21, 22. The determinant t11 * t22 - t12 * t21 must not be + second row t21, t22. The determinant t11 * t22 - t12 * t21 must not be zero, otherwise an error is raised. Modify stretchfactor, shearfactor and tiltangle according to the given matrix. @@ -1362,6 +1364,7 @@ Using events ------------ .. function:: onclick(fun, btn=1, add=None) + :noindex: :param fun: a function with two arguments which will be called with the coordinates of the clicked point on the canvas @@ -1510,7 +1513,7 @@ Special Turtle methods :param size: an integer or ``None`` - Set or disable undobuffer. If *size* is an integer an empty undobuffer of + Set or disable undobuffer. If *size* is an integer, an empty undobuffer of given size is installed. *size* gives the maximum number of turtle actions that can be undone by the :func:`undo` method/function. If *size* is ``None``, the undobuffer is disabled. @@ -1818,7 +1821,7 @@ Using screen events existing bindings are removed. Example for a TurtleScreen instance named ``screen`` and a Turtle instance - named turtle: + named ``turtle``: .. doctest:: :skipif: _tkinter is None @@ -2045,7 +2048,7 @@ Methods specific to Screen, not inherited from TurtleScreen .. function:: exitonclick() - Bind bye() method to mouse clicks on the Screen. + Bind ``bye()`` method to mouse clicks on the Screen. If the value "using_IDLE" in the configuration dictionary is ``False`` diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 79acdf4499afd2..81a2b7b9879706 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -34,7 +34,7 @@ Dynamic Type Creation freshly created class namespace. It should accept the class namespace as its sole argument and update the namespace directly with the class contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: ns``. + in ``lambda ns: None``. .. versionadded:: 3.3 @@ -109,6 +109,11 @@ Standard names are defined for the following types: The type of user-defined functions and functions created by :keyword:`lambda` expressions. + .. audit-event:: function.__new__ code types.FunctionType + + The audit event only occurs for direct instantiation of function objects, + and is not raised for normal compilation. + .. data:: GeneratorType @@ -138,10 +143,11 @@ Standard names are defined for the following types: The type for code objects such as returned by :func:`compile`. - .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags CodeType + .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType Note that the audited arguments may not match the names or positions - required by the initializer. + required by the initializer. The audit event only occurs for direct + instantiation of code objects, and is not raised for normal compilation. .. method:: CodeType.replace(**kwargs) @@ -203,7 +209,7 @@ Standard names are defined for the following types: .. class:: ModuleType(name, doc=None) - The type of :term:`modules `. Constructor takes the name of the + The type of :term:`modules `. The constructor takes the name of the module to be created and optionally its :term:`docstring`. .. note:: @@ -218,12 +224,23 @@ Standard names are defined for the following types: The :term:`loader` which loaded the module. Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__loader__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. .. attribute:: __name__ - The name of the module. + The name of the module. Expected to match + :attr:`importlib.machinery.ModuleSpec.name`. .. attribute:: __package__ @@ -232,9 +249,48 @@ Standard names are defined for the following types: to ``''``, else it should be set to the name of the package (which can be :attr:`__name__` if the module is a package itself). Defaults to ``None``. + This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` + as stored in the attr:`__spec__` object. + + .. note:: + A future version of Python may stop setting this attribute by default. + To guard against this potential change, preferrably read from the + :attr:`__spec__` attribute instead or use + ``getattr(module, "__package__", None)`` if you explicitly need to use + this attribute. + .. versionchanged:: 3.4 Defaults to ``None``. Previously the attribute was optional. + .. attribute:: __spec__ + + A record of the module's import-system-related state. Expected to be an + instance of :class:`importlib.machinery.ModuleSpec`. + + .. versionadded:: 3.4 + + +.. class:: GenericAlias(t_origin, t_args) + + The type of :ref:`parameterized generics ` such as + ``list[int]``. + + ``t_origin`` should be a non-parameterized generic class, such as ``list``, + ``tuple`` or ``dict``. ``t_args`` should be a :class:`tuple` (possibly of + length 1) of types which parameterize ``t_origin``:: + + >>> from types import GenericAlias + + >>> list[int] == GenericAlias(list, (int,)) + True + >>> dict[str, int] == GenericAlias(dict, (str, int)) + True + + .. versionadded:: 3.9 + + .. versionchanged:: 3.9.2 + This type can now be subclassed. + .. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) @@ -359,7 +415,9 @@ Additional Utility Classes and Functions return "{}({})".format(type(self).__name__, ", ".join(items)) def __eq__(self, other): - return self.__dict__ == other.__dict__ + if isinstance(self, SimpleNamespace) and isinstance(other, SimpleNamespace): + return self.__dict__ == other.__dict__ + return NotImplemented ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. However, for a structured record type use :func:`~collections.namedtuple` diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst index fa13c07c44ea3a..78ab760d86a547 100644 --- a/Doc/library/typing.rst +++ b/Doc/library/typing.rst @@ -1,3 +1,4 @@ +======================================== :mod:`typing` --- Support for type hints ======================================== @@ -34,13 +35,12 @@ In the function ``greeting``, the argument ``name`` is expected to be of type arguments. Type aliases ------------- +============ A type alias is defined by assigning the type to the alias. In this example, -``Vector`` and ``List[float]`` will be treated as interchangeable synonyms:: +``Vector`` and ``list[float]`` will be treated as interchangeable synonyms:: - from typing import List - Vector = List[float] + Vector = list[float] def scale(scalar: float, vector: Vector) -> Vector: return [scalar * num for num in vector] @@ -50,11 +50,11 @@ A type alias is defined by assigning the type to the alias. In this example, Type aliases are useful for simplifying complex type signatures. For example:: - from typing import Dict, Tuple, Sequence + from collections.abc import Sequence - ConnectionOptions = Dict[str, str] - Address = Tuple[str, int] - Server = Tuple[Address, ConnectionOptions] + ConnectionOptions = dict[str, str] + Address = tuple[str, int] + Server = tuple[Address, ConnectionOptions] def broadcast_message(message: str, servers: Sequence[Server]) -> None: ... @@ -63,7 +63,7 @@ Type aliases are useful for simplifying complex type signatures. For example:: # being exactly equivalent to this one. def broadcast_message( message: str, - servers: Sequence[Tuple[Tuple[str, int], Dict[str, str]]]) -> None: + servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: ... Note that ``None`` as a type hint is a special case and is replaced by @@ -72,7 +72,7 @@ Note that ``None`` as a type hint is a special case and is replaced by .. _distinct: NewType -------- +======= Use the :func:`NewType` helper function to create distinct types:: @@ -149,14 +149,14 @@ See :pep:`484` for more details. .. versionadded:: 3.5.2 Callable --------- +======== Frameworks expecting callback functions of specific signatures might be type hinted using ``Callable[[Arg1Type, Arg2Type], ReturnType]``. For example:: - from typing import Callable + from collections.abc import Callable def feeder(get_next_item: Callable[[], str]) -> None: # Body @@ -172,7 +172,7 @@ for the list of arguments in the type hint: ``Callable[..., ReturnType]``. .. _generics: Generics --------- +======== Since type information about objects kept in containers cannot be statically inferred in a generic way, abstract base classes have been extended to support @@ -180,7 +180,7 @@ subscription to denote expected types for container elements. :: - from typing import Mapping, Sequence + from collections.abc import Mapping, Sequence def notify_by_email(employees: Sequence[Employee], overrides: Mapping[str, str]) -> None: ... @@ -190,7 +190,8 @@ called :class:`TypeVar`. :: - from typing import Sequence, TypeVar + from collections.abc import Sequence + from typing import TypeVar T = TypeVar('T') # Declare type variable @@ -199,7 +200,7 @@ called :class:`TypeVar`. User-defined generic types --------------------------- +========================== A user-defined class can be defined as a generic class. @@ -234,7 +235,7 @@ class body. The :class:`Generic` base class defines :meth:`__class_getitem__` so that ``LoggedVar[t]`` is valid as a type:: - from typing import Iterable + from collections.abc import Iterable def zero_all_vars(vars: Iterable[LoggedVar[int]]) -> None: for var in vars: @@ -265,7 +266,8 @@ This is thus invalid:: You can use multiple inheritance with :class:`Generic`:: - from typing import TypeVar, Generic, Sized + from collections.abc import Sized + from typing import TypeVar, Generic T = TypeVar('T') @@ -274,7 +276,8 @@ You can use multiple inheritance with :class:`Generic`:: When inheriting from generic classes, some type variables could be fixed:: - from typing import TypeVar, Mapping + from collections.abc import Mapping + from typing import TypeVar T = TypeVar('T') @@ -287,13 +290,14 @@ Using a generic class without specifying type parameters assumes :data:`Any` for each position. In the following example, ``MyIterable`` is not generic but implicitly inherits from ``Iterable[Any]``:: - from typing import Iterable + from collections.abc import Iterable class MyIterable(Iterable): # Same as Iterable[Any] User defined generic type aliases are also supported. Examples:: - from typing import TypeVar, Iterable, Tuple, Union + from collections.abc import Iterable + from typing import TypeVar, Union S = TypeVar('S') Response = Union[Iterable[S], int] @@ -302,9 +306,9 @@ User defined generic type aliases are also supported. Examples:: ... T = TypeVar('T', int, float, complex) - Vec = Iterable[Tuple[T, T]] + Vec = Iterable[tuple[T, T]] - def inproduct(v: Vec[T]) -> T: # Same as Iterable[Tuple[T, T]] + def inproduct(v: Vec[T]) -> T: # Same as Iterable[tuple[T, T]] return sum(x*y for x, y in v) .. versionchanged:: 3.7 @@ -317,14 +321,14 @@ comparable for equality. The :data:`Any` type --------------------- +==================== A special kind of type is :data:`Any`. A static type checker will treat every type as being compatible with :data:`Any` and :data:`Any` as being compatible with every type. This means that it is possible to perform any operation or method call on a -value of type on :data:`Any` and assign it to any variable:: +value of type :data:`Any` and assign it to any variable:: from typing import Any @@ -395,19 +399,19 @@ manner. Use :data:`Any` to indicate that a value is dynamically typed. Nominal vs structural subtyping -------------------------------- +=============================== Initially :pep:`484` defined Python static type system as using *nominal subtyping*. This means that a class ``A`` is allowed where a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. This requirement previously also applied to abstract base classes, such as -:class:`Iterable`. The problem with this approach is that a class had +:class:`~collections.abc.Iterable`. The problem with this approach is that a class had to be explicitly marked to support them, which is unpythonic and unlike what one would normally do in idiomatic dynamically typed Python code. -For example, this conforms to the :pep:`484`:: +For example, this conforms to :pep:`484`:: - from typing import Sized, Iterable, Iterator + from collections.abc import Sized, Iterable, Iterator class Bucket(Sized, Iterable[int]): ... @@ -420,7 +424,7 @@ allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` and ``Iterable[int]`` by static type checkers. This is known as *structural subtyping* (or static duck-typing):: - from typing import Iterator, Iterable + from collections.abc import Iterator, Iterable class Bucket: # Note: no base classes ... @@ -434,106 +438,154 @@ Moreover, by subclassing a special class :class:`Protocol`, a user can define new custom protocols to fully enjoy structural subtyping (see examples below). +Module contents +=============== -Classes, functions, and decorators ----------------------------------- +The module defines the following classes, functions and decorators. -The module defines the following classes, functions and decorators: +.. note:: -.. class:: TypeVar + This module defines several types that are subclasses of pre-existing + standard library classes which also extend :class:`Generic` + to support type variables inside ``[]``. + These types became redundant in Python 3.9 when the + corresponding pre-existing classes were enhanced to support ``[]``. - Type variable. + The redundant types are deprecated as of Python 3.9 but no + deprecation warnings will be issued by the interpreter. + It is expected that type checkers will flag the deprecated types + when the checked program targets Python 3.9 or newer. - Usage:: + The deprecated types will be removed from the :mod:`typing` module + in the first Python version released 5 years after the release of Python 3.9.0. + See details in :pep:`585`—*Type Hinting Generics In Standard Collections*. - T = TypeVar('T') # Can be anything - A = TypeVar('A', str, bytes) # Must be str or bytes - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function definitions. See class Generic for more - information on generic types. Generic functions work as follows:: +Special typing primitives +------------------------- - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n +Special types +""""""""""""" - def longest(x: A, y: A) -> A: - """Return the longest of two strings.""" - return x if len(x) >= len(y) else y +These can be used as types in annotations and do not support ``[]``. - The latter example's signature is essentially the overloading - of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note - that if the arguments are instances of some subclass of :class:`str`, - the return type is still plain :class:`str`. +.. data:: Any - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, - :func:`isinstance` and :func:`issubclass` should not be used with types. + Special type indicating an unconstrained type. - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default type variables are invariant. Alternatively, - a type variable may specify an upper bound using ``bound=``. - This means that an actual type substituted (explicitly or implicitly) - for the type variable must be a subclass of the boundary type, - see :pep:`484`. + * Every type is compatible with :data:`Any`. + * :data:`Any` is compatible with every type. -.. class:: Generic +.. data:: NoReturn - Abstract base class for generic types. + Special type indicating that a function never returns. + For example:: - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: + from typing import NoReturn - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. + def stop() -> NoReturn: + raise RuntimeError('no way') - This class can then be used as follows:: + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 - X = TypeVar('X') - Y = TypeVar('Y') +Special forms +""""""""""""" - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: - try: - return mapping[key] - except KeyError: - return default +These can be used as types in annotations using ``[]``, each having a unique syntax. -.. class:: Protocol(Generic) +.. data:: Tuple - Base class for protocol classes. Protocol classes are defined like this:: + Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items + with the first item of type X and the second of type Y. The type of + the empty tuple can be written as ``Tuple[()]``. - class Proto(Protocol): - def meth(self) -> int: - ... + Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding + to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple + of an int, a float and a string. - Such classes are primarily used with static type checkers that recognize - structural subtyping (static duck-typing), for example:: + To specify a variable-length tuple of homogeneous type, + use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` + is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. - class C: - def meth(self) -> int: - return 0 + .. deprecated:: 3.9 + :class:`builtins.tuple ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - def func(x: Proto) -> int: - return x.meth() +.. data:: Union - func(C()) # Passes static type check + Union type; ``Union[X, Y]`` means either X or Y. - See :pep:`544` for details. Protocol classes decorated with - :func:`runtime_checkable` (described later) act as simple-minded runtime - protocols that check only the presence of given attributes, ignoring their - type signatures. + To define a union, use e.g. ``Union[int, str]``. Details: - Protocol classes can be generic, for example:: + * The arguments must be types and there must be at least one. - class GenProto(Protocol[T]): - def meth(self) -> T: - ... + * Unions of unions are flattened, e.g.:: - .. versionadded:: 3.8 + Union[Union[int, str], float] == Union[int, str, float] + + * Unions of a single argument vanish, e.g.:: + + Union[int] == int # The constructor actually returns int + + * Redundant arguments are skipped, e.g.:: + + Union[int, str, int] == Union[int, str] + + * When comparing unions, the argument order is ignored, e.g.:: + + Union[int, str] == Union[str, int] + + * You cannot subclass or instantiate a union. + + * You cannot write ``Union[X][Y]``. + + * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. + + .. versionchanged:: 3.7 + Don't remove explicit subclasses from unions at runtime. + +.. data:: Optional + + Optional type. + + ``Optional[X]`` is equivalent to ``Union[X, None]``. + + Note that this is not the same concept as an optional argument, + which is one that has a default. An optional argument with a + default does not require the ``Optional`` qualifier on its type + annotation just because it is optional. For example:: + + def foo(arg: int = 0) -> None: + ... + + On the other hand, if an explicit value of ``None`` is allowed, the + use of ``Optional`` is appropriate, whether the argument is optional + or not. For example:: + + def foo(arg: Optional[int] = None) -> None: + ... + +.. data:: Callable + + Callable type; ``Callable[[int], str]`` is a function of (int) -> str. + + The subscription syntax must always be used with exactly two + values: the argument list and the return type. The argument list + must be a list of types or an ellipsis; the return type must be + a single type. + + There is no syntax to indicate optional or keyword arguments; + such function types are rarely used as callback types. + ``Callable[..., ReturnType]`` (literal ellipsis) can be used to + type hint a callable taking any number of arguments and returning + ``ReturnType``. A plain :data:`Callable` is equivalent to + ``Callable[..., Any]``, and in turn to + :class:`collections.abc.Callable`. + + .. deprecated:: 3.9 + :class:`collections.abc.Callable` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. .. class:: Type(Generic[CT_co]) @@ -570,391 +622,366 @@ The module defines the following classes, functions and decorators: :ref:`type variables `, and unions of any of these types. For example:: - def new_non_team_user(user_class: Type[Union[BaseUser, ProUser]]): ... + def new_non_team_user(user_class: Type[Union[BasicUser, ProUser]]): ... ``Type[Any]`` is equivalent to ``Type`` which in turn is equivalent to ``type``, which is the root of Python's metaclass hierarchy. .. versionadded:: 3.5.2 -.. class:: Iterable(Generic[T_co]) + .. deprecated:: 3.9 + :class:`builtins.type ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - A generic version of :class:`collections.abc.Iterable`. +.. data:: Literal -.. class:: Iterator(Iterable[T_co]) + A type that can be used to indicate to type checkers that the + corresponding variable or function parameter has a value equivalent to + the provided literal (or one of several literals). For example:: - A generic version of :class:`collections.abc.Iterator`. + def validate_simple(data: Any) -> Literal[True]: # always returns True + ... -.. class:: Reversible(Iterable[T_co]) + MODE = Literal['r', 'rb', 'w', 'wb'] + def open_helper(file: str, mode: MODE) -> str: + ... - A generic version of :class:`collections.abc.Reversible`. + open_helper('/some/path', 'r') # Passes type check + open_helper('/other/path', 'typo') # Error in type checker -.. class:: SupportsInt + ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value + is allowed as type argument to ``Literal[...]``, but type checkers may + impose restrictions. See :pep:`586` for more details about literal types. - An ABC with one abstract method ``__int__``. + .. versionadded:: 3.8 -.. class:: SupportsFloat + .. versionchanged:: 3.9.1 + ``Literal`` now de-duplicates parameters. Equality comparisons of + ``Literal`` objects are no longer order dependent. ``Literal`` objects + will now raise a :exc:`TypeError` exception during equality comparisons + if one of their parameters are not :term:`hashable`. - An ABC with one abstract method ``__float__``. +.. data:: ClassVar -.. class:: SupportsComplex + Special type construct to mark class variables. - An ABC with one abstract method ``__complex__``. + As introduced in :pep:`526`, a variable annotation wrapped in ClassVar + indicates that a given attribute is intended to be used as a class variable + and should not be set on instances of that class. Usage:: -.. class:: SupportsBytes + class Starship: + stats: ClassVar[dict[str, int]] = {} # class variable + damage: int = 10 # instance variable - An ABC with one abstract method ``__bytes__``. + :data:`ClassVar` accepts only types and cannot be further subscribed. -.. class:: SupportsIndex + :data:`ClassVar` is not a class itself, and should not + be used with :func:`isinstance` or :func:`issubclass`. + :data:`ClassVar` does not change Python runtime behavior, but + it can be used by third-party type checkers. For example, a type checker + might flag the following code as an error:: - An ABC with one abstract method ``__index__``. + enterprise_d = Starship(3000) + enterprise_d.stats = {} # Error, setting class variable on instance + Starship.stats = {} # This is OK - .. versionadded:: 3.8 + .. versionadded:: 3.5.3 -.. class:: SupportsAbs +.. data:: Final - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. + A special typing construct to indicate to type checkers that a name + cannot be re-assigned or overridden in a subclass. For example:: -.. class:: SupportsRound + MAX_SIZE: Final = 9000 + MAX_SIZE += 1 # Error reported by type checker - An ABC with one abstract method ``__round__`` - that is covariant in its return type. + class Connection: + TIMEOUT: Final[int] = 10 -.. class:: Container(Generic[T_co]) + class FastConnector(Connection): + TIMEOUT = 1 # Error reported by type checker - A generic version of :class:`collections.abc.Container`. + There is no runtime checking of these properties. See :pep:`591` for + more details. -.. class:: Hashable + .. versionadded:: 3.8 - An alias to :class:`collections.abc.Hashable` +.. data:: Annotated -.. class:: Sized + A type, introduced in :pep:`593` (``Flexible function and variable + annotations``), to decorate existing types with context-specific metadata + (possibly multiple pieces of it, as ``Annotated`` is variadic). + Specifically, a type ``T`` can be annotated with metadata ``x`` via the + typehint ``Annotated[T, x]``. This metadata can be used for either static + analysis or at runtime. If a library (or tool) encounters a typehint + ``Annotated[T, x]`` and has no special logic for metadata ``x``, it + should ignore it and simply treat the type as ``T``. Unlike the + ``no_type_check`` functionality that currently exists in the ``typing`` + module which completely disables typechecking annotations on a function + or a class, the ``Annotated`` type allows for both static typechecking + of ``T`` (e.g., via mypy or Pyre, which can safely ignore ``x``) + together with runtime access to ``x`` within a specific application. - An alias to :class:`collections.abc.Sized` + Ultimately, the responsibility of how to interpret the annotations (if + at all) is the responsibility of the tool or library encountering the + ``Annotated`` type. A tool or library encountering an ``Annotated`` type + can scan through the annotations to determine if they are of interest + (e.g., using ``isinstance()``). -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) + When a tool or a library does not support annotations or encounters an + unknown annotation it should just ignore it and treat annotated type as + the underlying type. - A generic version of :class:`collections.abc.Collection` + It's up to the tool consuming the annotations to decide whether the + client is allowed to have several annotations on one type and how to + merge those annotations. - .. versionadded:: 3.6.0 + Since the ``Annotated`` type allows you to put several annotations of + the same (or different) type(s) on any node, the tools or libraries + consuming those annotations are in charge of dealing with potential + duplicates. For example, if you are doing value range analysis you might + allow this:: -.. class:: AbstractSet(Sized, Collection[T_co]) + T1 = Annotated[int, ValueRange(-10, 5)] + T2 = Annotated[T1, ValueRange(-20, 3)] - A generic version of :class:`collections.abc.Set`. + Passing ``include_extras=True`` to :func:`get_type_hints` lets one + access the extra annotations at runtime. -.. class:: MutableSet(AbstractSet[T]) + The details of the syntax: - A generic version of :class:`collections.abc.MutableSet`. + * The first argument to ``Annotated`` must be a valid type -.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) + * Multiple type annotations are supported (``Annotated`` supports variadic + arguments):: - A generic version of :class:`collections.abc.Mapping`. - This type can be used as follows:: + Annotated[int, ValueRange(3, 10), ctype("char")] - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] + * ``Annotated`` must be called with at least two arguments ( + ``Annotated[int]`` is not valid) -.. class:: MutableMapping(Mapping[KT, VT]) + * The order of the annotations is preserved and matters for equality + checks:: - A generic version of :class:`collections.abc.MutableMapping`. + Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ + int, ctype("char"), ValueRange(3, 10) + ] -.. class:: Sequence(Reversible[T_co], Collection[T_co]) + * Nested ``Annotated`` types are flattened, with metadata ordered + starting with the innermost annotation:: - A generic version of :class:`collections.abc.Sequence`. + Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ + int, ValueRange(3, 10), ctype("char") + ] -.. class:: MutableSequence(Sequence[T]) + * Duplicated annotations are not removed:: - A generic version of :class:`collections.abc.MutableSequence`. + Annotated[int, ValueRange(3, 10)] != Annotated[ + int, ValueRange(3, 10), ValueRange(3, 10) + ] -.. class:: ByteString(Sequence[int]) + * ``Annotated`` can be used with nested and generic aliases:: - A generic version of :class:`collections.abc.ByteString`. + T = TypeVar('T') + Vec = Annotated[list[tuple[T, T]], MaxLen(10)] + V = Vec[int] - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview`. + V == Annotated[list[tuple[int, int]], MaxLen(10)] - As a shorthand for this type, :class:`bytes` can be used to - annotate arguments of any of the types mentioned above. + .. versionadded:: 3.9 -.. class:: Deque(deque, MutableSequence[T]) +Building generic types +"""""""""""""""""""""" - A generic version of :class:`collections.deque`. +These are not used in annotations. They are building blocks for creating generic types. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 +.. class:: Generic -.. class:: List(list, MutableSequence[T]) + Abstract base class for generic types. - Generic version of :class:`list`. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable`. + A generic type is typically declared by inheriting from an + instantiation of this class with one or more type variables. + For example, a generic mapping type might be defined as:: - This type may be used as follows:: + class Mapping(Generic[KT, VT]): + def __getitem__(self, key: KT) -> VT: + ... + # Etc. - T = TypeVar('T', int, float) + This class can then be used as follows:: - def vec2(x: T, y: T) -> List[T]: - return [x, y] + X = TypeVar('X') + Y = TypeVar('Y') - def keep_positives(vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] + def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: + try: + return mapping[key] + except KeyError: + return default -.. class:: Set(set, MutableSet[T]) +.. class:: TypeVar - A generic version of :class:`builtins.set `. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`AbstractSet`. + Type variable. -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) + Usage:: - A generic version of :class:`builtins.frozenset `. + T = TypeVar('T') # Can be anything + A = TypeVar('A', str, bytes) # Must be str or bytes -.. class:: MappingView(Sized, Iterable[T_co]) + Type variables exist primarily for the benefit of static type + checkers. They serve as the parameters for generic types as well + as for generic function definitions. See :class:`Generic` for more + information on generic types. Generic functions work as follows:: - A generic version of :class:`collections.abc.MappingView`. + def repeat(x: T, n: int) -> Sequence[T]: + """Return a list containing n references to x.""" + return [x]*n -.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) + def longest(x: A, y: A) -> A: + """Return the longest of two strings.""" + return x if len(x) >= len(y) else y - A generic version of :class:`collections.abc.KeysView`. + The latter example's signature is essentially the overloading + of ``(str, str) -> str`` and ``(bytes, bytes) -> bytes``. Also note + that if the arguments are instances of some subclass of :class:`str`, + the return type is still plain :class:`str`. -.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) + At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. In general, + :func:`isinstance` and :func:`issubclass` should not be used with types. - A generic version of :class:`collections.abc.ItemsView`. + Type variables may be marked covariant or contravariant by passing + ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more + details. By default type variables are invariant. Alternatively, + a type variable may specify an upper bound using ``bound=``. + This means that an actual type substituted (explicitly or implicitly) + for the type variable must be a subclass of the boundary type, + see :pep:`484`. -.. class:: ValuesView(MappingView[VT_co]) +.. data:: AnyStr - A generic version of :class:`collections.abc.ValuesView`. + ``AnyStr`` is a type variable defined as + ``AnyStr = TypeVar('AnyStr', str, bytes)``. -.. class:: Awaitable(Generic[T_co]) + It is meant to be used for functions that may accept any kind of string + without allowing different kinds of strings to mix. For example:: - A generic version of :class:`collections.abc.Awaitable`. + def concat(a: AnyStr, b: AnyStr) -> AnyStr: + return a + b - .. versionadded:: 3.5.2 + concat(u"foo", u"bar") # Ok, output has type 'unicode' + concat(b"foo", b"bar") # Ok, output has type 'bytes' + concat(u"foo", b"bar") # Error, cannot mix unicode and bytes -.. class:: Coroutine(Awaitable[V_co], Generic[T_co T_contra, V_co]) +.. class:: Protocol(Generic) - A generic version of :class:`collections.abc.Coroutine`. - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: + Base class for protocol classes. Protocol classes are defined like this:: - from typing import List, Coroutine - c = None # type: Coroutine[List[str], str, int] - ... - x = c.send('hi') # type: List[str] - async def bar() -> None: - x = await c # type: int + class Proto(Protocol): + def meth(self) -> int: + ... - .. versionadded:: 3.5.3 + Such classes are primarily used with static type checkers that recognize + structural subtyping (static duck-typing), for example:: -.. class:: AsyncIterable(Generic[T_co]) + class C: + def meth(self) -> int: + return 0 - A generic version of :class:`collections.abc.AsyncIterable`. + def func(x: Proto) -> int: + return x.meth() - .. versionadded:: 3.5.2 + func(C()) # Passes static type check -.. class:: AsyncIterator(AsyncIterable[T_co]) + See :pep:`544` for details. Protocol classes decorated with + :func:`runtime_checkable` (described later) act as simple-minded runtime + protocols that check only the presence of given attributes, ignoring their + type signatures. - A generic version of :class:`collections.abc.AsyncIterator`. + Protocol classes can be generic, for example:: - .. versionadded:: 3.5.2 + class GenProto(Protocol[T]): + def meth(self) -> T: + ... -.. class:: ContextManager(Generic[T_co]) + .. versionadded:: 3.8 - A generic version of :class:`contextlib.AbstractContextManager`. +.. decorator:: runtime_checkable - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 + Mark a protocol class as a runtime protocol. -.. class:: AsyncContextManager(Generic[T_co]) + Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. + This raises :exc:`TypeError` when applied to a non-protocol class. This + allows a simple-minded structural check, very similar to "one trick ponies" + in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: - A generic version of :class:`contextlib.AbstractAsyncContextManager`. + @runtime_checkable + class Closable(Protocol): + def close(self): ... - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + assert isinstance(open('/some/file'), Closable) -.. class:: Dict(dict, MutableMapping[KT, VT]) + .. note:: - A generic version of :class:`dict`. - Useful for annotating return types. To annotate arguments it is preferred - to use an abstract collection type such as :class:`Mapping`. + :func:`runtime_checkable` will check only the presence of the required methods, + not their type signatures! For example, :class:`builtins.complex ` + implements :func:`__float__`, therefore it passes an :func:`issubclass` check + against :class:`SupportsFloat`. However, the ``complex.__float__`` method + exists only to raise a :class:`TypeError` with a more informative message. - This type can be used as follows:: + .. versionadded:: 3.8 - def count_words(text: str) -> Dict[str, int]: - ... +Other special directives +"""""""""""""""""""""""" -.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) +These are not used in annotations. They are building blocks for declaring types. - A generic version of :class:`collections.defaultdict`. +.. class:: NamedTuple - .. versionadded:: 3.5.2 + Typed version of :func:`collections.namedtuple`. -.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) + Usage:: - A generic version of :class:`collections.OrderedDict`. + class Employee(NamedTuple): + name: str + id: int - .. versionadded:: 3.7.2 + This is equivalent to:: -.. class:: Counter(collections.Counter, Dict[T, int]) + Employee = collections.namedtuple('Employee', ['name', 'id']) - A generic version of :class:`collections.Counter`. + To give a field a default value, you can assign to it in the class body:: - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + class Employee(NamedTuple): + name: str + id: int = 3 -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) + employee = Employee('Guido') + assert employee.id == 3 - A generic version of :class:`collections.ChainMap`. + Fields with a default value must come after any fields without a default. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 + The resulting class has an extra attribute ``__annotations__`` giving a + dict that maps the field names to the field types. (The field names are in + the ``_fields`` attribute and the default values are in the + ``_field_defaults`` attribute both of which are part of the namedtuple + API.) -.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) + ``NamedTuple`` subclasses can also have docstrings and methods:: - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: + class Employee(NamedTuple): + """Represents an employee.""" + name: str + id: int = 3 - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' + def __repr__(self) -> str: + return f'' - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. + Backward-compatible usage:: - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: + Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 - - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 - -.. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) - - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: - - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded - - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. - - If your generator will only yield values, set the ``SendType`` to - ``None``:: - - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) - - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) - - .. versionadded:: 3.6.1 - -.. class:: Text - - ``Text`` is an alias for ``str``. It is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. - - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: - - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' - - .. versionadded:: 3.5.2 - -.. class:: IO - TextIO - BinaryIO - - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. - -.. class:: Pattern - Match - - These type aliases - correspond to the return types from :func:`re.compile` and - :func:`re.match`. These types (and the corresponding functions) - are generic in ``AnyStr`` and can be made specific by writing - ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or - ``Match[bytes]``. - -.. class:: NamedTuple - - Typed version of :func:`collections.namedtuple`. - - Usage:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - To give a field a default value, you can assign to it in the class body:: - - class Employee(NamedTuple): - name: str - id: int = 3 - - employee = Employee('Guido') - assert employee.id == 3 - - Fields with a default value must come after any fields without a default. - - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the - ``_field_defaults`` attribute both of which are part of the namedtuple - API.) - - ``NamedTuple`` subclasses can also have docstrings and methods:: - - class Employee(NamedTuple): - """Represents an employee.""" - name: str - id: int = 3 - - def __repr__(self) -> str: - return f'' - - Backward-compatible usage:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - - .. versionchanged:: 3.6 - Added support for :pep:`526` variable annotation syntax. + .. versionchanged:: 3.6 + Added support for :pep:`526` variable annotation syntax. .. versionchanged:: 3.6.1 Added support for default values, methods, and docstrings. @@ -967,13 +994,23 @@ The module defines the following classes, functions and decorators: Removed the ``_field_types`` attribute in favor of the more standard ``__annotations__`` attribute which has the same information. +.. function:: NewType(name, tp) + + A helper function to indicate a distinct type to a typechecker, + see :ref:`distinct`. At runtime it returns a function that returns + its argument. Usage:: + + UserId = NewType('UserId', int) + first_user = UserId(1) + + .. versionadded:: 3.5.2 .. class:: TypedDict(dict) - A simple typed namespace. At runtime it is equivalent to - a plain :class:`dict`. + Special construct to add type hints to a dictionary. + At runtime it is a plain :class:`dict`. - ``TypedDict`` creates a dictionary type that expects all of its + ``TypedDict`` declares a dictionary type that expects all of its instances to have a certain set of keys, where each key is associated with a value of a consistent type. This expectation is not checked at runtime but is only enforced by type checkers. @@ -1014,474 +1051,710 @@ The module defines the following classes, functions and decorators: .. versionadded:: 3.8 -.. class:: ForwardRef +Generic concrete collections +---------------------------- - A class used for internal typing representation of string forward references. - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by - a user, but may be used by introspection tools. +Corresponding to built-in types +""""""""""""""""""""""""""""""" -.. function:: NewType(typ) +.. class:: Dict(dict, MutableMapping[KT, VT]) - A helper function to indicate a distinct types to a typechecker, - see :ref:`distinct`. At runtime it returns a function that returns - its argument. Usage:: + A generic version of :class:`dict`. + Useful for annotating return types. To annotate arguments it is preferred + to use an abstract collection type such as :class:`Mapping`. - UserId = NewType('UserId', int) - first_user = UserId(1) + This type can be used as follows:: - .. versionadded:: 3.5.2 + def count_words(text: str) -> Dict[str, int]: + ... -.. function:: cast(typ, val) + .. deprecated:: 3.9 + :class:`builtins.dict ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - Cast a value to a type. +.. class:: List(list, MutableSequence[T]) - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). + Generic version of :class:`list`. + Useful for annotating return types. To annotate arguments it is preferred + to use an abstract collection type such as :class:`Sequence` or + :class:`Iterable`. -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) + This type may be used as follows:: - Return a dictionary containing type hints for a function, method, module - or class object. + T = TypeVar('T', int, float) - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. If necessary, - ``Optional[t]`` is added for function and method annotations if a default - value equal to ``None`` is set. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. + def vec2(x: T, y: T) -> List[T]: + return [x, y] - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example:: + def keep_positives(vector: Sequence[T]) -> List[T]: + return [item for item in vector if item > 0] - class Student(NamedTuple): - name: Annotated[str, 'some marker'] + .. deprecated:: 3.9 + :class:`builtins.list ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - get_type_hints(Student) == {'name': str} - get_type_hints(Student, include_extras=False) == {'name': str} - get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } +.. class:: Set(set, MutableSet[T]) - .. versionchanged:: 3.9 - Added ``include_extras`` parameter as part of :pep:`593`. + A generic version of :class:`builtins.set `. + Useful for annotating return types. To annotate arguments it is preferred + to use an abstract collection type such as :class:`AbstractSet`. -.. function:: get_origin(tp) -.. function:: get_args(tp) + .. deprecated:: 3.9 + :class:`builtins.set ` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - Provide basic introspection for generic types and special typing forms. +.. class:: FrozenSet(frozenset, AbstractSet[T_co]) - For a typing object of the form ``X[Y, Z, ...]`` these functions return - ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or - :mod:`collections` class, it gets normalized to the original class. - For unsupported objects return ``None`` and ``()`` correspondingly. - Examples:: + A generic version of :class:`builtins.frozenset `. - assert get_origin(Dict[str, int]) is dict - assert get_args(Dict[int, str]) == (int, str) + .. deprecated:: 3.9 + :class:`builtins.frozenset ` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. - assert get_origin(Union[int, str]) is Union - assert get_args(Union[int, str]) == (int, str) +.. note:: :data:`Tuple` is a special form. - .. versionadded:: 3.8 +Corresponding to types in :mod:`collections` +"""""""""""""""""""""""""""""""""""""""""""" -.. decorator:: overload +.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - The ``@overload`` decorator allows describing functions and methods - that support multiple different combinations of argument types. A series - of ``@overload``-decorated definitions must be followed by exactly one - non-``@overload``-decorated definition (for the same function/method). - The ``@overload``-decorated definitions are for the benefit of the - type checker only, since they will be overwritten by the - non-``@overload``-decorated definition, while the latter is used at - runtime but should be ignored by a type checker. At runtime, calling - a ``@overload``-decorated function directly will raise - :exc:`NotImplementedError`. An example of overload that gives a more - precise type than can be expressed using a union or a type variable:: + A generic version of :class:`collections.defaultdict`. - @overload - def process(response: None) -> None: - ... - @overload - def process(response: int) -> Tuple[int, str]: - ... - @overload - def process(response: bytes) -> str: - ... - def process(response): - + .. versionadded:: 3.5.2 - See :pep:`484` for details and comparison with other typing semantics. + .. deprecated:: 3.9 + :class:`collections.defaultdict` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. -.. decorator:: final +.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) - A decorator to indicate to type checkers that the decorated method - cannot be overridden, and the decorated class cannot be subclassed. - For example:: + A generic version of :class:`collections.OrderedDict`. - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker - ... + .. versionadded:: 3.7.2 - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... + .. deprecated:: 3.9 + :class:`collections.OrderedDict` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - There is no runtime checking of these properties. See :pep:`591` for - more details. +.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) - .. versionadded:: 3.8 + A generic version of :class:`collections.ChainMap`. -.. decorator:: no_type_check + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - Decorator to indicate that annotations are not type hints. + .. deprecated:: 3.9 + :class:`collections.ChainMap` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - This works as class or function :term:`decorator`. With a class, it - applies recursively to all methods defined in that class (but not - to methods defined in its superclasses or subclasses). +.. class:: Counter(collections.Counter, Dict[T, int]) - This mutates the function(s) in place. + A generic version of :class:`collections.Counter`. -.. decorator:: no_type_check_decorator + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - Decorator to give another decorator the :func:`no_type_check` effect. + .. deprecated:: 3.9 + :class:`collections.Counter` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. +.. class:: Deque(deque, MutableSequence[T]) -.. decorator:: type_check_only + A generic version of :class:`collections.deque`. - Decorator to mark a class or function to be unavailable at runtime. + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.1 - This decorator is itself not available at runtime. It is mainly - intended to mark classes that are defined in type stub files if - an implementation returns an instance of a private class:: + .. deprecated:: 3.9 + :class:`collections.deque` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. - @type_check_only - class Response: # private or not available at runtime - code: int - def get_header(self, name: str) -> str: ... +Other concrete types +"""""""""""""""""""" - def fetch_response() -> Response: ... +.. class:: IO + TextIO + BinaryIO - Note that returning instances of private classes is not recommended. - It is usually preferable to make such classes public. + Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` + and ``BinaryIO(IO[bytes])`` + represent the types of I/O streams such as returned by + :func:`open`. -.. decorator:: runtime_checkable + .. deprecated-removed:: 3.8 3.12 + These types are also in the ``typing.io`` namespace, which was + never supported by type checkers and will be removed. - Mark a protocol class as a runtime protocol. +.. class:: Pattern + Match - Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This raises :exc:`TypeError` when applied to a non-protocol class. This - allows a simple-minded structural check, very similar to "one trick ponies" - in :mod:`collections.abc` such as :class:`Iterable`. For example:: + These type aliases + correspond to the return types from :func:`re.compile` and + :func:`re.match`. These types (and the corresponding functions) + are generic in ``AnyStr`` and can be made specific by writing + ``Pattern[str]``, ``Pattern[bytes]``, ``Match[str]``, or + ``Match[bytes]``. - @runtime_checkable - class Closable(Protocol): - def close(self): ... + .. deprecated-removed:: 3.8 3.12 + These types are also in the ``typing.re`` namespace, which was + never supported by type checkers and will be removed. - assert isinstance(open('/some/file'), Closable) + .. deprecated:: 3.9 + Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. + See :pep:`585` and :ref:`types-genericalias`. - **Warning:** this will check only the presence of the required methods, - not their type signatures! +.. class:: Text - .. versionadded:: 3.8 + ``Text`` is an alias for ``str``. It is provided to supply a forward + compatible path for Python 2 code: in Python 2, ``Text`` is an alias for + ``unicode``. -.. data:: Any + Use ``Text`` to indicate that a value must contain a unicode string in + a manner that is compatible with both Python 2 and Python 3:: - Special type indicating an unconstrained type. + def add_unicode_checkmark(text: Text) -> Text: + return text + u' \u2713' - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. + .. versionadded:: 3.5.2 -.. data:: NoReturn +Abstract Base Classes +--------------------- - Special type indicating that a function never returns. - For example:: +Corresponding to collections in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" - from typing import NoReturn +.. class:: AbstractSet(Sized, Collection[T_co]) - def stop() -> NoReturn: - raise RuntimeError('no way') + A generic version of :class:`collections.abc.Set`. - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 + .. deprecated:: 3.9 + :class:`collections.abc.Set` now supports ``[]``. See :pep:`585` and + :ref:`types-genericalias`. -.. data:: Union +.. class:: ByteString(Sequence[int]) - Union type; ``Union[X, Y]`` means either X or Y. + A generic version of :class:`collections.abc.ByteString`. - To define a union, use e.g. ``Union[int, str]``. Details: + This type represents the types :class:`bytes`, :class:`bytearray`, + and :class:`memoryview` of byte sequences. - * The arguments must be types and there must be at least one. + As a shorthand for this type, :class:`bytes` can be used to + annotate arguments of any of the types mentioned above. - * Unions of unions are flattened, e.g.:: + .. deprecated:: 3.9 + :class:`collections.abc.ByteString` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - Union[Union[int, str], float] == Union[int, str, float] +.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) - * Unions of a single argument vanish, e.g.:: + A generic version of :class:`collections.abc.Collection` - Union[int] == int # The constructor actually returns int + .. versionadded:: 3.6.0 - * Redundant arguments are skipped, e.g.:: + .. deprecated:: 3.9 + :class:`collections.abc.Collection` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - Union[int, str, int] == Union[int, str] +.. class:: Container(Generic[T_co]) - * When comparing unions, the argument order is ignored, e.g.:: + A generic version of :class:`collections.abc.Container`. - Union[int, str] == Union[str, int] + .. deprecated:: 3.9 + :class:`collections.abc.Container` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - * You cannot subclass or instantiate a union. +.. class:: ItemsView(MappingView, Generic[KT_co, VT_co]) - * You cannot write ``Union[X][Y]``. + A generic version of :class:`collections.abc.ItemsView`. - * You can use ``Optional[X]`` as a shorthand for ``Union[X, None]``. + .. deprecated:: 3.9 + :class:`collections.abc.ItemsView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - .. versionchanged:: 3.7 - Don't remove explicit subclasses from unions at runtime. +.. class:: KeysView(MappingView[KT_co], AbstractSet[KT_co]) -.. data:: Optional + A generic version of :class:`collections.abc.KeysView`. - Optional type. + .. deprecated:: 3.9 + :class:`collections.abc.KeysView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - ``Optional[X]`` is equivalent to ``Union[X, None]``. +.. class:: Mapping(Sized, Collection[KT], Generic[VT_co]) - Note that this is not the same concept as an optional argument, - which is one that has a default. An optional argument with a - default does not require the ``Optional`` qualifier on its type - annotation just because it is optional. For example:: + A generic version of :class:`collections.abc.Mapping`. + This type can be used as follows:: - def foo(arg: int = 0) -> None: - ... + def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: + return word_list[word] - On the other hand, if an explicit value of ``None`` is allowed, the - use of ``Optional`` is appropriate, whether the argument is optional - or not. For example:: + .. deprecated:: 3.9 + :class:`collections.abc.Mapping` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - def foo(arg: Optional[int] = None) -> None: - ... +.. class:: MappingView(Sized, Iterable[T_co]) -.. data:: Tuple + A generic version of :class:`collections.abc.MappingView`. - Tuple type; ``Tuple[X, Y]`` is the type of a tuple of two items - with the first item of type X and the second of type Y. The type of - the empty tuple can be written as ``Tuple[()]``. + .. deprecated:: 3.9 + :class:`collections.abc.MappingView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - Example: ``Tuple[T1, T2]`` is a tuple of two elements corresponding - to type variables T1 and T2. ``Tuple[int, float, str]`` is a tuple - of an int, a float and a string. +.. class:: MutableMapping(Mapping[KT, VT]) - To specify a variable-length tuple of homogeneous type, - use literal ellipsis, e.g. ``Tuple[int, ...]``. A plain :data:`Tuple` - is equivalent to ``Tuple[Any, ...]``, and in turn to :class:`tuple`. + A generic version of :class:`collections.abc.MutableMapping`. -.. data:: Callable + .. deprecated:: 3.9 + :class:`collections.abc.MutableMapping` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. - Callable type; ``Callable[[int], str]`` is a function of (int) -> str. +.. class:: MutableSequence(Sequence[T]) - The subscription syntax must always be used with exactly two - values: the argument list and the return type. The argument list - must be a list of types or an ellipsis; the return type must be - a single type. + A generic version of :class:`collections.abc.MutableSequence`. - There is no syntax to indicate optional or keyword arguments; - such function types are rarely used as callback types. - ``Callable[..., ReturnType]`` (literal ellipsis) can be used to - type hint a callable taking any number of arguments and returning - ``ReturnType``. A plain :data:`Callable` is equivalent to - ``Callable[..., Any]``, and in turn to - :class:`collections.abc.Callable`. + .. deprecated:: 3.9 + :class:`collections.abc.MutableSequence` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. -.. data:: Literal +.. class:: MutableSet(AbstractSet[T]) - A type that can be used to indicate to type checkers that the - corresponding variable or function parameter has a value equivalent to - the provided literal (or one of several literals). For example:: + A generic version of :class:`collections.abc.MutableSet`. - def validate_simple(data: Any) -> Literal[True]: # always returns True - ... + .. deprecated:: 3.9 + :class:`collections.abc.MutableSet` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - MODE = Literal['r', 'rb', 'w', 'wb'] - def open_helper(file: str, mode: MODE) -> str: - ... +.. class:: Sequence(Reversible[T_co], Collection[T_co]) - open_helper('/some/path', 'r') # Passes type check - open_helper('/other/path', 'typo') # Error in type checker + A generic version of :class:`collections.abc.Sequence`. - ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value - is allowed as type argument to ``Literal[...]``, but type checkers may - impose restrictions. See :pep:`586` for more details about literal types. + .. deprecated:: 3.9 + :class:`collections.abc.Sequence` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - .. versionadded:: 3.8 +.. class:: ValuesView(MappingView[VT_co]) -.. data:: ClassVar + A generic version of :class:`collections.abc.ValuesView`. - Special type construct to mark class variables. + .. deprecated:: 3.9 + :class:`collections.abc.ValuesView` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - As introduced in :pep:`526`, a variable annotation wrapped in ClassVar - indicates that a given attribute is intended to be used as a class variable - and should not be set on instances of that class. Usage:: +Corresponding to other types in :mod:`collections.abc` +"""""""""""""""""""""""""""""""""""""""""""""""""""""" - class Starship: - stats: ClassVar[Dict[str, int]] = {} # class variable - damage: int = 10 # instance variable +.. class:: Iterable(Generic[T_co]) - :data:`ClassVar` accepts only types and cannot be further subscribed. + A generic version of :class:`collections.abc.Iterable`. - :data:`ClassVar` is not a class itself, and should not - be used with :func:`isinstance` or :func:`issubclass`. - :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker - might flag the following code as an error:: + .. deprecated:: 3.9 + :class:`collections.abc.Iterable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - enterprise_d = Starship(3000) - enterprise_d.stats = {} # Error, setting class variable on instance - Starship.stats = {} # This is OK +.. class:: Iterator(Iterable[T_co]) - .. versionadded:: 3.5.3 + A generic version of :class:`collections.abc.Iterator`. -.. data:: Final + .. deprecated:: 3.9 + :class:`collections.abc.Iterator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - A special typing construct to indicate to type checkers that a name - cannot be re-assigned or overridden in a subclass. For example:: +.. class:: Generator(Iterator[T_co], Generic[T_co, T_contra, V_co]) - MAX_SIZE: Final = 9000 - MAX_SIZE += 1 # Error reported by type checker + A generator can be annotated by the generic type + ``Generator[YieldType, SendType, ReturnType]``. For example:: - class Connection: - TIMEOUT: Final[int] = 10 + def echo_round() -> Generator[int, float, str]: + sent = yield 0 + while sent >= 0: + sent = yield round(sent) + return 'Done' - class FastConnector(Connection): - TIMEOUT = 1 # Error reported by type checker + Note that unlike many other generics in the typing module, the ``SendType`` + of :class:`Generator` behaves contravariantly, not covariantly or + invariantly. - There is no runtime checking of these properties. See :pep:`591` for - more details. + If your generator will only yield values, set the ``SendType`` and + ``ReturnType`` to ``None``:: - .. versionadded:: 3.8 + def infinite_stream(start: int) -> Generator[int, None, None]: + while True: + yield start + start += 1 -.. data:: AnyStr + Alternatively, annotate your generator as having a return type of + either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - ``AnyStr`` is a type variable defined as - ``AnyStr = TypeVar('AnyStr', str, bytes)``. + def infinite_stream(start: int) -> Iterator[int]: + while True: + yield start + start += 1 - It is meant to be used for functions that may accept any kind of string - without allowing different kinds of strings to mix. For example:: + .. deprecated:: 3.9 + :class:`collections.abc.Generator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - def concat(a: AnyStr, b: AnyStr) -> AnyStr: - return a + b +.. class:: Hashable - concat(u"foo", u"bar") # Ok, output has type 'unicode' - concat(b"foo", b"bar") # Ok, output has type 'bytes' - concat(u"foo", b"bar") # Error, cannot mix unicode and bytes + An alias to :class:`collections.abc.Hashable` -.. data:: TYPE_CHECKING +.. class:: Reversible(Iterable[T_co]) - A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. Usage:: + A generic version of :class:`collections.abc.Reversible`. - if TYPE_CHECKING: - import expensive_mod + .. deprecated:: 3.9 + :class:`collections.abc.Reversible` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - def fun(arg: 'expensive_mod.SomeType') -> None: - local_var: expensive_mod.AnotherType = other_fun() +.. class:: Sized - Note that the first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. + An alias to :class:`collections.abc.Sized` - .. versionadded:: 3.5.2 +Asynchronous programming +"""""""""""""""""""""""" -.. data:: Annotated +.. class:: Coroutine(Awaitable[V_co], Generic[T_co, T_contra, V_co]) - A type, introduced in :pep:`593` (``Flexible function and variable - annotations``), to decorate existing types with context-specific metadata - (possibly multiple pieces of it, as ``Annotated`` is variadic). - Specifically, a type ``T`` can be annotated with metadata ``x`` via the - typehint ``Annotated[T, x]``. This metadata can be used for either static - analysis or at runtime. If a library (or tool) encounters a typehint - ``Annotated[T, x]`` and has no special logic for metadata ``x``, it - should ignore it and simply treat the type as ``T``. Unlike the - ``no_type_check`` functionality that currently exists in the ``typing`` - module which completely disables typechecking annotations on a function - or a class, the ``Annotated`` type allows for both static typechecking - of ``T`` (e.g., via mypy or Pyre, which can safely ignore ``x``) - together with runtime access to ``x`` within a specific application. + A generic version of :class:`collections.abc.Coroutine`. + The variance and order of type variables + correspond to those of :class:`Generator`, for example:: - Ultimately, the responsibility of how to interpret the annotations (if - at all) is the responsibility of the tool or library encountering the - ``Annotated`` type. A tool or library encountering an ``Annotated`` type - can scan through the annotations to determine if they are of interest - (e.g., using ``isinstance()``). + from collections.abc import Coroutine + c = None # type: Coroutine[list[str], str, int] + ... + x = c.send('hi') # type: list[str] + async def bar() -> None: + x = await c # type: int - When a tool or a library does not support annotations or encounters an - unknown annotation it should just ignore it and treat annotated type as - the underlying type. + .. versionadded:: 3.5.3 - It's up to the tool consuming the annotations to decide whether the - client is allowed to have several annotations on one type and how to - merge those annotations. + .. deprecated:: 3.9 + :class:`collections.abc.Coroutine` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - Since the ``Annotated`` type allows you to put several annotations of - the same (or different) type(s) on any node, the tools or libraries - consuming those annotations are in charge of dealing with potential - duplicates. For example, if you are doing value range analysis you might - allow this:: +.. class:: AsyncGenerator(AsyncIterator[T_co], Generic[T_co, T_contra]) - T1 = Annotated[int, ValueRange(-10, 5)] - T2 = Annotated[T1, ValueRange(-20, 3)] + An async generator can be annotated by the generic type + ``AsyncGenerator[YieldType, SendType]``. For example:: - Passing ``include_extras=True`` to :func:`get_type_hints` lets one - access the extra annotations at runtime. + async def echo_round() -> AsyncGenerator[int, float]: + sent = yield 0 + while sent >= 0.0: + rounded = await round(sent) + sent = yield rounded - The details of the syntax: + Unlike normal generators, async generators cannot return a value, so there + is no ``ReturnType`` type parameter. As with :class:`Generator`, the + ``SendType`` behaves contravariantly. - * The first argument to ``Annotated`` must be a valid type + If your generator will only yield values, set the ``SendType`` to + ``None``:: - * Multiple type annotations are supported (``Annotated`` supports variadic - arguments):: + async def infinite_stream(start: int) -> AsyncGenerator[int, None]: + while True: + yield start + start = await increment(start) - Annotated[int, ValueRange(3, 10), ctype("char")] + Alternatively, annotate your generator as having a return type of + either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - * ``Annotated`` must be called with at least two arguments ( - ``Annotated[int]`` is not valid) + async def infinite_stream(start: int) -> AsyncIterator[int]: + while True: + yield start + start = await increment(start) - * The order of the annotations is preserved and matters for equality - checks:: + .. versionadded:: 3.6.1 - Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ - int, ctype("char"), ValueRange(3, 10) - ] + .. deprecated:: 3.9 + :class:`collections.abc.AsyncGenerator` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. - * Nested ``Annotated`` types are flattened, with metadata ordered - starting with the innermost annotation:: +.. class:: AsyncIterable(Generic[T_co]) - Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ - int, ValueRange(3, 10), ctype("char") - ] + A generic version of :class:`collections.abc.AsyncIterable`. - * Duplicated annotations are not removed:: + .. versionadded:: 3.5.2 - Annotated[int, ValueRange(3, 10)] != Annotated[ - int, ValueRange(3, 10), ValueRange(3, 10) - ] + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. - * ``Annotated`` can be used with nested and generic aliases:: +.. class:: AsyncIterator(AsyncIterable[T_co]) - T = TypeVar('T') - Vec = Annotated[List[Tuple[T, T]], MaxLen(10)] - V = Vec[int] + A generic version of :class:`collections.abc.AsyncIterator`. - V == Annotated[List[Tuple[int, int]], MaxLen(10)] + .. versionadded:: 3.5.2 + + .. deprecated:: 3.9 + :class:`collections.abc.AsyncIterator` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. + +.. class:: Awaitable(Generic[T_co]) + + A generic version of :class:`collections.abc.Awaitable`. + + .. versionadded:: 3.5.2 + + .. deprecated:: 3.9 + :class:`collections.abc.Awaitable` now supports ``[]``. See :pep:`585` + and :ref:`types-genericalias`. + + +Context manager types +""""""""""""""""""""" + +.. class:: ContextManager(Generic[T_co]) + + A generic version of :class:`contextlib.AbstractContextManager`. + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.0 + + .. deprecated:: 3.9 + :class:`contextlib.AbstractContextManager` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. + +.. class:: AsyncContextManager(Generic[T_co]) + + A generic version of :class:`contextlib.AbstractAsyncContextManager`. + + .. versionadded:: 3.5.4 + .. versionadded:: 3.6.2 + + .. deprecated:: 3.9 + :class:`contextlib.AbstractAsyncContextManager` now supports ``[]``. See + :pep:`585` and :ref:`types-genericalias`. + +Protocols +--------- + +These protocols are decorated with :func:`runtime_checkable`. + +.. class:: SupportsAbs + + An ABC with one abstract method ``__abs__`` that is covariant + in its return type. + +.. class:: SupportsBytes + + An ABC with one abstract method ``__bytes__``. + +.. class:: SupportsComplex + + An ABC with one abstract method ``__complex__``. + +.. class:: SupportsFloat + + An ABC with one abstract method ``__float__``. + +.. class:: SupportsIndex + + An ABC with one abstract method ``__index__``. + + .. versionadded:: 3.8 + +.. class:: SupportsInt + + An ABC with one abstract method ``__int__``. + +.. class:: SupportsRound + + An ABC with one abstract method ``__round__`` + that is covariant in its return type. + +Functions and decorators +------------------------ + +.. function:: cast(typ, val) + + Cast a value to a type. + + This returns the value unchanged. To the type checker this + signals that the return value has the designated type, but at + runtime we intentionally don't check anything (we want this + to be as fast as possible). + +.. decorator:: overload + + The ``@overload`` decorator allows describing functions and methods + that support multiple different combinations of argument types. A series + of ``@overload``-decorated definitions must be followed by exactly one + non-``@overload``-decorated definition (for the same function/method). + The ``@overload``-decorated definitions are for the benefit of the + type checker only, since they will be overwritten by the + non-``@overload``-decorated definition, while the latter is used at + runtime but should be ignored by a type checker. At runtime, calling + a ``@overload``-decorated function directly will raise + :exc:`NotImplementedError`. An example of overload that gives a more + precise type than can be expressed using a union or a type variable:: + + @overload + def process(response: None) -> None: + ... + @overload + def process(response: int) -> tuple[int, str]: + ... + @overload + def process(response: bytes) -> str: + ... + def process(response): + + + See :pep:`484` for details and comparison with other typing semantics. + +.. decorator:: final + + A decorator to indicate to type checkers that the decorated method + cannot be overridden, and the decorated class cannot be subclassed. + For example:: + + class Base: + @final + def done(self) -> None: + ... + class Sub(Base): + def done(self) -> None: # Error reported by type checker + ... + + @final + class Leaf: + ... + class Other(Leaf): # Error reported by type checker + ... + + There is no runtime checking of these properties. See :pep:`591` for + more details. + + .. versionadded:: 3.8 + +.. decorator:: no_type_check + + Decorator to indicate that annotations are not type hints. + + This works as class or function :term:`decorator`. With a class, it + applies recursively to all methods defined in that class (but not + to methods defined in its superclasses or subclasses). + + This mutates the function(s) in place. + +.. decorator:: no_type_check_decorator + + Decorator to give another decorator the :func:`no_type_check` effect. + + This wraps the decorator with something that wraps the decorated + function in :func:`no_type_check`. + +.. decorator:: type_check_only + + Decorator to mark a class or function to be unavailable at runtime. + + This decorator is itself not available at runtime. It is mainly + intended to mark classes that are defined in type stub files if + an implementation returns an instance of a private class:: + + @type_check_only + class Response: # private or not available at runtime + code: int + def get_header(self, name: str) -> str: ... + + def fetch_response() -> Response: ... + + Note that returning instances of private classes is not recommended. + It is usually preferable to make such classes public. + +Introspection helpers +--------------------- + +.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) + + Return a dictionary containing type hints for a function, method, module + or class object. + + This is often the same as ``obj.__annotations__``. In addition, + forward references encoded as string literals are handled by evaluating + them in ``globals`` and ``locals`` namespaces. If necessary, + ``Optional[t]`` is added for function and method annotations if a default + value equal to ``None`` is set. For a class ``C``, return + a dictionary constructed by merging all the ``__annotations__`` along + ``C.__mro__`` in reverse order. + + The function recursively replaces all ``Annotated[T, ...]`` with ``T``, + unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for + more information). For example:: + + class Student(NamedTuple): + name: Annotated[str, 'some marker'] + + get_type_hints(Student) == {'name': str} + get_type_hints(Student, include_extras=False) == {'name': str} + get_type_hints(Student, include_extras=True) == { + 'name': Annotated[str, 'some marker'] + } + + .. versionchanged:: 3.9 + Added ``include_extras`` parameter as part of :pep:`593`. + +.. function:: get_args(tp) +.. function:: get_origin(tp) + + Provide basic introspection for generic types and special typing forms. + + For a typing object of the form ``X[Y, Z, ...]`` these functions return + ``X`` and ``(Y, Z, ...)``. If ``X`` is a generic alias for a builtin or + :mod:`collections` class, it gets normalized to the original class. + If ``X`` is a :class:`Union` or :class:`Literal` contained in another + generic type, the order of ``(Y, Z, ...)`` may be different from the order + of the original arguments ``[Y, Z, ...]`` due to type caching. + For unsupported objects return ``None`` and ``()`` correspondingly. + Examples:: + + assert get_origin(Dict[str, int]) is dict + assert get_args(Dict[int, str]) == (int, str) + + assert get_origin(Union[int, str]) is Union + assert get_args(Union[int, str]) == (int, str) + + .. versionadded:: 3.8 + +.. class:: ForwardRef + + A class used for internal typing representation of string forward references. + For example, ``List["SomeClass"]`` is implicitly transformed into + ``List[ForwardRef("SomeClass")]``. This class should not be instantiated by + a user, but may be used by introspection tools. + + .. note:: + :pep:`585` generic types such as ``list["SomeClass"]`` will not be + implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus + will not automatically resolve to ``list[SomeClass]``. + + .. versionadded:: 3.7.4 + +Constant +-------- + +.. data:: TYPE_CHECKING + + A special constant that is assumed to be ``True`` by 3rd party static + type checkers. It is ``False`` at runtime. Usage:: + + if TYPE_CHECKING: + import expensive_mod + + def fun(arg: 'expensive_mod.SomeType') -> None: + local_var: expensive_mod.AnotherType = other_fun() + + The first type annotation must be enclosed in quotes, making it a + "forward reference", to hide the ``expensive_mod`` reference from the + interpreter runtime. Type annotations for local variables are not + evaluated, so the second annotation does not need to be enclosed in quotes. + + .. note:: + + If ``from __future__ import annotations`` is used in Python 3.7 or later, + annotations are not evaluated at function definition time. + Instead, they are stored as strings in ``__annotations__``, + This makes it unnecessary to use quotes around the annotation. + (see :pep:`563`). + + .. versionadded:: 3.5.2 - .. versionadded:: 3.9 diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst index e650bb1e23e03e..24a18c68484686 100644 --- a/Doc/library/unittest.mock-examples.rst +++ b/Doc/library/unittest.mock-examples.rst @@ -893,7 +893,7 @@ Here's an example implementation: ... def __call__(self, /, *args, **kwargs): ... args = deepcopy(args) ... kwargs = deepcopy(kwargs) - ... return super(CopyingMock, self).__call__(*args, **kwargs) + ... return super().__call__(*args, **kwargs) ... >>> c = CopyingMock(return_value=None) >>> arg = set() diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst index 3643d1ad96ed11..b3e71705801088 100644 --- a/Doc/library/unittest.mock.rst +++ b/Doc/library/unittest.mock.rst @@ -30,7 +30,7 @@ module and class level attributes within the scope of a test, along with some examples of how to use :class:`Mock`, :class:`MagicMock` and :func:`patch`. -Mock is very easy to use and is designed for use with :mod:`unittest`. Mock +Mock is designed for use with :mod:`unittest` and is based on the 'action -> assertion' pattern instead of 'record -> replay' used by many mocking frameworks. @@ -327,8 +327,8 @@ the *new_callable* argument to :func:`patch`. .. method:: assert_called_once_with(*args, **kwargs) - Assert that the mock was called exactly once and that that call was - with the specified arguments. + Assert that the mock was called exactly once and that call was with the + specified arguments. >>> mock = Mock(return_value=None) >>> mock('foo', bar='baz') @@ -647,6 +647,9 @@ the *new_callable* argument to :func:`patch`. arguments and make more complex assertions. See :ref:`calls as tuples `. + .. versionchanged:: 3.8 + Added ``args`` and ``kwargs`` properties. + .. attribute:: call_args_list diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index b2e16cf331e036..392cd2473155fe 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -73,7 +73,7 @@ test runner for those new to unit testing. For production environments it is recommended that tests be driven by a continuous integration system such as `Buildbot `_, `Jenkins `_ - or `Hudson `_. + or `Travis-CI `_, or `AppVeyor `_. .. _unittest-minimal-example: @@ -330,7 +330,10 @@ Test modules and packages can customize test loading and discovery by through the `load_tests protocol`_. .. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages `. + Test discovery supports :term:`namespace packages ` + for the start directory. Note that you need to specify the top level + directory too (e.g. + ``python -m unittest discover -s root/namespace -t root``). .. _organizing-tests: @@ -593,8 +596,10 @@ The following decorators and exception implement test skipping and expected fail .. decorator:: expectedFailure - Mark the test as an expected failure. If the test fails it will be - considered a success. If the test passes, it will be considered a failure. + Mark the test as an expected failure or error. If the test fails or errors + in the test function itself (rather than in one of the :dfn:`test fixture` + methods) then it will be considered a success. If the test passes, it will + be considered a failure. .. exception:: SkipTest(reason) @@ -896,8 +901,7 @@ Test cases .. method:: assertIs(first, second, msg=None) assertIsNot(first, second, msg=None) - Test that *first* and *second* evaluate (or don't evaluate) to the - same object. + Test that *first* and *second* are (or are not) the same object. .. versionadded:: 3.1 @@ -1088,7 +1092,8 @@ Test cases If given, *logger* should be a :class:`logging.Logger` object or a :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages. + logger, which will catch all messages that were not blocked by a + non-propagating descendent logger. If given, *level* should be either a numeric logging level or its string equivalent (for example either ``"ERROR"`` or @@ -1476,11 +1481,11 @@ Test cases after :meth:`setUpClass` if :meth:`setUpClass` raises an exception. It is responsible for calling all the cleanup functions added by - :meth:`addCleanupClass`. If you need cleanup functions to be called + :meth:`addClassCleanup`. If you need cleanup functions to be called *prior* to :meth:`tearDownClass` then you can call - :meth:`doCleanupsClass` yourself. + :meth:`doClassCleanups` yourself. - :meth:`doCleanupsClass` pops methods off the stack of cleanup + :meth:`doClassCleanups` pops methods off the stack of cleanup functions one at a time, so it can be called at any time. .. versionadded:: 3.8 @@ -1848,11 +1853,15 @@ Loading and running tests .. versionchanged:: 3.4 Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - Discovery works for :term:`namespace packages `. - Paths are sorted before being imported so that execution order is - the same even if the underlying file system's ordering is not - dependent on file name. + not errors. + + .. versionchanged:: 3.4 + *start_dir* can be a :term:`namespace packages `. + + .. versionchanged:: 3.4 + Paths are sorted before being imported so that execution order is the + same even if the underlying file system's ordering is not dependent + on file name. .. versionchanged:: 3.5 Found packages are now checked for ``load_tests`` regardless of @@ -1945,7 +1954,7 @@ Loading and running tests A list containing 2-tuples of :class:`TestCase` instances and strings holding formatted tracebacks. Each tuple represents an expected failure - of the test case. + or error of the test case. .. attribute:: unexpectedSuccesses @@ -2071,8 +2080,8 @@ Loading and running tests .. method:: addExpectedFailure(test, err) - Called when the test case *test* fails, but was marked with the - :func:`expectedFailure` decorator. + Called when the test case *test* fails or errors, but was marked with + the :func:`expectedFailure` decorator. The default implementation appends a tuple ``(test, formatted_err)`` to the instance's :attr:`expectedFailures` attribute, where *formatted_err* diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst index 02f0c01e224ddf..c03765259c3d54 100644 --- a/Doc/library/urllib.parse.rst +++ b/Doc/library/urllib.parse.rst @@ -68,15 +68,15 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + >>> from urllib.parse import urlparse + >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', + >>> urlparse('www.cwi.nl/%7Eguido/Python.html') + ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', + >>> urlparse('help/Python.html') + ParseResult(scheme='', netloc='', path='help/Python.html', params='', query='', fragment='') The *scheme* argument gives the default addressing scheme, to be @@ -138,14 +138,14 @@ or on combining URL components into a URL string. .. doctest:: :options: +NORMALIZE_WHITESPACE - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') + >>> from urllib.parse import urlparse + >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') + >>> u + ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') + >>> u._replace(scheme='http') + ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', + params='', query='', fragment='') .. versionchanged:: 3.2 @@ -165,7 +165,7 @@ or on combining URL components into a URL string. now raise :exc:`ValueError`. -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a @@ -190,6 +190,9 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the + query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` parameter set to ``True``) to convert such dictionaries into query strings. @@ -201,8 +204,14 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.9.2 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.9.2 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None) +.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') Parse a query string given as a string argument (data of type :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of @@ -226,6 +235,8 @@ or on combining URL components into a URL string. read. If set, then throws a :exc:`ValueError` if there are more than *max_num_fields* fields read. + The optional argument *separator* is the symbol to use for separating the query arguments. It defaults to ``&``. + Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into query strings. @@ -235,6 +246,12 @@ or on combining URL components into a URL string. .. versionchanged:: 3.8 Added *max_num_fields* parameter. + .. versionchanged:: 3.9.2 + Added *separator* parameter with the default value of ``&``. Python + versions earlier than Python 3.9.2 allowed using both ``;`` and ``&`` as + query parameter separator. This has been changed to allow only a single + separator key, with ``&`` as the default separator. + .. function:: urlunparse(parts) @@ -294,6 +311,9 @@ or on combining URL components into a URL string. ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is decomposed before parsing, no error will be raised. + Following the `WHATWG spec`_ that updates RFC 3986, ASCII newline + ``\n``, ``\r`` and tab ``\t`` characters are stripped from the URL. + .. versionchanged:: 3.6 Out-of-range port numbers now raise :exc:`ValueError`, instead of returning :const:`None`. @@ -302,6 +322,10 @@ or on combining URL components into a URL string. Characters that affect netloc parsing under NFKC normalization will now raise :exc:`ValueError`. + .. versionchanged:: 3.9.5 + ASCII newline and tab characters are stripped from the URL. + +.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser .. function:: urlunsplit(parts) @@ -656,6 +680,10 @@ task isn't already covered by the URL parsing functions above. .. seealso:: + `WHATWG`_ - URL Living standard + Working Group for the URL Standard that defines URLs, domains, IP addresses, the + application/x-www-form-urlencoded format, and their API. + :rfc:`3986` - Uniform Resource Identifiers This is the current standard (STD66). Any changes to urllib.parse module should conform to this. Certain deviations could be observed, which are @@ -679,3 +707,5 @@ task isn't already covered by the URL parsing functions above. :rfc:`1738` - Uniform Resource Locators (URL) This specifies the formal syntax and semantics of absolute URLs. + +.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 03712c1f4a6eea..785ecf8fafdcc5 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -946,7 +946,7 @@ tracking URIs for which authentication credentials should always be sent. If *is_authenticated* is specified as ``True``, *realm* is ignored. -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) +.. method:: HTTPPasswordMgrWithPriorAuth.find_user_password(realm, authuri) Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst index 8abadc4df3cac8..e020570896acb5 100644 --- a/Doc/library/venv.rst +++ b/Doc/library/venv.rst @@ -400,7 +400,7 @@ subclass which installs setuptools and pip into a created virtual environment:: :param context: The information for the virtual environment creation request being processed. """ - url = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py' + url = 'https://bootstrap.pypa.io/get-pip.py' self.install_script(context, 'pip', url) def main(args=None): diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst index a481a3509d4ec8..9c1743cad23cb7 100644 --- a/Doc/library/warnings.rst +++ b/Doc/library/warnings.rst @@ -491,7 +491,7 @@ Available Functions Available Context Managers -------------------------- -.. class:: catch_warnings(\*, record=False, module=None) +.. class:: catch_warnings(*, record=False, module=None) A context manager that copies and, upon exit, restores the warnings filter and the :func:`showwarning` function. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst index 12eb985c344359..b88543e4453723 100644 --- a/Doc/library/weakref.rst +++ b/Doc/library/weakref.rst @@ -163,14 +163,6 @@ Extension types can easily be made to support weak references; see application without adding attributes to those objects. This can be especially useful with objects that override attribute accesses. - .. note:: - - Caution: Because a :class:`WeakKeyDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakKeyDictionary` because actions - performed by the program during iteration may cause items in the - dictionary to vanish "by magic" (as a side effect of garbage collection). - .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. @@ -192,14 +184,6 @@ than needed. Mapping class that references values weakly. Entries in the dictionary will be discarded when no strong reference to the value exists any more. - .. note:: - - Caution: Because a :class:`WeakValueDictionary` is built on top of a Python - dictionary, it must not change size when iterating over it. This can be - difficult to ensure for a :class:`WeakValueDictionary` because actions performed - by the program during iteration may cause items in the dictionary to vanish "by - magic" (as a side effect of garbage collection). - .. versionchanged:: 3.9 Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. @@ -398,7 +382,7 @@ the referent is accessed:: class ExtendedRef(weakref.ref): def __init__(self, ob, callback=None, /, **annotations): - super(ExtendedRef, self).__init__(ob, callback) + super().__init__(ob, callback) self.__counter = 0 for k, v in annotations.items(): setattr(self, k, v) @@ -407,7 +391,7 @@ the referent is accessed:: """Return a pair containing the referent and the number of times the reference has been called. """ - ob = super(ExtendedRef, self).__call__() + ob = super().__call__() if ob is not None: self.__counter += 1 ob = (ob, self.__counter) diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst index dccb7db27e90cc..487856a3ac6c60 100644 --- a/Doc/library/winreg.rst +++ b/Doc/library/winreg.rst @@ -791,7 +791,7 @@ integer handle, and also disconnect the Windows handle from the handle object. .. method:: PyHKEY.__enter__() - PyHKEY.__exit__(\*exc_info) + PyHKEY.__exit__(*exc_info) The HKEY object implements :meth:`~object.__enter__` and :meth:`~object.__exit__` and thus supports the context protocol for the diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst index 1e30aa4a898c13..e92a689de0b9ba 100644 --- a/Doc/library/wsgiref.rst +++ b/Doc/library/wsgiref.rst @@ -480,8 +480,8 @@ input, output, and error streams. rarely used and is not guaranteed by WSGI. On IIS<7, though, the setting can only be made on a vhost level, affecting all other script mappings, many of which break when exposed to the ``PATH_TRANSLATED`` bug. - For this reason IIS<7 is almost never deployed with the fix. (Even IIS7 - rarely uses it because there is still no UI for it.) + For this reason IIS<7 is almost never deployed with the fix (Even IIS7 + rarely uses it because there is still no UI for it.). There is no way for CGI code to tell whether the option was set, so a separate handler class is provided. It is used in the same way as diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst index 2c78cd939243a8..bf72c46561b7c7 100644 --- a/Doc/library/xml.dom.minidom.rst +++ b/Doc/library/xml.dom.minidom.rst @@ -132,7 +132,7 @@ module documentation. This section lists the differences between the API and ... # Work with dom. -.. method:: Node.writexml(writer, indent="", addindent="", newl="", +.. method:: Node.writexml(writer, indent="", addindent="", newl="", \ encoding=None, standalone=None) Write XML to the writer object. The writer receives texts but not bytes as input, @@ -174,7 +174,7 @@ module documentation. This section lists the differences between the API and The :meth:`toxml` method now preserves the attribute order specified by the user. -.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, +.. method:: Node.toprettyxml(indent="\\t", newl="\\n", encoding=None, \ standalone=None) Return a pretty-printed version of the document. *indent* specifies the diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 658bc3a54f86e5..f4bccf6609810e 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -15,6 +15,8 @@ for parsing and creating XML data. .. versionchanged:: 3.3 This module will use a fast implementation whenever available. + +.. deprecated:: 3.3 The :mod:`xml.etree.cElementTree` module is deprecated. @@ -249,12 +251,18 @@ We can remove elements using :meth:`Element.remove`. Let's say we want to remove all countries with a rank higher than 50:: >>> for country in root.findall('country'): + ... # using root.findall() to avoid removal during traversal ... rank = int(country.find('rank').text) ... if rank > 50: ... root.remove(country) ... >>> tree.write('output.xml') +Note that concurrent modification while iterating can lead to problems, +just like when iterating and modifying Python lists or dicts. +Therefore, the example first collects all matching elements with +``root.findall()``, and only then iterates over the list of matches. + Our XML now looks like this: .. code-block:: xml @@ -816,16 +824,25 @@ Functions loader fails, it can return None or raise an exception. -.. function:: xml.etree.ElementInclude.include( elem, loader=None) +.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ + max_depth=6) This function expands XInclude directives. *elem* is the root element. *loader* is an optional resource loader. If omitted, it defaults to :func:`default_loader`. If given, it should be a callable that implements the same interface as - :func:`default_loader`. Returns the expanded resource. If the parse mode is + :func:`default_loader`. *base_url* is base URL of the original file, to resolve + relative include file references. *max_depth* is the maximum number of recursive + inclusions. Limited to reduce the risk of malicious content explosion. Pass a + negative value to disable the limitation. + + Returns the expanded resource. If the parse mode is ``"xml"``, this is an ElementTree instance. If the parse mode is "text", this is a Unicode string. If the loader fails, it can return None or raise an exception. + .. versionadded:: 3.9 + The *base_url* and *max_depth* parameters. + .. _elementtree-element-objects: diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst index fb86b6f5564d76..1981cab7cd438d 100644 --- a/Doc/library/xml.rst +++ b/Doc/library/xml.rst @@ -20,7 +20,7 @@ Python's interfaces for processing XML are grouped in the ``xml`` package. The XML modules are not secure against erroneous or maliciously constructed data. If you need to parse untrusted or unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defused-packages` sections. + :ref:`defusedxml-package` sections. It is important to note that modules in the :mod:`xml` package require that there be at least one SAX-compliant XML parser available. The Expat parser is @@ -113,9 +113,9 @@ decompression bomb The documentation for `defusedxml`_ on PyPI has further information about all known attack vectors with examples and references. -.. _defused-packages: +.. _defusedxml-package: -The :mod:`defusedxml` and :mod:`defusedexpat` Packages +The :mod:`defusedxml` Package ------------------------------------------------------ `defusedxml`_ is a pure Python package with modified subclasses of all stdlib @@ -124,16 +124,8 @@ package is recommended for any server code that parses untrusted XML data. The package also ships with example exploits and extended documentation on more XML exploits such as XPath injection. -`defusedexpat`_ provides a modified libexpat and a patched -:mod:`pyexpat` module that have countermeasures against entity expansion -DoS attacks. The :mod:`defusedexpat` module still allows a sane and configurable amount of entity -expansions. The modifications may be included in some future release of Python, -but will not be included in any bugfix releases of -Python because they break backward compatibility. - .. _defusedxml: https://pypi.org/project/defusedxml/ -.. _defusedexpat: https://pypi.org/project/defusedexpat/ .. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs .. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb .. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst index 1b6f2e7bd15f00..3a4c12a73acd72 100644 --- a/Doc/library/zoneinfo.rst +++ b/Doc/library/zoneinfo.rst @@ -300,7 +300,7 @@ The behavior of a ``ZoneInfo`` file depends on how it was constructed: constructed from ``ZoneInfo("Europe/Berlin")``, one would expect the following behavior: - .. code-block:: + .. code-block:: pycon >>> a = ZoneInfo("Europe/Berlin") >>> b = pickle.loads(europe_berlin_pkl) @@ -314,7 +314,7 @@ The behavior of a ``ZoneInfo`` file depends on how it was constructed: constructed from ``ZoneInfo.no_cache("Europe/Berlin")``, one would expect the following behavior: - .. code-block:: + .. code-block:: pycon >>> a = ZoneInfo("Europe/Berlin") >>> b = pickle.loads(europe_berlin_pkl_nc) diff --git a/Doc/license.rst b/Doc/license.rst index 472a5cf3d88b34..d459ff664ceed3 100644 --- a/Doc/license.rst +++ b/Doc/license.rst @@ -72,6 +72,19 @@ make these releases possible. Terms and conditions for accessing or otherwise using Python ============================================================ +Python software and documentation are licensed under the +:ref:`PSF License Agreement `. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Agreement +and the :ref:`Zero-Clause BSD license `. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. +See :ref:`OtherLicenses` for an incomplete list of these licenses. + + +.. _PSF-license: PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------ @@ -87,7 +100,7 @@ PSF LICENSE AGREEMENT FOR PYTHON |release| analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python |release| alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2020 Python Software Foundation; All Rights + copyright, i.e., "Copyright © 2001-2021 Python Software Foundation; All Rights Reserved" are retained in Python |release| alone or in any derivative version prepared by Licensee. @@ -258,6 +271,27 @@ CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 SOFTWARE. +.. _BSD0: + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON |release| DOCUMENTATION +---------------------------------------------------------------------- + +.. parsed-literal:: + + Permission to use, copy, modify, and/or distribute this software for any + purpose with or without fee is hereby granted. + + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH + REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY + AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, + INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM + LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR + OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THIS SOFTWARE. + + +.. _OtherLicenses: + Licenses and Acknowledgements for Incorporated Software ======================================================= @@ -889,7 +923,7 @@ libmpdec The :mod:`_decimal` module is built using an included copy of the libmpdec library unless the build is configured ``--with-system-libmpdec``:: - Copyright (c) 2008-2016 Stefan Krah. All rights reserved. + Copyright (c) 2008-2020 Stefan Krah. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions @@ -921,7 +955,7 @@ W3C C14N test suite The C14N 2.0 test suite in the :mod:`test` package (``Lib/test/xmltestdata/c14n-20/``) was retrieved from the W3C website at https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the -3-clause BSD license: +3-clause BSD license:: Copyright (c) 2013 W3C(R) (MIT, ERCIM, Keio, Beihang), All Rights Reserved. diff --git a/Doc/make.bat b/Doc/make.bat index 6f8f172e95eb8e..7fde0636427713 100644 --- a/Doc/make.bat +++ b/Doc/make.bat @@ -13,7 +13,7 @@ if not defined SPHINXBUILD ( %PYTHON% -c "import sphinx" > nul 2> nul if errorlevel 1 ( echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install sphinx + %PYTHON% -m pip install sphinx==2.2.0 if errorlevel 1 exit /B ) set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" diff --git a/Doc/myfile.bz2 b/Doc/myfile.bz2 deleted file mode 100644 index 7ada20f60926b4..00000000000000 Binary files a/Doc/myfile.bz2 and /dev/null differ diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index c14e7c79fe14cf..75424162a48b07 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -44,7 +44,8 @@ executed:: Summarizing: -.. productionlist:: + +.. productionlist:: python-grammar compound_stmt: `if_stmt` : | `while_stmt` : | `for_stmt` @@ -89,7 +90,7 @@ The :keyword:`!if` statement The :keyword:`if` statement is used for conditional execution: -.. productionlist:: +.. productionlist:: python-grammar if_stmt: "if" `assignment_expression` ":" `suite` : ("elif" `assignment_expression` ":" `suite`)* : ["else" ":" `suite`] @@ -115,7 +116,7 @@ The :keyword:`!while` statement The :keyword:`while` statement is used for repeated execution as long as an expression is true: -.. productionlist:: +.. productionlist:: python-grammar while_stmt: "while" `assignment_expression` ":" `suite` : ["else" ":" `suite`] @@ -151,7 +152,7 @@ The :keyword:`!for` statement The :keyword:`for` statement is used to iterate over the elements of a sequence (such as a string, tuple or list) or other iterable object: -.. productionlist:: +.. productionlist:: python-grammar for_stmt: "for" `target_list` "in" `expression_list` ":" `suite` : ["else" ":" `suite`] @@ -234,7 +235,7 @@ The :keyword:`!try` statement The :keyword:`try` statement specifies exception handlers and/or cleanup code for a group of statements: -.. productionlist:: +.. productionlist:: python-grammar try_stmt: `try1_stmt` | `try2_stmt` try1_stmt: "try" ":" `suite` : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ @@ -253,7 +254,8 @@ present, must be last; it matches any exception. For an except clause with an expression, that expression is evaluated, and the clause matches the exception if the resulting object is "compatible" with the exception. An object is compatible with an exception if it is the class or a base class of the exception -object or a tuple containing an item compatible with the exception. +object, or a tuple containing an item that is the class or a base class of +the exception object. If no except clause matches the exception, the search for an exception handler continues in the surrounding code and on the invocation stack. [#]_ @@ -390,7 +392,7 @@ methods defined by a context manager (see section :ref:`context-managers`). This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` usage patterns to be encapsulated for convenient reuse. -.. productionlist:: +.. productionlist:: python-grammar with_stmt: "with" `with_item` ("," `with_item`)* ":" `suite` with_item: `expression` ["as" `target`] @@ -503,12 +505,11 @@ Function definitions A function definition defines a user-defined function object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` decorators: `decorator`+ decorator: "@" `assignment_expression` NEWLINE - dotted_name: `identifier` ("." `identifier`)* parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] : | `parameter_list_no_posonly` parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] @@ -585,19 +586,25 @@ e.g.:: return penguin .. index:: + single: / (slash); function definition single: * (asterisk); function definition single: **; function definition Function call semantics are described in more detail in section :ref:`calls`. A function call always assigns values to all parameters mentioned in the parameter -list, either from position arguments, from keyword arguments, or from default +list, either from positional arguments, from keyword arguments, or from default values. If the form "``*identifier``" is present, it is initialized to a tuple receiving any excess positional parameters, defaulting to the empty tuple. If the form "``**identifier``" is present, it is initialized to a new ordered mapping receiving any excess keyword arguments, defaulting to a new empty mapping of the same type. Parameters after "``*``" or "``*identifier``" are keyword-only parameters and may only be passed -used keyword arguments. +by keyword arguments. Parameters before "``/``" are positional-only parameters +and may only be passed by positional arguments. + +.. versionchanged:: 3.8 + The ``/`` function parameter syntax may be used to indicate positional-only + parameters. See :pep:`570` for details. .. index:: pair: function; annotations @@ -670,7 +677,7 @@ Class definitions A class definition defines a class object (see section :ref:`types`): -.. productionlist:: +.. productionlist:: python-grammar classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` inheritance: "(" [`argument_list`] ")" classname: `identifier` @@ -762,7 +769,7 @@ Coroutines Coroutine function definition ----------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" : ["->" `expression`] ":" `suite` @@ -795,15 +802,15 @@ An example of a coroutine function:: The :keyword:`!async for` statement ----------------------------------- -.. productionlist:: +.. productionlist:: python-grammar async_for_stmt: "async" `for_stmt` -An :term:`asynchronous iterable` is able to call asynchronous code in its -*iter* implementation, and :term:`asynchronous iterator` can call asynchronous -code in its *next* method. +An :term:`asynchronous iterable` provides an ``__aiter__`` method that directly +returns an :term:`asynchronous iterator`, which can call asynchronous code in +its ``__anext__`` method. The ``async for`` statement allows convenient iteration over asynchronous -iterators. +iterables. The following code:: @@ -840,7 +847,7 @@ body of a coroutine function. The :keyword:`!async with` statement ------------------------------------ -.. productionlist:: +.. productionlist:: python-grammar async_with_stmt: "async" `with_stmt` An :term:`asynchronous context manager` is a :term:`context manager` that is diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index c5a7f046992dd1..403012307004ad 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -187,6 +187,24 @@ Ellipsis related to mathematical numbers, but subject to the limitations of numerical representation in computers. + The string representations of the numeric classes, computed by + :meth:`__repr__` and :meth:`__str__`, have the following + properties: + + * They are valid numeric literals which, when passed to their + class constructor, produce an object having the value of the + original numeric. + + * The representation is in base 10, when possible. + + * Leading zeros, possibly excepting a single zero before a + decimal point, are not shown. + + * Trailing zeros, possibly excepting a single zero after a + decimal point, are not shown. + + * A sign is shown only when the number is negative. + Python distinguishes between integers, floating point numbers, and complex numbers: @@ -199,7 +217,6 @@ Ellipsis There are two types of integers: Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) memory only. For the purpose of shift and mask operations, a binary representation is assumed, and negative numbers are represented in a variant of @@ -988,6 +1005,9 @@ Internal types :attr:`f_lasti` gives the precise instruction (this is an index into the bytecode string of the code object). + Accessing ``f_code`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. + .. index:: single: f_trace (frame attribute) single: f_trace_lines (frame attribute) @@ -1072,6 +1092,9 @@ Internal types :keyword:`try` statement with no matching except clause or with a finally clause. + Accessing ``tb_frame`` raises an :ref:`auditing event ` + ``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. + .. index:: single: tb_next (traceback attribute) @@ -1377,12 +1400,14 @@ Basic customization context (e.g., in the condition of an ``if`` statement), Python will call :func:`bool` on the value to determine if the result is true or false. - By default, :meth:`__ne__` delegates to :meth:`__eq__` and - inverts the result unless it is ``NotImplemented``. There are no other - implied relationships among the comparison operators, for example, - the truth of ``(x` ``object.__getattr__`` with arguments + ``obj`` and ``name``. + .. method:: object.__setattr__(self, name, value) @@ -1551,12 +1582,24 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. call the base class method with the same name, for example, ``object.__setattr__(self, name, value)``. + .. audit-event:: object.__setattr__ obj,name,value object.__setattr__ + + For certain sensitive attribute assignments, raises an + :ref:`auditing event ` ``object.__setattr__`` with arguments + ``obj``, ``name``, ``value``. + .. method:: object.__delattr__(self, name) Like :meth:`__setattr__` but for attribute deletion instead of assignment. This should only be implemented if ``del obj.name`` is meaningful for the object. + .. audit-event:: object.__delattr__ obj,name object.__delattr__ + + For certain sensitive attribute deletions, raises an + :ref:`auditing event ` ``object.__delattr__`` with arguments + ``obj`` and ``name``. + .. method:: object.__dir__(self) @@ -1736,7 +1779,7 @@ Super Binding immediately preceding ``B`` and then invokes the descriptor with the call: ``A.__dict__['m'].__get__(obj, obj.__class__)``. -For instance bindings, the precedence of descriptor invocation depends on the +For instance bindings, the precedence of descriptor invocation depends on which descriptor methods are defined. A descriptor can define any combination of :meth:`__get__`, :meth:`__set__` and :meth:`__delete__`. If it does not define :meth:`__get__`, then accessing the attribute will return the descriptor @@ -2130,7 +2173,7 @@ Emulating callable objects .. index:: pair: call; instance Called when the instance is "called" as a function; if this method is defined, - ``x(arg1, arg2, ...)`` is a shorthand for ``x.__call__(arg1, arg2, ...)``. + ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. .. _sequence-types: @@ -2376,10 +2419,11 @@ left undefined. .. note:: - If the right operand's type is a subclass of the left operand's type and that - subclass provides the reflected method for the operation, this method will be - called before the left operand's non-reflected method. This behavior allows - subclasses to override their ancestors' operations. + If the right operand's type is a subclass of the left operand's type and + that subclass provides a different implementation of the reflected method + for the operation, this method will be called before the left operand's + non-reflected method. This behavior allows subclasses to override their + ancestors' operations. .. method:: object.__iadd__(self, other) @@ -2409,6 +2453,13 @@ left undefined. :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact part of the data model. + .. note:: + + Due to a bug in the dispatching mechanism for ``**=``, a class that + defines :meth:`__ipow__` but returns ``NotImplemented`` would fail to + fall back to ``x.__pow__(y)`` and ``y.__rpow__(x)``. This bug is fixed + in Python 3.10. + .. method:: object.__neg__(self) object.__pos__(self) @@ -2601,7 +2652,7 @@ Awaitable Objects ----------------- An :term:`awaitable` object generally implements an :meth:`__await__` method. -:term:`Coroutine` objects returned from :keyword:`async def` functions +:term:`Coroutine objects ` returned from :keyword:`async def` functions are awaitable. .. note:: @@ -2626,7 +2677,7 @@ are awaitable. Coroutine Objects ----------------- -:term:`Coroutine` objects are :term:`awaitable` objects. +:term:`Coroutine objects ` are :term:`awaitable` objects. A coroutine's execution can be controlled by calling :meth:`__await__` and iterating over the result. When the coroutine has finished executing and returns, the iterator raises :exc:`StopIteration`, and the exception's @@ -2771,6 +2822,6 @@ An example of an asynchronous context manager class:: method—that will instead have the opposite effect of explicitly *blocking* such fallback. -.. [#] For operands of the same type, it is assumed that if the non-reflected method - (such as :meth:`__add__`) fails the operation is not supported, which is why the - reflected method is not called. +.. [#] For operands of the same type, it is assumed that if the non-reflected + method -- such as :meth:`__add__` -- fails then the overall operation is not + supported, which is why the reflected method is not called. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 8036a491c29ab1..51fa750d95b1b9 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -13,7 +13,7 @@ This chapter explains the meaning of the elements of expressions in Python. be used to describe syntax, not lexical analysis. When (one alternative of) a syntax rule has the form -.. productionlist:: * +.. productionlist:: python-grammar name: `othername` and no semantics are given, the semantics of this form of ``name`` are the same @@ -54,7 +54,7 @@ Atoms are the most basic elements of expressions. The simplest atoms are identifiers or literals. Forms enclosed in parentheses, brackets or braces are also categorized syntactically as atoms. The syntax for atoms is: -.. productionlist:: +.. productionlist:: python-grammar atom: `identifier` | `literal` | `enclosure` enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` : | `generator_expression` | `yield_atom` @@ -77,6 +77,8 @@ When the name is bound to an object, evaluation of the atom yields that object. When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` exception. +.. _private-name-mangling: + .. index:: pair: name; mangling pair: private; names @@ -103,7 +105,7 @@ Literals Python supports string and bytes literals and various numeric literals: -.. productionlist:: +.. productionlist:: python-grammar literal: `stringliteral` | `bytesliteral` : | `integer` | `floatnumber` | `imagnumber` @@ -134,7 +136,7 @@ Parenthesized forms A parenthesized form is an optional expression list enclosed in parentheses: -.. productionlist:: +.. productionlist:: python-grammar parenth_form: "(" [`starred_expression`] ")" A parenthesized expression list yields whatever that expression list yields: if @@ -162,6 +164,8 @@ ambiguities and allow common typos to pass uncaught. Displays for lists, sets and dictionaries ----------------------------------------- +.. index:: single: comprehensions + For constructing a list, a set or a dictionary Python provides special syntax called "displays", each of them in two flavors: @@ -177,11 +181,11 @@ called "displays", each of them in two flavors: Common syntax elements for comprehensions are: -.. productionlist:: +.. productionlist:: python-grammar comprehension: `assignment_expression` `comp_for` comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` - comp_if: "if" `expression_nocond` [`comp_iter`] + comp_if: "if" `or_test` [`comp_iter`] The comprehension consists of a single expression followed by at least one :keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` clauses. @@ -243,7 +247,7 @@ List displays A list display is a possibly empty series of expressions enclosed in square brackets: -.. productionlist:: +.. productionlist:: python-grammar list_display: "[" [`starred_list` | `comprehension`] "]" A list display yields a new list object, the contents being specified by either @@ -260,6 +264,7 @@ Set displays .. index:: pair: set; display + pair: set; comprehensions object: set single: {} (curly brackets); set expression single: , (comma); expression list @@ -267,7 +272,7 @@ Set displays A set display is denoted by curly braces and distinguishable from dictionary displays by the lack of colons separating keys and values: -.. productionlist:: +.. productionlist:: python-grammar set_display: "{" (`starred_list` | `comprehension`) "}" A set display yields a new mutable set object, the contents being specified by @@ -287,6 +292,7 @@ Dictionary displays .. index:: pair: dictionary; display + pair: dictionary; comprehensions key, datum, key/datum pair object: dictionary single: {} (curly brackets); dictionary expression @@ -296,7 +302,7 @@ Dictionary displays A dictionary display is a possibly empty series of key/datum pairs enclosed in curly braces: -.. productionlist:: +.. productionlist:: python-grammar dict_display: "{" [`key_datum_list` | `dict_comprehension`] "}" key_datum_list: `key_datum` ("," `key_datum`)* [","] key_datum: `expression` ":" `expression` | "**" `or_expr` @@ -355,7 +361,7 @@ Generator expressions A generator expression is a compact generator notation in parentheses: -.. productionlist:: +.. productionlist:: python-grammar generator_expression: "(" `expression` `comp_for` ")" A generator expression yields a new generator object. Its syntax is the same as @@ -409,7 +415,7 @@ Yield expressions pair: yield; expression pair: generator; function -.. productionlist:: +.. productionlist:: python-grammar yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] @@ -472,8 +478,8 @@ allowing any pending :keyword:`finally` clauses to execute. .. index:: single: from; yield from expression -When ``yield from `` is used, it treats the supplied expression as -a subiterator. All values produced by that subiterator are passed directly +When ``yield from `` is used, the supplied expression must be an +iterable. The values produced by iterating that iterable are passed directly to the caller of the current generator's methods. Any values passed in with :meth:`~generator.send` and any exceptions passed in with :meth:`~generator.throw` are passed to the underlying iterator if it has the @@ -746,7 +752,7 @@ Primaries Primaries represent the most tightly bound operations of the language. Their syntax is: -.. productionlist:: +.. productionlist:: python-grammar primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` @@ -761,7 +767,7 @@ Attribute references An attribute reference is a primary followed by a period and a name: -.. productionlist:: +.. productionlist:: python-grammar attributeref: `primary` "." `identifier` .. index:: @@ -796,10 +802,10 @@ Subscriptions object: dictionary pair: sequence; item -A subscription selects an item of a sequence (string, tuple or list) or mapping -(dictionary) object: +Subscription of a sequence (string, tuple or list) or mapping (dictionary) +object usually selects an item from the collection: -.. productionlist:: +.. productionlist:: python-grammar subscription: `primary` "[" `expression_list` "]" The primary must evaluate to an object that supports subscription (lists or @@ -833,6 +839,11 @@ this method will need to explicitly add that support. A string's items are characters. A character is not a separate data type but a string of exactly one character. +Subscription of certain :term:`classes ` or :term:`types ` +creates a :ref:`generic alias `. +In this case, user-defined classes can support subscription by providing a +:meth:`__class_getitem__` classmethod. + .. _slicings: @@ -855,7 +866,7 @@ A slicing selects a range of items in a sequence object (e.g., a string, tuple or list). Slicings may be used as expressions or as targets in assignment or :keyword:`del` statements. The syntax for a slicing: -.. productionlist:: +.. productionlist:: python-grammar slicing: `primary` "[" `slice_list` "]" slice_list: `slice_item` ("," `slice_item`)* [","] slice_item: `expression` | `proper_slice` @@ -905,7 +916,7 @@ Calls A call calls a callable object (e.g., a :term:`function`) with a possibly empty series of :term:`arguments `: -.. productionlist:: +.. productionlist:: python-grammar call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" argument_list: `positional_arguments` ["," `starred_and_keywords`] : ["," `keywords_arguments`] @@ -1088,7 +1099,7 @@ Await expression Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. Can only be used inside a :term:`coroutine function`. -.. productionlist:: +.. productionlist:: python-grammar await_expr: "await" `primary` .. versionadded:: 3.5 @@ -1106,7 +1117,7 @@ The power operator The power operator binds more tightly than unary operators on its left; it binds less tightly than unary operators on its right. The syntax is: -.. productionlist:: +.. productionlist:: python-grammar power: (`await_expr` | `primary`) ["**" `u_expr`] Thus, in an unparenthesized sequence of power and unary operators, the operators @@ -1139,7 +1150,7 @@ Unary arithmetic and bitwise operations All unary arithmetic and bitwise operations have the same priority: -.. productionlist:: +.. productionlist:: python-grammar u_expr: `power` | "-" `u_expr` | "+" `u_expr` | "~" `u_expr` .. index:: @@ -1183,7 +1194,7 @@ that some of these operations also apply to certain non-numeric types. Apart from the power operator, there are only two levels, one for multiplicative operators and one for additive operators: -.. productionlist:: +.. productionlist:: python-grammar m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | : `m_expr` "%" `u_expr` @@ -1279,7 +1290,7 @@ Shifting operations The shifting operations have lower priority than the arithmetic operations: -.. productionlist:: +.. productionlist:: python-grammar shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` These operators accept integers as arguments. They shift the first argument to @@ -1300,7 +1311,7 @@ Binary bitwise operations Each of the three bitwise operations has a different priority level: -.. productionlist:: +.. productionlist:: python-grammar and_expr: `shift_expr` | `and_expr` "&" `shift_expr` xor_expr: `and_expr` | `xor_expr` "^" `and_expr` or_expr: `xor_expr` | `or_expr` "|" `xor_expr` @@ -1349,7 +1360,7 @@ lower than that of any arithmetic, shifting or bitwise operation. Also unlike C, expressions like ``a < b < c`` have the interpretation that is conventional in mathematics: -.. productionlist:: +.. productionlist:: python-grammar comparison: `or_expr` (`comp_operator` `or_expr`)* comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" : | "is" ["not"] | ["not"] "in" @@ -1608,7 +1619,7 @@ Boolean operations pair: Conditional; expression pair: Boolean; operation -.. productionlist:: +.. productionlist:: python-grammar or_test: `and_test` | `or_test` "or" `and_test` and_test: `not_test` | `and_test` "and" `not_test` not_test: `comparison` | "not" `not_test` @@ -1647,12 +1658,29 @@ returns a boolean value regardless of the type of its argument Assignment expressions ====================== -.. productionlist:: +.. productionlist:: python-grammar assignment_expression: [`identifier` ":="] `expression` -.. TODO: BPO-39868 +An assignment expression (sometimes also called a "named expression" or +"walrus") assigns an :token:`expression` to an :token:`identifier`, while also +returning the value of the :token:`expression`. + +One common use case is when handling matched regular expressions: + +.. code-block:: python + + if matching := pattern.search(data): + do_something(matching) + +Or, when processing a file stream in chunks: + +.. code-block:: python + + while chunk := file.read(9000): + process(chunk) -See :pep:`572` for more details about assignment expressions. +.. versionadded:: 3.8 + See :pep:`572` for more details about assignment expressions. .. _if_expr: @@ -1666,10 +1694,9 @@ Conditional expressions single: if; conditional expression single: else; conditional expression -.. productionlist:: +.. productionlist:: python-grammar conditional_expression: `or_test` ["if" `or_test` "else" `expression`] expression: `conditional_expression` | `lambda_expr` - expression_nocond: `or_test` | `lambda_expr_nocond` Conditional expressions (sometimes called a "ternary operator") have the lowest priority of all Python operations. @@ -1693,9 +1720,8 @@ Lambdas pair: anonymous; function single: : (colon); lambda expression -.. productionlist:: +.. productionlist:: python-grammar lambda_expr: "lambda" [`parameter_list`] ":" `expression` - lambda_expr_nocond: "lambda" [`parameter_list`] ":" `expression_nocond` Lambda expressions (sometimes called lambda forms) are used to create anonymous functions. The expression ``lambda parameters: expression`` yields a function @@ -1720,7 +1746,7 @@ Expression lists pair: expression; list single: , (comma); expression list -.. productionlist:: +.. productionlist:: python-grammar expression_list: `expression` ("," `expression`)* [","] starred_list: `starred_item` ("," `starred_item`)* [","] starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] @@ -1783,8 +1809,8 @@ Operator precedence .. index:: pair: operator; precedence -The following table summarizes the operator precedence in Python, from lowest -precedence (least binding) to highest precedence (most binding). Operators in +The following table summarizes the operator precedence in Python, from highest +precedence (most binding) to lowest precedence (least binding). Operators in the same box have the same precedence. Unless the syntax is explicitly given, operators are binary. Operators in the same box group left to right (except for exponentiation, which groups from right to left). @@ -1797,50 +1823,50 @@ precedence and have a left-to-right chaining feature as described in the +-----------------------------------------------+-------------------------------------+ | Operator | Description | +===============================================+=====================================+ -| ``:=`` | Assignment expression | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`lambda` | Lambda expression | +| ``(expressions...)``, | Binding or parenthesized | +| | expression, | +| ``[expressions...]``, | list display, | +| ``{key: value...}``, | dictionary display, | +| ``{expressions...}`` | set display | +-----------------------------------------------+-------------------------------------+ -| :keyword:`if ` -- :keyword:`!else` | Conditional expression | +| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | +| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +-----------------------------------------------+-------------------------------------+ -| :keyword:`or` | Boolean OR | +| :keyword:`await` ``x`` | Await expression | +-----------------------------------------------+-------------------------------------+ -| :keyword:`and` | Boolean AND | +| ``**`` | Exponentiation [#]_ | +-----------------------------------------------+-------------------------------------+ -| :keyword:`not` ``x`` | Boolean NOT | +| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +-----------------------------------------------+-------------------------------------+ -| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | -| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | -| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | +| | multiplication, division, floor | +| | division, remainder [#]_ | +-----------------------------------------------+-------------------------------------+ -| ``|`` | Bitwise OR | +| ``+``, ``-`` | Addition and subtraction | +-----------------------------------------------+-------------------------------------+ -| ``^`` | Bitwise XOR | +| ``<<``, ``>>`` | Shifts | +-----------------------------------------------+-------------------------------------+ | ``&`` | Bitwise AND | +-----------------------------------------------+-------------------------------------+ -| ``<<``, ``>>`` | Shifts | +| ``^`` | Bitwise XOR | +-----------------------------------------------+-------------------------------------+ -| ``+``, ``-`` | Addition and subtraction | +| ``|`` | Bitwise OR | +-----------------------------------------------+-------------------------------------+ -| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication, division, floor | -| | division, remainder [#]_ | +| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | +| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | +| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | +-----------------------------------------------+-------------------------------------+ -| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | +| :keyword:`not` ``x`` | Boolean NOT | +-----------------------------------------------+-------------------------------------+ -| ``**`` | Exponentiation [#]_ | +| :keyword:`and` | Boolean AND | +-----------------------------------------------+-------------------------------------+ -| :keyword:`await` ``x`` | Await expression | +| :keyword:`or` | Boolean OR | +-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | -| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | +| :keyword:`if ` -- :keyword:`!else` | Conditional expression | +-----------------------------------------------+-------------------------------------+ -| ``(expressions...)``, | Binding or parenthesized | -| | expression, | -| ``[expressions...]``, | list display, | -| ``{key: value...}``, | dictionary display, | -| ``{expressions...}`` | set display | +| :keyword:`lambda` | Lambda expression | ++-----------------------------------------------+-------------------------------------+ +| ``:=`` | Assignment expression | +-----------------------------------------------+-------------------------------------+ @@ -1884,8 +1910,8 @@ precedence and have a left-to-right chaining feature as described in the the :keyword:`is` operator, like those involving comparisons between instance methods, or constants. Check their documentation for more info. -.. [#] The ``%`` operator is also used for string formatting; the same - precedence applies. - .. [#] The power operator ``**`` binds less tightly than an arithmetic or bitwise unary operator on its right, that is, ``2**-1`` is ``0.5``. + +.. [#] The ``%`` operator is also used for string formatting; the same + precedence applies. diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst index 83d0f8532c6366..acf83765b5796c 100644 --- a/Doc/reference/grammar.rst +++ b/Doc/reference/grammar.rst @@ -1,7 +1,19 @@ Full Grammar specification ========================== -This is the full Python grammar, as it is read by the parser generator and used -to parse Python source files: +This is the full Python grammar, derived directly from the grammar +used to generate the CPython parser (see :source:`Grammar/python.gram`). +The version here omits details related to code generation and +error recovery. -.. literalinclude:: ../../Grammar/Grammar +The notation is a mixture of `EBNF +`_ +and `PEG `_. +In particular, ``&`` followed by a symbol, token or parenthesized +group indicates a positive lookahead (i.e., is required to match but +not consumed), while ``!`` indicates a negative lookahead (i.e., is +required _not_ to match). We use the ``|`` separator to mean PEG's +"ordered choice" (written as ``/`` in traditional PEG grammars). + +.. literalinclude:: ../../Grammar/python.gram + :language: peg diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst index 4c36e15dc06063..c5952426fdcf8d 100644 --- a/Doc/reference/import.rst +++ b/Doc/reference/import.rst @@ -680,7 +680,7 @@ Here are the exact rules used: Cached bytecode invalidation ---------------------------- -Before Python loads cached bytecode from ``.pyc`` file, it checks whether the +Before Python loads cached bytecode from a ``.pyc`` file, it checks whether the cache is up-to-date with the source ``.py`` file. By default, Python does this by storing the source's last-modified timestamp and size in the cache file when writing it. At runtime, the import system then validates the cache file by @@ -857,9 +857,8 @@ module. ``find_spec()`` returns a fully populated spec for the module. This spec will always have "loader" set (with one exception). To indicate to the import machinery that the spec represents a namespace -:term:`portion`, the path entry finder sets "loader" on the spec to -``None`` and "submodule_search_locations" to a list containing the -portion. +:term:`portion`, the path entry finder sets "submodule_search_locations" to +a list containing the portion. .. versionchanged:: 3.4 :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced @@ -875,18 +874,7 @@ portion. :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the fully qualified name of the module being imported. ``find_loader()`` returns a 2-tuple where the first item is the loader and the second item - is a namespace :term:`portion`. When the first item (i.e. the loader) is - ``None``, this means that while the path entry finder does not have a - loader for the named module, it knows that the path entry contributes to - a namespace portion for the named module. This will almost always be the - case where Python is asked to import a namespace package that has no - physical presence on the file system. When a path entry finder returns - ``None`` for the loader, the second item of the 2-tuple return value must - be a sequence, although it can be empty. - - If ``find_loader()`` returns a non-``None`` loader value, the portion is - ignored and the loader is returned from the path based finder, terminating - the search through the path entries. + is a namespace :term:`portion`. For backwards compatibility with other implementations of the import protocol, many path entry finders also support the same, diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst index bb7e3906dba607..72e874ee98e466 100644 --- a/Doc/reference/introduction.rst +++ b/Doc/reference/introduction.rst @@ -93,7 +93,7 @@ Notation The descriptions of lexical analysis and syntax use a modified BNF grammar notation. This uses the following style of definition: -.. productionlist:: * +.. productionlist:: notation name: `lc_letter` (`lc_letter` | "_")* lc_letter: "a"..."z" diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 3f420817eefea2..77e0578f5d89b6 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -296,7 +296,7 @@ Unicode Character Database as included in the :mod:`unicodedata` module. Identifiers are unlimited in length. Case is significant. -.. productionlist:: +.. productionlist:: python-grammar identifier: `xid_start` `xid_continue`* id_start: id_continue: @@ -325,7 +325,7 @@ of identifiers is based on NFKC. A non-normative HTML file listing all valid identifier characters for Unicode 4.1 can be found at -https://www.dcl.hpi.uni-potsdam.de/home/loewis/table-3131.html. +https://www.unicode.org/Public/13.0.0/ucd/DerivedCoreProperties.txt .. _keywords: @@ -412,7 +412,7 @@ String and Bytes literals String literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar stringliteral: [`stringprefix`](`shortstring` | `longstring`) stringprefix: "r" | "u" | "R" | "U" | "f" | "F" : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" @@ -424,7 +424,7 @@ String literals are described by the following lexical definitions: longstringchar: stringescapeseq: "\" -.. productionlist:: +.. productionlist:: python-grammar bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' @@ -637,9 +637,11 @@ and formatted string literals may be concatenated with plain string literals. single: string; formatted literal single: string; interpolated literal single: f-string + single: fstring single: {} (curly brackets); in formatted string literal single: ! (exclamation); in formatted string literal single: : (colon); in formatted string literal + single: = (equals); for help in debugging using string literals .. _f-strings: Formatted string literals @@ -657,9 +659,9 @@ Escape sequences are decoded like in ordinary string literals (except when a literal is also marked as a raw string). After decoding, the grammar for the contents of the string is: -.. productionlist:: +.. productionlist:: python-grammar f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["!" `conversion`] [":" `format_spec`] "}" + replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" f_expression: (`conditional_expression` | "*" `or_expr`) : ("," `conditional_expression` | "," "*" `or_expr`)* [","] : | `yield_expression` @@ -671,10 +673,11 @@ The parts of the string outside curly braces are treated literally, except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced with the corresponding single curly brace. A single opening curly bracket ``'{'`` marks a replacement field, which starts with a -Python expression. After the expression, there may be a conversion field, -introduced by an exclamation point ``'!'``. A format specifier may also -be appended, introduced by a colon ``':'``. A replacement field ends -with a closing curly bracket ``'}'``. +Python expression. To display both the expression text and its value after +evaluation, (useful in debugging), an equal sign ``'='`` may be added after the +expression. A conversion field, introduced by an exclamation point ``'!'`` may +follow. A format specifier may also be appended, introduced by a colon ``':'``. +A replacement field ends with a closing curly bracket ``'}'``. Expressions in formatted string literals are treated like regular Python expressions surrounded by parentheses, with a few exceptions. @@ -690,6 +693,17 @@ left to right. containing an :keyword:`async for` clause were illegal in the expressions in formatted string literals due to a problem with the implementation. +When the equal sign ``'='`` is provided, the output will have the expression +text, the ``'='`` and the evaluated value. Spaces after the opening brace +``'{'``, within the expression and after the ``'='`` are all retained in the +output. By default, the ``'='`` causes the :func:`repr` of the expression to be +provided, unless there is a format specified. When a format is specified it +defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is +declared. + +.. versionadded:: 3.8 + The equal sign ``'='``. + If a conversion is specified, the result of evaluating the expression is converted before formatting. Conversion ``'!s'`` calls :func:`str` on the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. @@ -704,7 +718,7 @@ Top-level format specifiers may include nested replacement fields. These nested fields may include their own conversion fields and :ref:`format specifiers `, but may not include more deeply-nested replacement fields. The :ref:`format specifier mini-language ` is the same as that used by -the string .format() method. +the :meth:`str.format` method. Formatted string literals may be concatenated, but replacement fields cannot be split across literals. @@ -724,9 +738,22 @@ Some examples of formatted string literals:: >>> today = datetime(year=2017, month=1, day=27) >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' + >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging + 'today=January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier '0x400' + >>> foo = "bar" + >>> f"{ foo = }" # preserves whitespace + " foo = 'bar'" + >>> line = "The mill's closed" + >>> f"{line = }" + 'line = "The mill\'s closed"' + >>> f"{line = :20}" + "line = The mill's closed " + >>> f"{line = !r:20}" + 'line = "The mill\'s closed" ' + A consequence of sharing the same syntax as regular string literals is that characters in the replacement fields must not conflict with the @@ -793,7 +820,7 @@ Integer literals Integer literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ @@ -837,7 +864,7 @@ Floating point literals Floating point literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar floatnumber: `pointfloat` | `exponentfloat` pointfloat: [`digitpart`] `fraction` | `digitpart` "." exponentfloat: (`digitpart` | `pointfloat`) `exponent` @@ -867,7 +894,7 @@ Imaginary literals Imaginary literals are described by the following lexical definitions: -.. productionlist:: +.. productionlist:: python-grammar imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") An imaginary literal yields a complex number with a real part of 0.0. Complex diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index a8ec0fbe8b732c..fcca12dcf5c4ba 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -11,7 +11,7 @@ A simple statement is comprised within a single logical line. Several simple statements may occur on a single line separated by semicolons. The syntax for simple statements is: -.. productionlist:: +.. productionlist:: python-grammar simple_stmt: `expression_stmt` : | `assert_stmt` : | `assignment_stmt` @@ -46,7 +46,7 @@ result; in Python, procedures return the value ``None``). Other uses of expression statements are allowed and occasionally useful. The syntax for an expression statement is: -.. productionlist:: +.. productionlist:: python-grammar expression_stmt: `starred_expression` An expression statement evaluates the expression list (which may be a single @@ -82,7 +82,7 @@ Assignment statements Assignment statements are used to (re)bind names to values and to modify attributes or items of mutable objects: -.. productionlist:: +.. productionlist:: python-grammar assignment_stmt: (`target_list` "=")+ (`starred_expression` | `yield_expression`) target_list: `target` ("," `target`)* [","] target: `identifier` @@ -280,7 +280,7 @@ Augmented assignment statements Augmented assignment is the combination, in a single statement, of a binary operation and an assignment statement: -.. productionlist:: +.. productionlist:: python-grammar augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) augtarget: `identifier` | `attributeref` | `subscription` | `slicing` augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" @@ -328,7 +328,7 @@ Annotated assignment statements :term:`Annotation ` assignment is the combination, in a single statement, of a variable or attribute annotation and an optional assignment statement: -.. productionlist:: +.. productionlist:: python-grammar annotated_assignment_stmt: `augtarget` ":" `expression` : ["=" (`starred_expression` | `yield_expression`)] @@ -385,7 +385,7 @@ The :keyword:`!assert` statement Assert statements are a convenient way to insert debugging assertions into a program: -.. productionlist:: +.. productionlist:: python-grammar assert_stmt: "assert" `expression` ["," `expression`] The simple form, ``assert expression``, is equivalent to :: @@ -425,7 +425,7 @@ The :keyword:`!pass` statement pair: null; operation pair: null; operation -.. productionlist:: +.. productionlist:: python-grammar pass_stmt: "pass" :keyword:`pass` is a null operation --- when it is executed, nothing happens. @@ -447,7 +447,7 @@ The :keyword:`!del` statement pair: deletion; target triple: deletion; target; list -.. productionlist:: +.. productionlist:: python-grammar del_stmt: "del" `target_list` Deletion is recursively defined very similar to the way assignment is defined. @@ -486,7 +486,7 @@ The :keyword:`!return` statement pair: function; definition pair: class; definition -.. productionlist:: +.. productionlist:: python-grammar return_stmt: "return" [`expression_list`] :keyword:`return` may only occur syntactically nested in a function definition, @@ -525,7 +525,7 @@ The :keyword:`!yield` statement single: function; generator exception: StopIteration -.. productionlist:: +.. productionlist:: python-grammar yield_stmt: `yield_expression` A :keyword:`yield` statement is semantically equivalent to a :ref:`yield @@ -560,7 +560,7 @@ The :keyword:`!raise` statement pair: raising; exception single: __traceback__ (exception attribute) -.. productionlist:: +.. productionlist:: python-grammar raise_stmt: "raise" [`expression` ["from" `expression`]] If no expressions are present, :keyword:`raise` re-raises the last exception @@ -591,10 +591,13 @@ instance, with its traceback set to its argument), like so:: __context__ (exception attribute) The ``from`` clause is used for exception chaining: if given, the second -*expression* must be another exception class or instance, which will then be -attached to the raised exception as the :attr:`__cause__` attribute (which is -writable). If the raised exception is not handled, both exceptions will be -printed:: +*expression* must be another exception class or instance. If the second +expression is an exception instance, it will be attached to the raised +exception as the :attr:`__cause__` attribute (which is writable). If the +expression is an exception class, the class will be instantiated and the +resulting exception instance will be attached to the raised exception as the +:attr:`__cause__` attribute. If the raised exception is not handled, both +exceptions will be printed:: >>> try: ... print(1 / 0) @@ -663,7 +666,7 @@ The :keyword:`!break` statement statement: while pair: loop; statement -.. productionlist:: +.. productionlist:: python-grammar break_stmt: "break" :keyword:`break` may only occur syntactically nested in a :keyword:`for` or @@ -698,7 +701,7 @@ The :keyword:`!continue` statement pair: loop; statement keyword: finally -.. productionlist:: +.. productionlist:: python-grammar continue_stmt: "continue" :keyword:`continue` may only occur syntactically nested in a :keyword:`for` or @@ -725,13 +728,13 @@ The :keyword:`!import` statement exception: ImportError single: , (comma); import statement -.. productionlist:: +.. productionlist:: python-grammar import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* : | "from" `relative_module` "import" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] : ("," `identifier` ["as" `identifier`])* [","] ")" - : | "from" `module` "import" "*" + : | "from" `relative_module` "import" "*" module: (`identifier` ".")* `identifier` relative_module: "."* `module` | "."+ @@ -859,7 +862,7 @@ that introduce incompatible changes to the language. It allows use of the new features on a per-module basis before the release in which the feature becomes standard. -.. productionlist:: * +.. productionlist:: python-grammar future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] : ("," `feature` ["as" `identifier`])* : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] @@ -874,8 +877,8 @@ can appear before a future statement are: * blank lines, and * other future statements. -The only feature in Python 3.7 that requires using the future statement is -``annotations``. +The only feature that requires using the future statement is +``annotations`` (see :pep:`563`). All historical features enabled by the future statement are still recognized by Python 3. The list includes ``absolute_import``, ``division``, @@ -937,7 +940,7 @@ The :keyword:`!global` statement triple: global; name; binding single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar global_stmt: "global" `identifier` ("," `identifier`)* The :keyword:`global` statement is a declaration which holds for the entire @@ -982,7 +985,7 @@ The :keyword:`!nonlocal` statement .. index:: statement: nonlocal single: , (comma); identifier list -.. productionlist:: +.. productionlist:: python-grammar nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* .. XXX add when implemented diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst index d5ffb37b2e58cd..319c9de484241e 100644 --- a/Doc/reference/toplevel_components.rst +++ b/Doc/reference/toplevel_components.rst @@ -66,7 +66,7 @@ File input All input read from non-interactive files has the same form: -.. productionlist:: +.. productionlist:: python-grammar file_input: (NEWLINE | `statement`)* This syntax is used in the following situations: @@ -85,7 +85,7 @@ Interactive input Input in interactive mode is parsed using the following grammar: -.. productionlist:: +.. productionlist:: python-grammar interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE Note that a (top-level) compound statement must be followed by a blank line in @@ -103,5 +103,5 @@ Expression input :func:`eval` is used for expression input. It ignores leading whitespace. The string argument to :func:`eval` must have the following form: -.. productionlist:: +.. productionlist:: python-grammar eval_input: `expression_list` NEWLINE* diff --git a/Doc/requirements.txt b/Doc/requirements.txt index 198446b350ff2d..47b78eeac817e0 100644 --- a/Doc/requirements.txt +++ b/Doc/requirements.txt @@ -1,5 +1,12 @@ -# Requirements for docs build on netlify -# Pin sphinx to version specified in .travis.yml -sphinx==2.2.0 +# Requirements to build the Python documentation + +# Sphinx version is pinned so that new versions that introduce new warnings +# won't suddenly cause build failures. Updating the version is fine as long +# as no warnings are raised by doing so. +sphinx==2.4.4 + blurb + +# The theme used by the documentation is stored separately, so we need +# to install that as well. python-docs-theme diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/asdl_highlight.py index 7d2ef011c1b766..b1989e53957072 100644 --- a/Doc/tools/extensions/asdl_highlight.py +++ b/Doc/tools/extensions/asdl_highlight.py @@ -1,6 +1,9 @@ import os import sys -sys.path.append(os.path.abspath("../Parser/")) +from pathlib import Path + +CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(CPYTHON_ROOT / "Parser")) from pygments.lexer import RegexLexer, bygroups, include, words from pygments.token import (Comment, Generic, Keyword, Name, Operator, diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py index fa8244a8fd318e..76c9d920cbe31f 100644 --- a/Doc/tools/extensions/c_annotations.py +++ b/Doc/tools/extensions/c_annotations.py @@ -79,9 +79,9 @@ def add_annotations(self, app, doctree): classes=['stableabi'])) if par['objtype'] != 'function': continue - if not par[0].has_key('names') or not par[0]['names']: + if not par[0].has_key('ids') or not par[0]['ids']: continue - name = par[0]['names'][0] + name = par[0]['ids'][0] if name.startswith("c."): name = name[2:] entry = self.get(name) diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py new file mode 100644 index 00000000000000..8bc24670fbe0ab --- /dev/null +++ b/Doc/tools/extensions/peg_highlight.py @@ -0,0 +1,75 @@ +from pygments.lexer import RegexLexer, bygroups, include +from pygments.token import Comment, Generic, Keyword, Name, Operator, Punctuation, Text + +from sphinx.highlighting import lexers + + +class PEGLexer(RegexLexer): + """Pygments Lexer for PEG grammar (.gram) files + + This lexer strips the following elements from the grammar: + + - Meta-tags + - Variable assignments + - Actions + - Lookaheads + - Rule types + - Rule options + - Rules named `invalid_*` or `incorrect_*` + """ + + name = "PEG" + aliases = ["peg"] + filenames = ["*.gram"] + _name = r"([^\W\d]\w*)" + _text_ws = r"(\s*)" + + tokens = { + "ws": [(r"\n", Text), (r"\s+", Text), (r"#.*$", Comment.Singleline),], + "lookaheads": [ + (r"(?<=\|\s)(&\w+\s?)", bygroups(None)), + (r"(?<=\|\s)(&'.+'\s?)", bygroups(None)), + (r'(?<=\|\s)(&".+"\s?)', bygroups(None)), + (r"(?<=\|\s)(&\(.+\)\s?)", bygroups(None)), + ], + "metas": [ + (r"(@\w+ '''(.|\n)+?''')", bygroups(None)), + (r"^(@.*)$", bygroups(None)), + ], + "actions": [(r"{(.|\n)+?}", bygroups(None)),], + "strings": [ + (r"'\w+?'", Keyword), + (r'"\w+?"', Keyword), + (r"'\W+?'", Text), + (r'"\W+?"', Text), + ], + "variables": [(_name + _text_ws + "(=)", bygroups(None, None, None),),], + "invalids": [ + (r"^(\s+\|\s+invalid_\w+\s*\n)", bygroups(None)), + (r"^(\s+\|\s+incorrect_\w+\s*\n)", bygroups(None)), + (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),), + ], + "root": [ + include("invalids"), + include("ws"), + include("lookaheads"), + include("metas"), + include("actions"), + include("strings"), + include("variables"), + (r"\b(?!(NULL|EXTRA))([A-Z_]+)\b\s*(?!\()", Text,), + ( + r"^\s*" + _name + r"\s*" + r"(\[.*\])?" + r"\s*" + r"(\(.+\))?" + r"\s*(:)", + bygroups(Name.Function, None, None, Punctuation), + ), + (_name, Name.Function), + (r"[\||\.|\+|\*|\?]", Operator), + (r"{|}|\(|\)|\[|\]", Punctuation), + (r".", Text), + ], + } + + +def setup(app): + lexers["peg"] = PEGLexer() + return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index bc51555fa0512b..74de6ffa75559e 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -31,7 +31,12 @@ from sphinx.util.nodes import split_explicit_title from sphinx.writers.text import TextWriter, TextTranslator from sphinx.writers.latex import LaTeXTranslator -from sphinx.domains.python import PyModulelevel, PyClassmember + +try: + from sphinx.domains.python import PyFunction, PyMethod +except ImportError: + from sphinx.domains.python import PyClassmember as PyMethod + from sphinx.domains.python import PyModulelevel as PyFunction # Support for checking for suspicious markup @@ -39,7 +44,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/master/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.9/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -186,6 +191,7 @@ def run(self): info['source'].append((env.docname, target)) pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) + pnode.line = self.lineno if self.content: self.state.nested_parse(self.content, self.content_offset, pnode) else: @@ -238,17 +244,18 @@ def needs_arglist(self): return False -class PyDecoratorFunction(PyDecoratorMixin, PyModulelevel): +class PyDecoratorFunction(PyDecoratorMixin, PyFunction): def run(self): # a decorator function is a function after all self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyDecoratorMethod(PyDecoratorMixin, PyClassmember): +# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible +class PyDecoratorMethod(PyDecoratorMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) class PyCoroutineMixin(object): @@ -265,31 +272,31 @@ def handle_signature(self, sig, signode): return ret -class PyCoroutineFunction(PyCoroutineMixin, PyModulelevel): +class PyCoroutineFunction(PyCoroutineMixin, PyFunction): def run(self): self.name = 'py:function' - return PyModulelevel.run(self) + return PyFunction.run(self) -class PyCoroutineMethod(PyCoroutineMixin, PyClassmember): +class PyCoroutineMethod(PyCoroutineMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAwaitableFunction(PyAwaitableMixin, PyClassmember): +class PyAwaitableFunction(PyAwaitableMixin, PyFunction): def run(self): self.name = 'py:function' - return PyClassmember.run(self) + return PyFunction.run(self) -class PyAwaitableMethod(PyAwaitableMixin, PyClassmember): +class PyAwaitableMethod(PyAwaitableMixin, PyMethod): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) -class PyAbstractMethod(PyClassmember): +class PyAbstractMethod(PyMethod): def handle_signature(self, sig, signode): ret = super(PyAbstractMethod, self).handle_signature(sig, signode) @@ -299,7 +306,7 @@ def handle_signature(self, sig, signode): def run(self): self.name = 'py:method' - return PyClassmember.run(self) + return PyMethod.run(self) # Support for documenting version of removal in deprecations @@ -311,7 +318,8 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _deprecated_label = 'Deprecated since version {deprecated}, will be removed in version {removed}' + _removed_label = 'Deprecated since version {deprecated}, removed in version {removed}' def run(self): node = addnodes.versionmodified() @@ -319,7 +327,15 @@ def run(self): node['type'] = 'deprecated-removed' version = (self.arguments[0], self.arguments[1]) node['version'] = version - label = translators['sphinx'].gettext(self._label) + env = self.state.document.settings.env + current_version = tuple(int(e) for e in env.config.version.split('.')) + removed_version = tuple(int(e) for e in self.arguments[1].split('.')) + if current_version < removed_version: + label = self._deprecated_label + else: + label = self._removed_label + + label = translators['sphinx'].gettext(label) text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index e1ef91a8dfc686..1a1c7d0fa57e23 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,11 +10,11 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.9': 'dev (3.9)', + '3.10': 'dev (3.10)', + '3.9': 'pre (3.9)', '3.8': '3.8', '3.7': '3.7', '3.6': '3.6', - '3.5': '3.5', '2.7': '2.7', }; diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv index 7be8d0abd69a57..ad5f55e8038520 100644 --- a/Doc/tools/susp-ignored.csv +++ b/Doc/tools/susp-ignored.csv @@ -5,14 +5,13 @@ c-api/sequence,,:i2,o[i1:i2] c-api/tuple,,:high,p[low:high] c-api/unicode,,:end,str[start:end] c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,267,`,This is the description of the ``foobar`` package. +distutils/examples,,`,This is the description of the ``foobar`` package. distutils/setupscript,,::, extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {" extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {" faq/programming,,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" -faq/programming,,::,for x in sequence[::-1]: faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," faq/windows,,:d48eceb,"Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32" @@ -24,6 +23,9 @@ howto/curses,,:blue,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. howto/curses,,:magenta,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" howto/curses,,:cyan,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" howto/curses,,:white,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" +howto/descriptor,,:root,"INFO:root" +howto/descriptor,,:Updating,"root:Updating" +howto/descriptor,,:Accessing,"root:Accessing" howto/instrumentation,,::,python$target:::function-entry howto/instrumentation,,:function,python$target:::function-entry howto/instrumentation,,::,python$target:::function-return @@ -147,8 +149,10 @@ library/ipaddress,,:db8,IPv6Address('2001:db8::') library/ipaddress,,::,IPv6Address('2001:db8::') library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,IPv6Address('2001:db8::1000') -library/ipaddress,,::,IPv6Address('2001:db8::1000') +library/ipaddress,,:db8,'2001:db8::1000' +library/ipaddress,,::,'2001:db8::1000' +library/ipaddress,,:db8,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" +library/ipaddress,,::,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" library/ipaddress,,::,IPv6Address('ff02::5678%1') library/ipaddress,,::,fe80::1234 library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" @@ -167,6 +171,7 @@ library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: library/ipaddress,,::,2001:db00::0/ffff:ff00:: library/itertools,,:step,elements from seq[start:stop:step] library/itertools,,:stop,elements from seq[start:stop:step] +library/itertools,,::,kernel = tuple(kernel)[::-1] library/logging.handlers,,:port,host:port library/mmap,,:i2,obj[i1:i2] library/multiprocessing,,`,# Add more tasks using `put()` @@ -206,9 +211,8 @@ library/smtplib,,:port,method must support that as well as a regular host:port library/socket,,::,'5aef:2b::8' library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:age,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" +library/sqlite3,,:year,"cur.execute(""select * from lang where first_appeared=:year"", {""year"": 1972})" library/sqlite3,,:memory, -library/sqlite3,,:who,"cur.execute(""select * from people where name_last=:who and age=:age"", {""who"": who, ""age"": age})" library/sqlite3,,:path,"db = sqlite3.connect('file:path/to/database?mode=ro', uri=True)" library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index fca44e9163cac7..21af621efce95b 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -4,7 +4,7 @@

{{ _('This Page') }}

  • {% trans %}Report a Bug{% endtrans %}
  • - {{ _('Show Source') }}
    • diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 8d94137b01b519..68ae3ad148ec27 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -5,3 +5,4 @@ {% trans %}CPython implementation detail:{% endtrans %} {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} +{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 4730a5fe5db7bc..1c1cb5484a4f67 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,11 +2,11 @@

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    {% trans %}Docs by version{% endtrans %}

    diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst index 06bdd0d93515ed..0d780e3ba89643 100644 --- a/Doc/tutorial/classes.rst +++ b/Doc/tutorial/classes.rst @@ -114,8 +114,8 @@ accessible. "Directly accessible" here means that an unqualified reference to a name attempts to find the name in the namespace. Although scopes are determined statically, they are used dynamically. At any -time during execution, there are at least three nested scopes whose namespaces -are directly accessible: +time during execution, there are 3 or 4 nested scopes whose namespaces are +directly accessible: * the innermost scope, which is searched first, contains the local names * the scopes of any enclosing functions, which are searched starting with the @@ -849,7 +849,7 @@ defines :meth:`__next__`, then :meth:`__iter__` can just return ``self``:: Generators ========== -:term:`Generator`\s are a simple and powerful tool for creating iterators. They +:term:`Generators ` are a simple and powerful tool for creating iterators. They are written like regular functions but use the :keyword:`yield` statement whenever they want to return data. Each time :func:`next` is called on it, the generator resumes where it left off (it remembers all the data values and which diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index f05f5edd5ccc40..129ab21cc48ce4 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -104,14 +104,14 @@ The given end point is never part of the generated sequence; ``range(10)`` gener is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: - range(5, 10) - 5, 6, 7, 8, 9 + >>> list(range(5, 10)) + [5, 6, 7, 8, 9] - range(0, 10, 3) - 0, 3, 6, 9 + >>> list(range(0, 10, 3)) + [0, 3, 6, 9] - range(-10, -100, -30) - -10, -40, -70 + >>> list(range(-10, -100, -30)) + [-10, -40, -70] To iterate over the indices of a sequence, you can combine :func:`range` and :func:`len` as follows:: @@ -131,7 +131,7 @@ function, see :ref:`tut-loopidioms`. A strange thing happens if you just print a range:: - >>> print(range(10)) + >>> range(10) range(0, 10) In many ways the object returned by :func:`range` behaves as if it is a list, @@ -149,13 +149,7 @@ that takes an iterable is :func:`sum`:: 6 Later we will see more functions that return iterables and take iterables as -arguments. Lastly, maybe you are curious about how to get a list from a range. -Here is the solution:: - - >>> list(range(4)) - [0, 1, 2, 3] - -In chapter :ref:`tut-structures`, we will discuss in more detail about +arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about :func:`list`. .. _tut-break: @@ -207,15 +201,16 @@ iteration of the loop:: ... if num % 2 == 0: ... print("Found an even number", num) ... continue - ... print("Found a number", num) + ... print("Found an odd number", num) + ... Found an even number 2 - Found a number 3 + Found an odd number 3 Found an even number 4 - Found a number 5 + Found an odd number 5 Found an even number 6 - Found a number 7 + Found an odd number 7 Found an even number 8 - Found a number 9 + Found an odd number 9 .. _tut-pass: @@ -294,14 +289,14 @@ referenced. The actual parameters (arguments) to a function call are introduced in the local symbol table of the called function when it is called; thus, arguments are passed using *call by value* (where the *value* is always an object *reference*, -not the value of the object). [#]_ When a function calls another function, a new +not the value of the object). [#]_ When a function calls another function, +or calls itself recursively, a new local symbol table is created for that call. -A function definition introduces the function name in the current symbol table. -The value of the function name has a type that is recognized by the interpreter -as a user-defined function. This value can be assigned to another name which -can then also be used as a function. This serves as a general renaming -mechanism:: +A function definition associates the function name with the function object in +the current symbol table. The interpreter recognizes the object pointed to by +that name as a user-defined function. Other names can also point to that same +function object and can also be used to access the function:: >>> fib @@ -659,7 +654,7 @@ Finally, consider this function definition which has a potential collision betwe return 'name' in kwds There is no possible call that will make it return ``True`` as the keyword ``'name'`` -will always to bind to the first parameter. For example:: +will always bind to the first parameter. For example:: >>> foo(1, **{'name': 2}) Traceback (most recent call last): @@ -866,7 +861,7 @@ function. Parameter annotations are defined by a colon after the parameter name by an expression evaluating to the value of the annotation. Return annotations are defined by a literal ``->``, followed by an expression, between the parameter list and the colon denoting the end of the :keyword:`def` statement. The -following example has a positional argument, a keyword argument, and the return +following example has a required argument, an optional argument, and the return value annotated:: >>> def f(ham: str, eggs: str = 'eggs') -> str: diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index ff4c797f66cd63..e42b380db3d23c 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -78,7 +78,7 @@ objects: Return the number of times *x* appears in the list. -.. method:: list.sort(key=None, reverse=False) +.. method:: list.sort(*, key=None, reverse=False) :noindex: Sort the items of the list in place (the arguments can be used for sort @@ -661,9 +661,8 @@ operators, not just comparisons. The comparison operators ``in`` and ``not in`` check whether a value occurs (does not occur) in a sequence. The operators ``is`` and ``is not`` compare -whether two objects are really the same object; this only matters for mutable -objects like lists. All comparison operators have the same priority, which is -lower than that of all numerical operators. +whether two objects are really the same object. All comparison operators have +the same priority, which is lower than that of all numerical operators. Comparisons can be chained. For example, ``a < b == c`` tests whether ``a`` is less than ``b`` and moreover ``b`` equals ``c``. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 0ce96466e8c286..516c925b72f5e7 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -272,16 +272,16 @@ re-raise the exception:: Exception Chaining ================== -The :keyword:`raise` statement allows an optional :keyword:`from` which enables -chaining exceptions by setting the ``__cause__`` attribute of the raised -exception. For example:: +The :keyword:`raise` statement allows an optional :keyword:`from` which enables +chaining exceptions. For example:: - raise RuntimeError from OSError + # exc must be exception instance or None. + raise RuntimeError from exc This can be useful when you are transforming exceptions. For example:: >>> def func(): - ... raise IOError + ... raise IOError ... >>> try: ... func() @@ -297,22 +297,23 @@ This can be useful when you are transforming exceptions. For example:: Traceback (most recent call last): File "", line 4, in - RuntimeError + RuntimeError: Failed to open database -The expression following the :keyword:`from` must be either an exception or -``None``. Exception chaining happens automatically when an exception is raised -inside an exception handler or :keyword:`finally` section. Exception chaining -can be disabled by using ``from None`` idiom: +Exception chaining happens automatically when an exception is raised inside an +:keyword:`except` or :keyword:`finally` section. Exception chaining can be +disabled by using ``from None`` idiom: >>> try: ... open('database.sqlite') - ... except IOError: + ... except OSError: ... raise RuntimeError from None ... Traceback (most recent call last): File "", line 4, in RuntimeError +For more information about chaining mechanics, see :ref:`bltin-exceptions`. + .. _tut-userexceptions: @@ -404,6 +405,10 @@ points discuss more complex cases when an exception occurs: or :keyword:`!else` clause. Again, the exception is re-raised after the :keyword:`!finally` clause has been executed. +* If the :keyword:`!finally` clause executes a :keyword:`break`, + :keyword:`continue` or :keyword:`return` statement, exceptions are not + re-raised. + * If the :keyword:`!try` statement reaches a :keyword:`break`, :keyword:`continue` or :keyword:`return` statement, the :keyword:`!finally` clause will execute just prior to the diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index a404f4be19f1b9..7f83c4d4612eb3 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -172,7 +172,7 @@ Positional and keyword arguments can be arbitrarily combined:: If you have a really long format string that you don't want to split up, it would be nice if you could reference the variables to be formatted by name instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys :: +square brackets ``'[]'`` to access the keys. :: >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' @@ -257,10 +257,10 @@ left with zeros. It understands about plus and minus signs:: Old string formatting --------------------- -The ``%`` operator can also be used for string formatting. It interprets the -left argument much like a :c:func:`sprintf`\ -style format string to be applied -to the right argument, and returns the string resulting from this formatting -operation. For example:: +The % operator (modulo) can also be used for string formatting. Given ``'string' +% values``, instances of ``%`` in ``string`` are replaced with zero or more +elements of ``values``. This operation is commonly known as string +interpolation. For example:: >>> import math >>> print('The value of pi is approximately %5.3f.' % math.pi) @@ -329,11 +329,16 @@ equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: If you're not using the :keyword:`with` keyword, then you should call ``f.close()`` to close the file and immediately free up any system -resources used by it. If you don't explicitly close a file, Python's -garbage collector will eventually destroy the object and close the -open file for you, but the file may stay open for a while. Another -risk is that different Python implementations will do this clean-up at -different times. +resources used by it. + +.. warning:: + Calling ``f.write()`` without using the :keyword:`!with` keyword or calling + ``f.close()`` **might** result in the arguments + of ``f.write()`` not being completely written to the disk, even if the + program exits successfully. + +.. + See also https://bugs.python.org/issue17852 After a file object is closed, either by a :keyword:`with` statement or by calling ``f.close()``, attempts to use the file object will @@ -475,7 +480,8 @@ If you have an object ``x``, you can view its JSON string representation with a simple line of code:: >>> import json - >>> json.dumps([1, 'simple', 'list']) + >>> x = [1, 'simple', 'list'] + >>> json.dumps(x) '[1, "simple", "list"]' Another variant of the :func:`~json.dumps` function, called :func:`~json.dump`, diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst index 2a1666128a2015..4613cf76c53099 100644 --- a/Doc/tutorial/introduction.rst +++ b/Doc/tutorial/introduction.rst @@ -73,7 +73,7 @@ operator; to calculate the remainder you can use ``%``:: 5 >>> 17 % 3 # the % operator returns the remainder of the division 2 - >>> 5 * 3 + 2 # result * divisor + remainder + >>> 5 * 3 + 2 # floored quotient * divisor + remainder 17 With Python, it is possible to use the ``**`` operator to calculate powers [#]_:: diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst index f422146aae8a30..2782db9da077db 100644 --- a/Doc/tutorial/venv.rst +++ b/Doc/tutorial/venv.rst @@ -95,20 +95,9 @@ Managing Packages with pip You can install, upgrade, and remove packages using a program called :program:`pip`. By default ``pip`` will install packages from the Python Package Index, . You can browse the Python -Package Index by going to it in your web browser, or you can use ``pip``'s -limited search feature: +Package Index by going to it in your web browser. -.. code-block:: bash - - (tutorial-env) $ pip search astronomy - skyfield - Elegant astronomy for Python - gary - Galactic astronomy and gravitational dynamics. - novas - The United States Naval Observatory NOVAS astronomy library - astroobs - Provides astronomy ephemeris to plan telescope observations - PyAstronomy - A collection of astronomy related tools for Python. - ... - -``pip`` has a number of subcommands: "search", "install", "uninstall", +``pip`` has a number of subcommands: "install", "uninstall", "freeze", etc. (Consult the :ref:`installing-index` guide for complete documentation for ``pip``.) diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index b0911956a9eb86..95342f3a510e19 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -538,6 +538,14 @@ conflict. within a Python program as the variable :data:`sys.path`. +.. envvar:: PYTHONPLATLIBDIR + + If this is set to a non-empty string, it overrides the :data:`sys.platlibdir` + value. + + .. versionadded:: 3.9 + + .. envvar:: PYTHONSTARTUP If this is the name of a readable file, the Python commands in that file are @@ -547,7 +555,7 @@ conflict. the interactive session. You can also change the prompts :data:`sys.ps1` and :data:`sys.ps2` and the hook :data:`sys.__interactivehook__` in this file. - .. audit-event:: cpython.run_startup filename PYTHONSTARTUP + .. audit-event:: cpython.run_startup filename envvar-PYTHONSTARTUP Raises an :ref:`auditing event ` ``cpython.run_startup`` with the filename as the argument when called on startup. diff --git a/Doc/using/mac.rst b/Doc/using/mac.rst index baf737ddaa9171..ead71e1b079b3b 100644 --- a/Doc/using/mac.rst +++ b/Doc/using/mac.rst @@ -27,9 +27,8 @@ What you get after installing is a number of things: * A :file:`Python 3.9` folder in your :file:`Applications` folder. In here you find IDLE, the development environment that is a standard part of official - Python distributions; PythonLauncher, which handles double-clicking Python - scripts from the Finder; and the "Build Applet" tool, which allows you to - package Python scripts as standalone applications on your system. + Python distributions; and PythonLauncher, which handles double-clicking Python + scripts from the Finder. * A framework :file:`/Library/Frameworks/Python.framework`, which includes the Python executable and libraries. The installer adds this location to your shell @@ -159,11 +158,6 @@ https://riverbankcomputing.com/software/pyqt/intro. Distributing Python Applications on the Mac =========================================== -The "Build Applet" tool that is placed in the MacPython 3.6 folder is fine for -packaging small Python scripts on your own machine to run as a standard Mac -application. This tool, however, is not robust enough to distribute Python -applications to other users. - The standard tool for deploying standalone Python applications on the Mac is :program:`py2app`. More information on installing and using py2app can be found at http://undefined.org/python/#py2app. diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index c39048d6188127..ddb36f94667d9f 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -67,7 +67,7 @@ The command, if run with ``-h``, will show the available options:: Once an environment has been created, you may wish to activate it, e.g. by sourcing an activate script in its bin directory. -.. versionchanged:: 3.8 +.. versionchanged:: 3.9 Add ``--upgrade-deps`` option to upgrade pip + setuptools to the latest on PyPI .. versionchanged:: 3.4 @@ -91,7 +91,7 @@ The command, if run with ``-h``, will show the available options:: PS C:\> Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser See `About Execution Policies - `_ + `_ for more information. The created ``pyvenv.cfg`` file also includes the @@ -126,6 +126,10 @@ directory containing the virtual environment): | | PowerShell | PS C:\\> \\Scripts\\Activate.ps1 | +-------------+-----------------+-----------------------------------------+ +When a virtual environment is active, the :envvar:`VIRTUAL_ENV` environment +variable is set to the path of the virtual environment. This can be used to +check if one is running inside a virtual environment. + You don't specifically *need* to activate an environment; activation just prepends the virtual environment's binary directory to your path, so that "python" invokes the virtual environment's Python interpreter and you can run diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 97e9cdfeb09396..c04a34a62f62e2 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -23,8 +23,8 @@ available for application-local distributions. As specified in :pep:`11`, a Python release only supports a Windows platform while Microsoft considers the platform under extended support. This means that -Python |version| supports Windows Vista and newer. If you require Windows XP -support then please install Python 3.4. +Python |version| supports Windows 8.1 and newer. If you require Windows 7 +support, please install Python 3.8. There are a number of different installers available for Windows, each with certain benefits and downsides. @@ -103,14 +103,12 @@ paths longer than this would not resolve and errors would result. In the latest versions of Windows, this limitation can be expanded to approximately 32,000 characters. Your administrator will need to activate the -"Enable Win32 long paths" group policy, or set the registry value -``HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem@LongPathsEnabled`` -to ``1``. +"Enable Win32 long paths" group policy, or set ``LongPathsEnabled`` to ``1`` +in the registry key +``HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem``. This allows the :func:`open` function, the :mod:`os` module and most other -path functionality to accept and return paths longer than 260 characters when -using strings. (Use of bytes as paths is deprecated on Windows, and this feature -is not available when using bytes.) +path functionality to accept and return paths longer than 260 characters. After changing the above option, no further configuration is required. @@ -341,6 +339,11 @@ full write access to shared locations such as ``TEMP`` and the registry. Instead, it will write to a private copy. If your scripts must modify the shared locations, you will need to install the full installer. +For more detail on the technical basis for these limitations, please consult +Microsoft's documentation on packaged full-trust apps, currently available at +`docs.microsoft.com/en-us/windows/msix/desktop/desktop-to-uwp-behind-the-scenes +`_ + .. _windows-nuget: @@ -1109,7 +1112,7 @@ shipped with PyWin32. It is an embeddable IDE with a built-in debugger. cx_Freeze --------- -`cx_Freeze `_ is a :mod:`distutils` +`cx_Freeze `_ is a :mod:`distutils` extension (see :ref:`extending-distutils`) which wraps Python scripts into executable Windows programs (:file:`{*}.exe` files). When you have done this, you can distribute your application without requiring your users to install @@ -1152,8 +1155,6 @@ For extension modules, consult :ref:`building-on-windows`. MinGW gcc under Windows" or "Installing Python extension with distutils and without Microsoft Visual C++" by Sébastien Sauvage, 2003 - `MingW -- Python extensions `_ - Other Platforms =============== diff --git a/Doc/whatsnew/3.2.rst b/Doc/whatsnew/3.2.rst index ca3eda05c515af..06bee9966c0be2 100644 --- a/Doc/whatsnew/3.2.rst +++ b/Doc/whatsnew/3.2.rst @@ -2311,7 +2311,7 @@ Multi-threading =============== * The mechanism for serializing execution of concurrently running Python threads - (generally known as the :term:`GIL` or :term:`Global Interpreter Lock`) has + (generally known as the :term:`GIL` or Global Interpreter Lock) has been rewritten. Among the objectives were more predictable switching intervals and reduced overhead due to lock contention and the number of ensuing system calls. The notion of a "check interval" to allow thread diff --git a/Doc/whatsnew/3.3.rst b/Doc/whatsnew/3.3.rst index f1a033c6dae61f..361e6db07c3cdb 100644 --- a/Doc/whatsnew/3.3.rst +++ b/Doc/whatsnew/3.3.rst @@ -2309,9 +2309,9 @@ Encoders: :c:func:`PyUnicode_AsUTF8String` * :c:func:`PyUnicode_EncodeUTF32` * :c:func:`PyUnicode_EncodeUTF16` -* :c:func:`PyUnicode_EncodeUnicodeEscape:` use +* :c:func:`PyUnicode_EncodeUnicodeEscape` use :c:func:`PyUnicode_AsUnicodeEscapeString` -* :c:func:`PyUnicode_EncodeRawUnicodeEscape:` use +* :c:func:`PyUnicode_EncodeRawUnicodeEscape` use :c:func:`PyUnicode_AsRawUnicodeEscapeString` * :c:func:`PyUnicode_EncodeLatin1`: use :c:func:`PyUnicode_AsLatin1String` * :c:func:`PyUnicode_EncodeASCII`: use :c:func:`PyUnicode_AsASCIIString` diff --git a/Doc/whatsnew/3.5.rst b/Doc/whatsnew/3.5.rst index b4540ac1dd9028..1defee4090f288 100644 --- a/Doc/whatsnew/3.5.rst +++ b/Doc/whatsnew/3.5.rst @@ -412,7 +412,7 @@ uses were to provide type hints to function parameters and return values. It became evident that it would be beneficial for Python users, if the standard library included the base definitions and tools for type annotations. -:pep:`484` introduces a :term:`provisional module ` to +:pep:`484` introduces a :term:`provisional module ` to provide these standard definitions and tools, along with some conventions for situations where annotations are not available. @@ -726,7 +726,7 @@ New Modules typing ------ -The new :mod:`typing` :term:`provisional ` module +The new :mod:`typing` :term:`provisional ` module provides standard definitions and tools for function type annotations. See :ref:`Type Hints ` for more information. @@ -772,7 +772,7 @@ Steven Bethard, paul j3 and Daniel Eriksson in :issue:`14910`.) asyncio ------- -Since the :mod:`asyncio` module is :term:`provisional `, +Since the :mod:`asyncio` module is :term:`provisional `, all changes introduced in Python 3.5 have also been backported to Python 3.4.x. Notable changes in the :mod:`asyncio` module since Python 3.4.0: @@ -1867,7 +1867,7 @@ A new :func:`~sys.set_coroutine_wrapper` function allows setting a global hook that will be called whenever a :term:`coroutine object ` is created by an :keyword:`async def` function. A corresponding :func:`~sys.get_coroutine_wrapper` can be used to obtain a currently set -wrapper. Both functions are :term:`provisional `, +wrapper. Both functions are :term:`provisional `, and are intended for debugging purposes only. (Contributed by Yury Selivanov in :issue:`24017`.) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 04c1f7e71db321..03a877a3d91785 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1597,7 +1597,7 @@ to filter block traces by their address space (domain). typing ------ -Since the :mod:`typing` module is :term:`provisional `, +Since the :mod:`typing` module is :term:`provisional `, all changes introduced in Python 3.6 have also been backported to Python 3.5.x. @@ -2443,3 +2443,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.6.13 +================================ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 59b96621bdd4b5..9204cc7fbf8c47 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -171,7 +171,7 @@ on a per-module basis in Python 3.7 using a :mod:`__future__` import:: from __future__ import annotations -It will become the default in Python 4.0. +It will become the default in Python 3.10. .. seealso:: @@ -636,7 +636,7 @@ The :mod:`asyncio` module has received many new features, usability and :ref:`performance improvements `. Notable changes include: -* The new :term:`provisional ` :func:`asyncio.run` function can +* The new :term:`provisional ` :func:`asyncio.run` function can be used to run a coroutine from synchronous code by automatically creating and destroying the event loop. (Contributed by Yury Selivanov in :issue:`32314`.) @@ -2556,3 +2556,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.7.10 +================================ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.8.rst b/Doc/whatsnew/3.8.rst index fdfc0a8f472cd6..dbc3875aae61b3 100644 --- a/Doc/whatsnew/3.8.rst +++ b/Doc/whatsnew/3.8.rst @@ -646,7 +646,8 @@ loop on every invocation: (Contributed by Yury Selivanov in :issue:`37028`.) The exception :class:`asyncio.CancelledError` now inherits from -:class:`BaseException` rather than :class:`Exception`. +:class:`BaseException` rather than :class:`Exception` and no longer inherits +from :class:`concurrent.futures.CancelledError`. (Contributed by Yury Selivanov in :issue:`32528`.) On Windows, the default event loop is now :class:`~asyncio.ProactorEventLoop`. @@ -869,8 +870,18 @@ clipboard. Converting strings from Tcl to Python and back now never fails. (Many people worked on this for eight years but the problem was finally solved by Serhiy Storchaka in :issue:`13153`.) +New in 3.8.1: + +Add option to toggle cursor blink off. (Contributed by Zackery Spytz +in :issue:`4603`.) + +Escape key now closes IDLE completion windows. (Contributed by Johnny +Najera in :issue:`38944`.) + The changes above have been backported to 3.7 maintenance releases. +Add keywords to module name completion list. (Contributed by Terry J. +Reedy in :issue:`37765`.) inspect ------- @@ -1692,7 +1703,7 @@ Deprecated :meth:`~gettext.NullTranslations.set_output_charset`, and the *codeset* parameter of functions :func:`~gettext.translation` and :func:`~gettext.install` are also deprecated, since they are only used for - for the ``l*gettext()`` functions. + the ``l*gettext()`` functions. (Contributed by Serhiy Storchaka in :issue:`33710`.) * The :meth:`~threading.Thread.isAlive()` method of :class:`threading.Thread` @@ -1951,7 +1962,8 @@ Changes in the Python API (Contributed by Anthony Sottile in :issue:`36264`.) * The exception :class:`asyncio.CancelledError` now inherits from - :class:`BaseException` rather than :class:`Exception`. + :class:`BaseException` rather than :class:`Exception` and no longer inherits + from :class:`concurrent.futures.CancelledError`. (Contributed by Yury Selivanov in :issue:`32528`.) * The function :func:`asyncio.wait_for` now correctly waits for cancellation @@ -2101,9 +2113,6 @@ Changes in the C API (Contributed by Antoine Pitrou in :issue:`32388`.) -* The :c:func:`PyCode_New` has a new parameter in the second position (*posonlyargcount*) - to support :pep:`570`, indicating the number of positional-only arguments. - * The functions :c:func:`PyNode_AddChild` and :c:func:`PyParser_AddToken` now accept two additional ``int`` arguments *end_lineno* and *end_col_offset*. @@ -2225,3 +2234,16 @@ because of the behavior of the socket option ``SO_REUSEADDR`` in UDP. For more details, see the documentation for ``loop.create_datagram_endpoint()``. (Contributed by Kyle Stanley, Antoine Pitrou, and Yury Selivanov in :issue:`37228`.) + +Notable changes in Python 3.8.8 +=============================== + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) diff --git a/Doc/whatsnew/3.9.rst b/Doc/whatsnew/3.9.rst index 593f523828703e..c29715d192f953 100644 --- a/Doc/whatsnew/3.9.rst +++ b/Doc/whatsnew/3.9.rst @@ -4,22 +4,21 @@ :Release: |release| :Date: |today| +:Editor: Åukasz Langa .. Rules for maintenance: - * Anyone can add text to this document. Do not spend very much time - on the wording of your changes, because your text will probably - get rewritten to some degree. + * Anyone can add text to this document. Your text might get + rewritten to some degree. * The maintainer will go through Misc/NEWS periodically and add changes; it's therefore more important to add your changes to Misc/NEWS than to this file. * This is not a complete list of every single change; completeness - is the purpose of Misc/NEWS. Some changes I consider too small + is the purpose of Misc/NEWS. Some changes will be too small or esoteric to include. If such a change is added to the text, - I'll just remove it. (This is another reason you shouldn't spend - too much time on writing your addition.) + it might get removed during final editing. * If you want to draw your new text to the attention of the maintainer, add 'XXX' to the beginning of the paragraph or @@ -46,14 +45,13 @@ when researching a change. This article explains the new features in Python 3.9, compared to 3.8. +Python 3.9 was released on October 5th, 2020. For full details, see the :ref:`changelog `. -.. note:: +.. seealso:: - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.9 moves towards release, - so it's worth checking back even after reading earlier versions. + :pep:`596` - Python 3.9 Release Schedule Summary -- Release highlights @@ -62,27 +60,70 @@ Summary -- Release highlights .. This section singles out the most important changes in Python 3.9. Brevity is key. +New syntax features: + +* :pep:`584`, union operators added to ``dict``; +* :pep:`585`, type hinting generics in standard collections; +* :pep:`614`, relaxed grammar restrictions on decorators. + +New built-in features: + +* :pep:`616`, string methods to remove prefixes and suffixes. + +New features in the standard library: + +* :pep:`593`, flexible function and variable annotations; +* :func:`os.pidfd_open` added that allows process management without races + and signals. + +Interpreter improvements: -.. PEP-sized items next. +* :pep:`573`, fast access to module state from methods of C extension + types; +* :pep:`617`, CPython now uses a new parser based on PEG; +* a number of Python builtins (range, tuple, set, frozenset, list, dict) are + now sped up using :pep:`590` vectorcall; +* garbage collection does not block on resurrected objects; +* a number of Python modules (:mod:`_abc`, :mod:`audioop`, :mod:`_bz2`, + :mod:`_codecs`, :mod:`_contextvars`, :mod:`_crypt`, :mod:`_functools`, + :mod:`_json`, :mod:`_locale`, :mod:`math`, :mod:`operator`, :mod:`resource`, + :mod:`time`, :mod:`_weakref`) now use multiphase initialization as defined + by PEP 489; +* a number of standard library modules (:mod:`audioop`, :mod:`ast`, :mod:`grp`, + :mod:`_hashlib`, :mod:`pwd`, :mod:`_posixsubprocess`, :mod:`random`, + :mod:`select`, :mod:`struct`, :mod:`termios`, :mod:`zlib`) are now using + the stable ABI defined by PEP 384. + +New library modules: + +* :pep:`615`, the IANA Time Zone Database is now present in the standard + library in the :mod:`zoneinfo` module; +* an implementation of a topological sort of a graph is now provided in + the new :mod:`graphlib` module. + +Release process changes: + +* :pep:`602`, CPython adopts an annual release cycle. You should check for DeprecationWarning in your code ==================================================== -When Python 2.7 was still supported, many functions were kept for backward -compatibility with Python 2.7. With the end of Python 2.7 support, these -backward compatibility layers have been removed, or will be removed soon. -Most of them emitted a :exc:`DeprecationWarning` warning for several years. For -example, using ``collections.Mapping`` instead of ``collections.abc.Mapping`` -emits a :exc:`DeprecationWarning` since Python 3.3, released in 2012. +When Python 2.7 was still supported, a lot of functionality in Python 3 +was kept for backward compatibility with Python 2.7. With the end of Python +2 support, these backward compatibility layers have been removed, or will +be removed soon. Most of them emitted a :exc:`DeprecationWarning` warning for +several years. For example, using ``collections.Mapping`` instead of +``collections.abc.Mapping`` emits a :exc:`DeprecationWarning` since Python +3.3, released in 2012. Test your application with the :option:`-W` ``default`` command-line option to see :exc:`DeprecationWarning` and :exc:`PendingDeprecationWarning`, or even with :option:`-W` ``error`` to treat them as errors. :ref:`Warnings Filter ` can be used to ignore warnings from third-party code. -It has been decided to keep a few backward compatibility layers for one last -release, to give more time to Python projects maintainers to organize the +Python 3.9 is the last version providing those Python 2 backward compatibility +layers, to give more time to Python projects maintainers to organize the removal of the Python 2 support and add support for Python 3.9. Aliases to :ref:`Abstract Base Classes ` in @@ -94,6 +135,9 @@ More generally, try to run your tests in the :ref:`Python Development Mode ` which helps to prepare your code to make it compatible with the next Python version. +Note: a number of pre-existing deprecations were removed in this version of +Python as well. Consult the :ref:`removed-in-python-39` section. + New Features ============ @@ -102,21 +146,33 @@ Dictionary Merge & Update Operators ----------------------------------- Merge (``|``) and update (``|=``) operators have been added to the built-in -:class:`dict` class. See :pep:`584` for a full description. +:class:`dict` class. Those complement the existing ``dict.update`` and +``{**d1, **d2}`` methods of merging dictionaries. + +Example:: + + >>> x = {"key1": "value1 from x", "key2": "value2 from x"} + >>> y = {"key2": "value2 from y", "key3": "value3 from y"} + >>> x | y + {'key1': 'value1 from x', 'key2': 'value2 from y', 'key3': 'value3 from y'} + >>> y | x + {'key2': 'value2 from x', 'key3': 'value3 from y', 'key1': 'value1 from x'} + +See :pep:`584` for a full description. (Contributed by Brandt Bucher in :issue:`36144`.) -PEP 616: New removeprefix() and removesuffix() string methods -------------------------------------------------------------- +New String Methods to Remove Prefixes and Suffixes +-------------------------------------------------- :meth:`str.removeprefix(prefix)` and :meth:`str.removesuffix(suffix)` have been added to easily remove an unneeded prefix or a suffix from a string. Corresponding ``bytes``, ``bytearray``, and ``collections.UserString`` methods have also been added. See :pep:`616` for a full description. (Contributed by Dennis Sweeney in -:issue:`18939`.) +:issue:`39939`.) -PEP 585: Builtin Generic Types ------------------------------- +Type Hinting Generics in Standard Collections +--------------------------------------------- In type annotations you can now use built-in collection types such as ``list`` and ``dict`` as generic types instead of importing the @@ -135,8 +191,8 @@ Example: See :pep:`585` for more details. (Contributed by Guido van Rossum, Ethan Smith, and Batuhan TaÅŸkaya in :issue:`39481`.) -PEP 617: New Parser -------------------- +New Parser +---------- Python 3.9 uses a new parser, based on `PEG `_ instead @@ -156,7 +212,7 @@ back to the LL(1) parser using a command line switch (``-X oldparser``) or an environment variable (``PYTHONOLDPARSER=1``). See :pep:`617` for more details. (Contributed by Guido van Rossum, -Pablo Galindo and Lysandros Nikolau in :issue:`40334`.) +Pablo Galindo and Lysandros Nikolaou in :issue:`40334`.) Other Language Changes @@ -167,7 +223,6 @@ Other Language Changes its top-level package. (Contributed by Ngalim Siregar in :issue:`37444`.) - * Python now gets the absolute path of the script filename specified on the command line (ex: ``python3 script.py``): the ``__file__`` attribute of the :mod:`__main__` module became an absolute path, rather than a relative @@ -201,6 +256,21 @@ Other Language Changes for the correspondent concrete type (``list`` in this case). (Contributed by Serhiy Storchaka in :issue:`40257`.) +* Parallel running of :meth:`~agen.aclose` / :meth:`~agen.asend` / + :meth:`~agen.athrow` is now prohibited, and ``ag_running`` now reflects + the actual running status of the async generator. + (Contributed by Yury Selivanov in :issue:`30773`.) + +* Unexpected errors in calling the ``__iter__`` method are no longer masked by + ``TypeError`` in the :keyword:`in` operator and functions + :func:`~operator.contains`, :func:`~operator.indexOf` and + :func:`~operator.countOf` of the :mod:`operator` module. + (Contributed by Serhiy Storchaka in :issue:`40824`.) + +* Unparenthesized lambda expressions can no longer be the expression part in an + ``if`` clause in comprehensions and generator expressions. See :issue:`41848` + and :issue:`43755` for details. + New Modules =========== @@ -245,6 +315,15 @@ PyPI and maintained by the CPython core team. PEP written and implemented by Paul Ganssle +graphlib +--------- + +A new module, :mod:`graphlib`, was added that contains the +:class:`graphlib.TopologicalSorter` class to offer functionality to perform +topological sorting of graphs. (Contributed by Pablo Galindo, Tim Peters and +Larry Hastings in :issue:`17005`.) + + Improved Modules ================ @@ -282,6 +361,21 @@ that schedules a shutdown for the default executor that waits on the Added :class:`asyncio.PidfdChildWatcher`, a Linux-specific child watcher implementation that polls process file descriptors. (:issue:`38692`) +Added a new :term:`coroutine` :func:`asyncio.to_thread`. It is mainly used for +running IO-bound functions in a separate thread to avoid blocking the event +loop, and essentially works as a high-level version of +:meth:`~asyncio.loop.run_in_executor` that can directly take keyword arguments. +(Contributed by Kyle Stanley and Yury Selivanov in :issue:`32309`.) + +When cancelling the task due to a timeout, :meth:`asyncio.wait_for` will now +wait until the cancellation is complete also in the case when *timeout* is +<= 0, like it does with positive timeouts. +(Contributed by Elvis Pranskevichus in :issue:`32751`.) + +:mod:`asyncio` now raises :exc:`TyperError` when calling incompatible +methods with an :class:`ssl.SSLSocket` socket. +(Contributed by Ido Michael in :issue:`37404`.) + compileall ---------- @@ -314,7 +408,7 @@ startup overhead and reduces the amount of lost CPU time to idle workers. curses ------ -Add :func:`curses.get_escdelay`, :func:`curses.set_escdelay`, +Added :func:`curses.get_escdelay`, :func:`curses.set_escdelay`, :func:`curses.get_tabsize`, and :func:`curses.set_tabsize` functions. (Contributed by Anthony Sottile in :issue:`38312`.) @@ -325,6 +419,13 @@ and :meth:`~datetime.datetime.isocalendar()` of :class:`datetime.datetime` methods now returns a :func:`~collections.namedtuple` instead of a :class:`tuple`. (Contributed by Dong-hee Na in :issue:`24416`.) +distutils +--------- + +The :command:`upload` command now creates SHA2-256 and Blake2b-256 hash +digests. It skips MD5 on platforms that block MD5 digest. +(Contributed by Christian Heimes in :issue:`40698`.) + fcntl ----- @@ -339,13 +440,6 @@ ftplib if the given timeout for their constructor is zero to prevent the creation of a non-blocking socket. (Contributed by Dong-hee Na in :issue:`39259`.) -functools ---------- - -Add the :class:`functools.TopologicalSorter` class to offer functionality to perform -topological sorting of graphs. (Contributed by Pablo Galindo, Tim Peters and Larry -Hastings in :issue:`17005`.) - gc -- @@ -361,18 +455,37 @@ finalized by the garbage collector. (Contributed by Pablo Galindo in hashlib ------- +The :mod:`hashlib` module can now use SHA3 hashes and SHAKE XOF from OpenSSL +when available. +(Contributed by Christian Heimes in :issue:`37630`.) + Builtin hash modules can now be disabled with ``./configure --without-builtin-hashlib-hashes`` or selectively enabled with e.g. ``./configure --with-builtin-hashlib-hashes=sha3,blake2`` to force use of OpenSSL based implementation. (Contributed by Christian Heimes in :issue:`40479`) + http ---- HTTP status codes ``103 EARLY_HINTS``, ``418 IM_A_TEAPOT`` and ``425 TOO_EARLY`` are added to :class:`http.HTTPStatus`. (Contributed by Dong-hee Na in :issue:`39509` and Ross Rhodes in :issue:`39507`.) +IDLE and idlelib +---------------- + +Added option to toggle cursor blink off. (Contributed by Zackery Spytz +in :issue:`4603`.) + +Escape key now closes IDLE completion windows. (Contributed by Johnny +Najera in :issue:`38944`.) + +Added keywords to module name completion list. (Contributed by Terry J. +Reedy in :issue:`37765`.) + +The changes above have been backported to 3.8 maintenance releases. + imaplib ------- @@ -398,6 +511,17 @@ now raises :exc:`ImportError` instead of :exc:`ValueError` for invalid relative import attempts. (Contributed by Ngalim Siregar in :issue:`37444`.) +Import loaders which publish immutable module objects can now publish +immutable packages in addition to individual modules. +(Contributed by Dino Viehland in :issue:`39336`.) + +Added :func:`importlib.resources.files` function with support for +subdirectories in package data, matching backport in ``importlib_resources`` +version 1.5. +(Contributed by Jason R. Coombs in :issue:`39791`.) + +Refreshed ``importlib.metadata`` from ``importlib_metadata`` version 1.6.1. + inspect ------- @@ -413,6 +537,10 @@ Scoped IPv6 addresses can be parsed using :class:`ipaddress.IPv6Address`. If present, scope zone ID is available through the :attr:`~ipaddress.IPv6Address.scope_id` attribute. (Contributed by Oleksandr Pavliuk in :issue:`34788`.) +Starting with Python 3.9.5 the :mod:`ipaddress` module no longer +accepts any leading zeros in IPv4 address strings. +(Contributed by Christian Heimes in :issue:`36384`). + math ---- @@ -420,15 +548,15 @@ Expanded the :func:`math.gcd` function to handle multiple arguments. Formerly, it only supported two arguments. (Contributed by Serhiy Storchaka in :issue:`39648`.) -Add :func:`math.lcm`: return the least common multiple of specified arguments. +Added :func:`math.lcm`: return the least common multiple of specified arguments. (Contributed by Mark Dickinson, Ananthakrishnan and Serhiy Storchaka in :issue:`39479` and :issue:`39648`.) -Add :func:`math.nextafter`: return the next floating-point value after *x* +Added :func:`math.nextafter`: return the next floating-point value after *x* towards *y*. (Contributed by Victor Stinner in :issue:`39288`.) -Add :func:`math.ulp`: return the value of the least significant bit +Added :func:`math.ulp`: return the value of the least significant bit of a float. (Contributed by Victor Stinner in :issue:`39310`.) @@ -464,7 +592,7 @@ The :func:`os.putenv` and :func:`os.unsetenv` functions are now always available. (Contributed by Victor Stinner in :issue:`39395`.) -Add :func:`os.waitstatus_to_exitcode` function: +Added :func:`os.waitstatus_to_exitcode` function: convert a wait status to an exit code. (Contributed by Victor Stinner in :issue:`40094`.) @@ -475,6 +603,12 @@ Added :meth:`pathlib.Path.readlink()` which acts similarly to :func:`os.readlink`. (Contributed by Girts Folkmanis in :issue:`30618`) +pdb +--- + +On Windows now :class:`~pdb.Pdb` supports ``~/.pdbrc``. +(Contributed by Tim Hopper and Dan Lidral-Porter in :issue:`20523`.) + poplib ------ @@ -498,7 +632,7 @@ method etc, but for any object that has its own ``__doc__`` attribute. random ------ -Add a new :attr:`random.Random.randbytes` method: generate random bytes. +Added a new :attr:`random.Random.randbytes` method: generate random bytes. (Contributed by Victor Stinner in :issue:`40286`.) signal @@ -524,6 +658,14 @@ The :mod:`socket` module now exports the :data:`~socket.CAN_RAW_JOIN_FILTERS` constant on Linux 4.1 and greater. (Contributed by Stefan Tatschner and Zackery Spytz in :issue:`25780`.) +The socket module now supports the :data:`~socket.CAN_J1939` protocol on +platforms that support it. (Contributed by Karl Ding in :issue:`40291`.) + +The socket module now has the :func:`socket.send_fds` and +:func:`socket.recv_fds` functions. (Contributed by Joannah Nanjekye, Shinya +Okano and Victor Stinner in :issue:`28724`.) + + time ---- @@ -535,17 +677,22 @@ which has nanosecond resolution, rather than sys --- -Add a new :attr:`sys.platlibdir` attribute: name of the platform-specific -library directory. It is used to build the path of platform-specific dynamic -libraries and the path of the standard library. It is equal to ``"lib"`` on -most platforms. On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit -platforms. +Added a new :attr:`sys.platlibdir` attribute: name of the platform-specific +library directory. It is used to build the path of standard library and the +paths of installed extension modules. It is equal to ``"lib"`` on most +platforms. On Fedora and SuSE, it is equal to ``"lib64"`` on 64-bit platforms. (Contributed by Jan MatÄ›jek, MatÄ›j Cepl, Charalampos Stratakis and Victor Stinner in :issue:`1294959`.) Previously, :attr:`sys.stderr` was block-buffered when non-interactive. Now ``stderr`` defaults to always being line-buffered. (Contributed by Jendrik Seipp in :issue:`13601`.) +tracemalloc +----------- + +Added :func:`tracemalloc.reset_peak` to set the peak size of traced memory +blocks to the current size, to measure the peak of specific pieces of code. +(Contributed by Huon Wilson in :issue:`40630`.) typing ------ @@ -589,12 +736,12 @@ Optimizations sums = [s for s in [0] for x in data for s in [s + x]] - Unlike to the ``:=`` operator this idiom does not leak a variable to the + Unlike the ``:=`` operator this idiom does not leak a variable to the outer scope. (Contributed by Serhiy Storchaka in :issue:`32856`.) -* Optimize signal handling in multithreaded applications. If a thread different +* Optimized signal handling in multithreaded applications. If a thread different than the main thread gets a signal, the bytecode evaluation loop is no longer interrupted at each bytecode instruction to check for pending signals which cannot be handled. Only the main thread of the main interpreter can handle @@ -604,10 +751,91 @@ Optimizations until the main thread handles signals. (Contributed by Victor Stinner in :issue:`40010`.) -* Optimize the :mod:`subprocess` module on FreeBSD using ``closefrom()``. +* Optimized the :mod:`subprocess` module on FreeBSD using ``closefrom()``. (Contributed by Ed Maste, Conrad Meyer, Kyle Evans, Kubilay Kocak and Victor Stinner in :issue:`38061`.) +* :c:func:`PyLong_FromDouble` is now up to 1.87x faster for values that + fit into :c:type:`long`. + (Contributed by Sergey Fedoseev in :issue:`37986`.) + +* A number of Python builtins (:class:`range`, :class:`tuple`, :class:`set`, + :class:`frozenset`, :class:`list`, :class:`dict`) are now sped up by using + :pep:`590` vectorcall protocol. + (Contributed by Dong-hee Na, Mark Shannon, Jeroen Demeyer and Petr Viktorin in :issue:`37207`.) + +* Optimized :func:`~set.difference_update` for the case when the other set + is much larger than the base set. + (Suggested by Evgeny Kapun with code contributed by Michele Orrù in :issue:`8425`.) + +* Python's small object allocator (``obmalloc.c``) now allows (no more than) + one empty arena to remain available for immediate reuse, without returning + it to the OS. This prevents thrashing in simple loops where an arena could + be created and destroyed anew on each iteration. + (Contributed by Tim Peters in :issue:`37257`.) + +* :term:`floor division` of float operation now has a better performance. Also + the message of :exc:`ZeroDivisionError` for this operation is updated. + (Contributed by Dong-hee Na in :issue:`39434`.) + +* Decoding short ASCII strings with UTF-8 and ascii codecs is now about + 15% faster. (Contributed by Inada Naoki in :issue:`37348`.) + +Here's a summary of performance improvements from Python 3.4 through Python 3.9: + +.. code-block:: none + + Python version 3.4 3.5 3.6 3.7 3.8 3.9 + -------------- --- --- --- --- --- --- + + Variable and attribute read access: + read_local 7.1 7.1 5.4 5.1 3.9 3.9 + read_nonlocal 7.1 8.1 5.8 5.4 4.4 4.5 + read_global 15.5 19.0 14.3 13.6 7.6 7.8 + read_builtin 21.1 21.6 18.5 19.0 7.5 7.8 + read_classvar_from_class 25.6 26.5 20.7 19.5 18.4 17.9 + read_classvar_from_instance 22.8 23.5 18.8 17.1 16.4 16.9 + read_instancevar 32.4 33.1 28.0 26.3 25.4 25.3 + read_instancevar_slots 27.8 31.3 20.8 20.8 20.2 20.5 + read_namedtuple 73.8 57.5 45.0 46.8 18.4 18.7 + read_boundmethod 37.6 37.9 29.6 26.9 27.7 41.1 + + Variable and attribute write access: + write_local 8.7 9.3 5.5 5.3 4.3 4.3 + write_nonlocal 10.5 11.1 5.6 5.5 4.7 4.8 + write_global 19.7 21.2 18.0 18.0 15.8 16.7 + write_classvar 92.9 96.0 104.6 102.1 39.2 39.8 + write_instancevar 44.6 45.8 40.0 38.9 35.5 37.4 + write_instancevar_slots 35.6 36.1 27.3 26.6 25.7 25.8 + + Data structure read access: + read_list 24.2 24.5 20.8 20.8 19.0 19.5 + read_deque 24.7 25.5 20.2 20.6 19.8 20.2 + read_dict 24.3 25.7 22.3 23.0 21.0 22.4 + read_strdict 22.6 24.3 19.5 21.2 18.9 21.5 + + Data structure write access: + write_list 27.1 28.5 22.5 21.6 20.0 20.0 + write_deque 28.7 30.1 22.7 21.8 23.5 21.7 + write_dict 31.4 33.3 29.3 29.2 24.7 25.4 + write_strdict 28.4 29.9 27.5 25.2 23.1 24.5 + + Stack (or queue) operations: + list_append_pop 93.4 112.7 75.4 74.2 50.8 50.6 + deque_append_pop 43.5 57.0 49.4 49.2 42.5 44.2 + deque_append_popleft 43.7 57.3 49.7 49.7 42.8 46.4 + + Timing loop: + loop_overhead 0.5 0.6 0.4 0.3 0.3 0.3 + +These results were generated from the variable access benchmark script at: +``Tools/scripts/var_access_benchmark.py``. The benchmark script displays timings +in nanoseconds. The benchmarks were measured on an +`Intel® Coreâ„¢ i7-4960HQ processor +`_ +running the macOS 64-bit builds found at +`python.org `_. + Deprecated ========== @@ -622,9 +850,15 @@ Deprecated Python versions it will raise a :exc:`TypeError` for all floats. (Contributed by Serhiy Storchaka in :issue:`37315`.) -* The :mod:`parser` module is deprecated and will be removed in future versions - of Python. For the majority of use cases, users can leverage the Abstract Syntax - Tree (AST) generation and compilation stage, using the :mod:`ast` module. +* The :mod:`parser` and :mod:`symbol` modules are deprecated and will be + removed in future versions of Python. For the majority of use cases, + users can leverage the Abstract Syntax Tree (AST) generation and compilation + stage, using the :mod:`ast` module. + +* The Public C API functions :c:func:`PyParser_SimpleParseStringFlags`, + :c:func:`PyParser_SimpleParseStringFlagsFilename`, + :c:func:`PyParser_SimpleParseFileFlags` and :c:func:`PyNode_Compile` + are deprecated and will be removed in Python 3.10 together with the old parser. * Using :data:`NotImplemented` in a boolean context has been deprecated, as it is almost exclusively the result of incorrect rich comparator @@ -665,7 +899,7 @@ Deprecated and will be removed in future Python versions. ``value`` itself should be used instead of ``Index(value)``. ``Tuple(slices, Load())`` should be used instead of ``ExtSlice(slices)``. - (Contributed by Serhiy Storchaka in :issue:`32892`.) + (Contributed by Serhiy Storchaka in :issue:`34822`.) * :mod:`ast` classes ``Suite``, ``Param``, ``AugLoad`` and ``AugStore`` are considered deprecated and will be removed in future Python versions. @@ -683,6 +917,9 @@ Deprecated * Passing ``None`` as the first argument to the :func:`shlex.split` function has been deprecated. (Contributed by Zackery Spytz in :issue:`33262`.) +* :func:`smtpd.MailmanProxy` is now deprecated as it is unusable without + an external module, ``mailman``. (Contributed by Samuel Colvin in :issue:`35800`.) + * The :mod:`lib2to3` module now emits a :exc:`PendingDeprecationWarning`. Python 3.9 switched to a PEG parser (see :pep:`617`), and Python 3.10 may include new language syntax that is not parsable by lib2to3's LL(1) parser. @@ -691,9 +928,14 @@ Deprecated `parso`_. (Contributed by Carl Meyer in :issue:`40360`.) +* The *random* parameter of :func:`random.shuffle` has been deprecated. + (Contributed by Raymond Hettinger in :issue:`40465`) + .. _LibCST: https://libcst.readthedocs.io/ .. _parso: https://parso.readthedocs.io/ +.. _removed-in-python-39: + Removed ======= @@ -745,11 +987,6 @@ Removed module have been removed. They were deprecated in Python 3.2. Use ``iter(x)`` or ``list(x)`` instead of ``x.getchildren()`` and ``x.iter()`` or ``list(x.iter())`` instead of ``x.getiterator()``. - The ``xml.etree.cElementTree`` module has been removed, - use the :mod:`xml.etree.ElementTree` module instead. - Since Python 3.3 the ``xml.etree.cElementTree`` module has been deprecated, - the ``xml.etree.ElementTree`` module uses a fast implementation whenever - available. (Contributed by Serhiy Storchaka in :issue:`36543`.) * The old :mod:`plistlib` API has been removed, it was deprecated since Python @@ -758,9 +995,6 @@ Removed removed, standard :class:`bytes` objects are always used instead. (Contributed by Jon Janzen in :issue:`36409`.) -* The C function ``PyThreadState_DeleteCurrent()`` has been removed. It was not documented. - (Contributed by Joannah Nanjekye in :issue:`37878`.) - * The C function ``PyGen_NeedsFinalizing`` has been removed. It was not documented, tested, or used anywhere within CPython after the implementation of :pep:`442`. Patch by Joannah Nanjekye. @@ -797,7 +1031,7 @@ Removed (Contributed by Victor Stinner in :issue:`39489`.) * The ``_field_types`` attribute of the :class:`typing.NamedTuple` class - has been removed. It was deprecated deprecated since Python 3.8. Use + has been removed. It was deprecated since Python 3.8. Use the ``__annotations__`` attribute instead. (Contributed by Serhiy Storchaka in :issue:`40182`.) @@ -805,6 +1039,16 @@ Removed deprecated since 2006, and only returning ``False`` when it's called. (Contributed by Batuhan Taskaya in :issue:`40208`) +* The :meth:`asyncio.Task.current_task` and :meth:`asyncio.Task.all_tasks` + have been removed. They were deprecated since Python 3.7 and you can use + :func:`asyncio.current_task` and :func:`asyncio.all_tasks` instead. + (Contributed by Rémi Lapeyre in :issue:`40967`) + +* The ``unescape()`` method in the :class:`html.parser.HTMLParser` class + has been removed (it was deprecated since Python 3.4). :func:`html.unescape` + should be used for converting character references to the corresponding + unicode characters. + Porting to Python 3.9 ===================== @@ -861,6 +1105,87 @@ Changes in the Python API of ``wchar_t`` since Python 3.3. (Contributed by Inada Naoki in :issue:`34538`.) +* The :func:`logging.getLogger` API now returns the root logger when passed + the name ``'root'``, whereas previously it returned a non-root logger named + ``'root'``. This could affect cases where user code explicitly wants a + non-root logger named ``'root'``, or instantiates a logger using + ``logging.getLogger(__name__)`` in some top-level module called ``'root.py'``. + (Contributed by Vinay Sajip in :issue:`37742`.) + +* Division handling of :class:`~pathlib.PurePath` now returns ``NotImplemented`` + instead of raising a :exc:`TypeError` when passed something other than an + instance of ``str`` or :class:`~pathlib.PurePath`. This allows creating + compatible classes that don't inherit from those mentioned types. + (Contributed by Roger Aiudi in :issue:`34775`). + +* Starting with Python 3.9.5 the :mod:`ipaddress` module no longer + accepts any leading zeros in IPv4 address strings. Leading zeros are + ambiguous and interpreted as octal notation by some libraries. For example + the legacy function :func:`socket.inet_aton` treats leading zeros as octal + notatation. glibc implementation of modern :func:`~socket.inet_pton` does + not accept any leading zeros. + (Contributed by Christian Heimes in :issue:`36384`). + +* :func:`codecs.lookup` now normalizes the encoding name the same way as + :func:`encodings.normalize_encoding`, except that :func:`codecs.lookup` also + converts the name to lower case. For example, ``"latex+latin1"`` encoding + name is now normalized to ``"latex_latin1"``. + (Contributed by Jordon Xu in :issue:`37751`.) + + +Changes in the C API +-------------------- + +* Instances of heap-allocated types (such as those created with + :c:func:`PyType_FromSpec` and similar APIs) hold a reference to their type + object since Python 3.8. As indicated in the "Changes in the C API" of Python + 3.8, for the vast majority of cases, there should be no side effect but for + types that have a custom :c:member:`~PyTypeObject.tp_traverse` function, + ensure that all custom ``tp_traverse`` functions of heap-allocated types + visit the object's type. + + Example: + + .. code-block:: c + + int + foo_traverse(foo_struct *self, visitproc visit, void *arg) { + // Rest of the traverse function + #if PY_VERSION_HEX >= 0x03090000 + // This was not needed before Python 3.9 (Python issue 35810 and 40217) + Py_VISIT(Py_TYPE(self)); + #endif + } + + If your traverse function delegates to ``tp_traverse`` of its base class + (or another type), ensure that ``Py_TYPE(self)`` is visited only once. + Note that only heap types are expected to visit the type in ``tp_traverse``. + + For example, if your ``tp_traverse`` function includes: + + .. code-block:: c + + base->tp_traverse(self, visit, arg) + + then add: + + .. code-block:: c + + #if PY_VERSION_HEX >= 0x03090000 + // This was not needed before Python 3.9 (Python issue 35810 and 40217) + if (base->tp_flags & Py_TPFLAGS_HEAPTYPE) { + // a heap type's tp_traverse already visited Py_TYPE(self) + } else { + Py_VISIT(Py_TYPE(self)); + } + #else + + (See :issue:`35810` and :issue:`40217` for more information.) + +* The functions ``PyEval_CallObject``, ``PyEval_CallFunction``, + ``PyEval_CallMethod`` and ``PyEval_CallObjectWithKeywords`` are deprecated. + Use :c:func:`PyObject_Call` and its variants instead. + (See more details in :issue:`29548`.) CPython bytecode changes ------------------------ @@ -870,11 +1195,21 @@ CPython bytecode changes correctly if the :exc:`AssertionError` exception was being shadowed. (Contributed by Zackery Spytz in :issue:`34880`.) +* The :opcode:`COMPARE_OP` opcode was split into four distinct instructions: + + * ``COMPARE_OP`` for rich comparisons + * ``IS_OP`` for 'is' and 'is not' tests + * ``CONTAINS_OP`` for 'in' and 'not in' tests + * ``JUMP_IF_NOT_EXC_MATCH`` for checking exceptions in 'try-except' + statements. + + (Contributed by Mark Shannon in :issue:`39156`.) + Build Changes ============= -* Add ``--with-platlibdir`` option to the ``configure`` script: name of the +* Added ``--with-platlibdir`` option to the ``configure`` script: name of the platform-specific library directory, stored in the new :attr:`sys.platlibdir` attribute. See :attr:`sys.platlibdir` attribute for more information. (Contributed by Jan MatÄ›jek, MatÄ›j Cepl, Charalampos Stratakis @@ -887,6 +1222,34 @@ Build Changes functions are now required to build Python. (Contributed by Victor Stinner in :issue:`39395`.) +* On non-Windows platforms, creating ``bdist_wininst`` installers is now + officially unsupported. (See :issue:`10945` for more details.) + +* When building Python on macOS from source, ``_tkinter`` now links with + non-system Tcl and Tk frameworks if they are installed in + ``/Library/Frameworks``, as had been the case on older releases + of macOS. If a macOS SDK is explicitly configured, by using + ``--enable-universalsdk=`` or ``-isysroot``, only the SDK itself is + searched. The default behavior can still be overridden with + ``--with-tcltk-includes`` and ``--with-tcltk-libs``. + (Contributed by Ned Deily in :issue:`34956`.) + +* Python can now be built for Windows 10 ARM64. + (Contributed by Steve Dower in :issue:`33125`.) + +* Some individual tests are now skipped when ``--pgo`` is used. The tests + in question increased the PGO task time significantly and likely + didn't help improve optimization of the final executable. This + speeds up the task by a factor of about 15x. Running the full unit test + suite is slow. This change may result in a slightly less optimized build + since not as many code branches will be executed. If you are willing to + wait for the much slower build, the old behavior can be restored using + ``./configure [..] PROFILE_TASK="-m test --pgo-extended"``. We make no + guarantees as to which PGO task set produces a faster build. Users who care + should run their own relevant benchmarks as results can depend on the + environment, workload, and compiler tool chain. + (See :issue:`36044` and :issue:`37707` for more details.) + C API Changes ============= @@ -894,22 +1257,29 @@ C API Changes New Features ------------ -* Add :c:func:`PyFrame_GetCode` function: get a frame code. - Add :c:func:`PyFrame_GetBack` function: get the frame next outer frame. +* :pep:`573`: Added :c:func:`PyType_FromModuleAndSpec` to associate + a module with a class; :c:func:`PyType_GetModule` and + :c:func:`PyType_GetModuleState` to retrieve the module and its state; and + :c:data:`PyCMethod` and :c:data:`METH_METHOD` to allow a method to + access the class it was defined in. + (Contributed by Marcel Plch and Petr Viktorin in :issue:`38787`.) + +* Added :c:func:`PyFrame_GetCode` function: get a frame code. + Added :c:func:`PyFrame_GetBack` function: get the frame next outer frame. (Contributed by Victor Stinner in :issue:`40421`.) -* Add :c:func:`PyFrame_GetLineNumber` to the limited C API. +* Added :c:func:`PyFrame_GetLineNumber` to the limited C API. (Contributed by Victor Stinner in :issue:`40421`.) -* Add :c:func:`PyThreadState_GetInterpreter` and +* Added :c:func:`PyThreadState_GetInterpreter` and :c:func:`PyInterpreterState_Get` functions to get the interpreter. - Add :c:func:`PyThreadState_GetFrame` function to get the current frame of a + Added :c:func:`PyThreadState_GetFrame` function to get the current frame of a Python thread state. - Add :c:func:`PyThreadState_GetID` function: get the unique identifier of a + Added :c:func:`PyThreadState_GetID` function: get the unique identifier of a Python thread state. (Contributed by Victor Stinner in :issue:`39947`.) -* Add a new public :c:func:`PyObject_CallNoArgs` function to the C API, which +* Added a new public :c:func:`PyObject_CallNoArgs` function to the C API, which calls a callable Python object without any arguments. It is the most efficient way to call a callable Python object without any argument. (Contributed by Victor Stinner in :issue:`37194`.) @@ -931,11 +1301,19 @@ New Features to a module. (Contributed by Dong-hee Na in :issue:`40024`.) -* Add the functions :c:func:`PyObject_GC_IsTracked` and +* Added the functions :c:func:`PyObject_GC_IsTracked` and :c:func:`PyObject_GC_IsFinalized` to the public API to allow to query if Python objects are being currently tracked or have been already finalized by - the garbage collector respectively. (Contributed by Pablo Galindo in - :issue:`40241`.) + the garbage collector respectively. + (Contributed by Pablo Galindo Salgado in :issue:`40241`.) + +* Added :c:func:`_PyObject_FunctionStr` to get a user-friendly string + representation of a function-like object. + (Patch by Jeroen Demeyer in :issue:`37645`.) + +* Added :c:func:`PyObject_CallOneArg` for calling an object with one + positional argument + (Patch by Jeroen Demeyer in :issue:`37483`.) Porting to Python 3.9 @@ -971,11 +1349,55 @@ Porting to Python 3.9 and refers to a constant string. (Contributed by Serhiy Storchaka in :issue:`38650`.) +* The :c:type:`PyGC_Head` structure is now opaque. It is only defined in the + internal C API (``pycore_gc.h``). + (Contributed by Victor Stinner in :issue:`40241`.) + +* The ``Py_UNICODE_COPY``, ``Py_UNICODE_FILL``, ``PyUnicode_WSTR_LENGTH``, + :c:func:`PyUnicode_FromUnicode`, :c:func:`PyUnicode_AsUnicode`, + ``_PyUnicode_AsUnicode``, and :c:func:`PyUnicode_AsUnicodeAndSize` are + marked as deprecated in C. They have been deprecated by :pep:`393` since + Python 3.3. + (Contributed by Inada Naoki in :issue:`36346`.) + +* The :c:func:`Py_FatalError` function is replaced with a macro which logs + automatically the name of the current function, unless the + ``Py_LIMITED_API`` macro is defined. + (Contributed by Victor Stinner in :issue:`39882`.) + +* The vectorcall protocol now requires that the caller passes only strings as + keyword names. (See :issue:`37540` for more information.) + +* Implementation details of a number of macros and functions are now hidden: + + * :c:func:`PyObject_IS_GC` macro was converted to a function. + + * The :c:func:`PyObject_NEW` macro becomes an alias to the + :c:func:`PyObject_New` macro, and the :c:func:`PyObject_NEW_VAR` macro + becomes an alias to the :c:func:`PyObject_NewVar` macro. They no longer + access directly the :c:member:`PyTypeObject.tp_basicsize` member. + + * :c:func:`PyType_HasFeature` now always calls :c:func:`PyType_GetFlags`. + Previously, it accessed directly the :c:member:`PyTypeObject.tp_flags` + member when the limited C API was not used. + + * :c:func:`PyObject_GET_WEAKREFS_LISTPTR` macro was converted to a function: + the macro accessed directly the :c:member:`PyTypeObject.tp_weaklistoffset` + member. + + * :c:func:`PyObject_CheckBuffer` macro was converted to a function: the macro + accessed directly the :c:member:`PyTypeObject.tp_as_buffer` member. + + * :c:func:`PyIndex_Check` is now always declared as an opaque function to hide + implementation details: removed the ``PyIndex_Check()`` macro. The macro accessed + directly the :c:member:`PyTypeObject.tp_as_number` member. + + (See :issue:`40170` for more details.) Removed ------- -* Exclude ``PyFPE_START_PROTECT()`` and ``PyFPE_END_PROTECT()`` macros of +* Excluded ``PyFPE_START_PROTECT()`` and ``PyFPE_END_PROTECT()`` macros of ``pyfpe.h`` from the limited C API. (Contributed by Victor Stinner in :issue:`38835`.) @@ -986,8 +1408,10 @@ Removed * Changes in the limited C API (if ``Py_LIMITED_API`` macro is defined): - * Exclude the following functions from the limited C API: + * Excluded the following functions from the limited C API: + * ``PyThreadState_DeleteCurrent()`` + (Contributed by Joannah Nanjekye in :issue:`37878`.) * ``_Py_CheckRecursionLimit`` * ``_Py_NewReference()`` * ``_Py_ForgetReference()`` @@ -1001,7 +1425,7 @@ Removed * ``Py_TRASHCAN_SAFE_BEGIN`` * ``Py_TRASHCAN_SAFE_END`` - * Move following functions and definitions to the internal C API: + * Moved following functions and definitions to the internal C API: * ``_PyDebug_PrintTotalRefs()`` * ``_Py_PrintReferences()`` @@ -1011,12 +1435,12 @@ Removed (Contributed by Victor Stinner in :issue:`38644` and :issue:`39542`.) -* Remove ``_PyRuntime.getframe`` hook and remove ``_PyThreadState_GetFrame`` +* Removed ``_PyRuntime.getframe`` hook and removed ``_PyThreadState_GetFrame`` macro which was an alias to ``_PyRuntime.getframe``. They were only exposed - by the internal C API. Remove also ``PyThreadFrameGetter`` type. + by the internal C API. Removed also ``PyThreadFrameGetter`` type. (Contributed by Victor Stinner in :issue:`39946`.) -* Remove the following functions from the C API. Call :c:func:`PyGC_Collect` +* Removed the following functions from the C API. Call :c:func:`PyGC_Collect` explicitly to clear all free lists. (Contributed by Inada Naoki and Victor Stinner in :issue:`37340`, :issue:`38896` and :issue:`40428`.) @@ -1035,5 +1459,118 @@ Removed * ``PyUnicode_ClearFreeList()``: the Unicode free list has been removed in Python 3.3. -* Remove ``_PyUnicode_ClearStaticStrings()`` function. +* Removed ``_PyUnicode_ClearStaticStrings()`` function. (Contributed by Victor Stinner in :issue:`39465`.) + +* Removed ``Py_UNICODE_MATCH``. It has been deprecated by :pep:`393`, and + broken since Python 3.3. The :c:func:`PyUnicode_Tailmatch` function can be + used instead. + (Contributed by Inada Naoki in :issue:`36346`.) + +* Cleaned header files of interfaces defined but with no implementation. + The public API symbols being removed are: + ``_PyBytes_InsertThousandsGroupingLocale``, + ``_PyBytes_InsertThousandsGrouping``, ``_Py_InitializeFromArgs``, + ``_Py_InitializeFromWideArgs``, ``_PyFloat_Repr``, ``_PyFloat_Digits``, + ``_PyFloat_DigitsInit``, ``PyFrame_ExtendStack``, ``_PyAIterWrapper_Type``, + ``PyNullImporter_Type``, ``PyCmpWrapper_Type``, ``PySortWrapper_Type``, + ``PyNoArgsFunction``. + (Contributed by Pablo Galindo Salgado in :issue:`39372`.) + +Notable changes in Python 3.9.1 +=============================== + +typing +------ + +The behavior of :class:`typing.Literal` was changed to conform with :pep:`586` +and to match the behavior of static type checkers specified in the PEP. + +1. ``Literal`` now de-duplicates parameters. +2. Equality comparisons between ``Literal`` objects are now order independent. +3. ``Literal`` comparisons now respect types. For example, + ``Literal[0] == Literal[False]`` previously evaluated to ``True``. It is + now ``False``. To support this change, the internally used type cache now + supports differentiating types. +4. ``Literal`` objects will now raise a :exc:`TypeError` exception during + equality comparisons if any of their parameters are not :term:`hashable`. + Note that declaring ``Literal`` with mutable parameters will not throw + an error:: + + >>> from typing import Literal + >>> Literal[{0}] + >>> Literal[{0}] == Literal[{False}] + Traceback (most recent call last): + File "", line 1, in + TypeError: unhashable type: 'set' + +(Contributed by Yurii Karabas in :issue:`42345`.) + +macOS 11.0 (Big Sur) and Apple Silicon Mac support +-------------------------------------------------- + +As of 3.9.1, Python now fully supports building and running on macOS 11.0 +(Big Sur) and on Apple Silicon Macs (based on the ``ARM64`` architecture). +A new universal build variant, ``universal2``, is now available to natively +support both ``ARM64`` and ``Intel 64`` in one set of executables. Binaries +can also now be built on current versions of macOS to be deployed on a range +of older macOS versions (tested to 10.9) while making some newer OS +functions and options conditionally available based on the operating system +version in use at runtime ("weaklinking"). + +(Contributed by Ronald Oussoren and Lawrence D'Anna in :issue:`41100`.) + +Notable changes in Python 3.9.2 +=============================== + +collections.abc +--------------- + +:class:`collections.abc.Callable` generic now flattens type parameters, similar +to what :data:`typing.Callable` currently does. This means that +``collections.abc.Callable[[int, str], str]`` will have ``__args__`` of +``(int, str, str)``; previously this was ``([int, str], str)``. To allow this +change, :class:`types.GenericAlias` can now be subclassed, and a subclass will +be returned when subscripting the :class:`collections.abc.Callable` type. +Code which accesses the arguments via :func:`typing.get_args` or ``__args__`` +need to account for this change. A :exc:`DeprecationWarning` may be emitted for +invalid forms of parameterizing :class:`collections.abc.Callable` which may have +passed silently in Python 3.9.1. This :exc:`DeprecationWarning` will +become a :exc:`TypeError` in Python 3.10. +(Contributed by Ken Jin in :issue:`42195`.) + +urllib.parse +------------ + +Earlier Python versions allowed using both ``;`` and ``&`` as +query parameter separators in :func:`urllib.parse.parse_qs` and +:func:`urllib.parse.parse_qsl`. Due to security concerns, and to conform with +newer W3C recommendations, this has been changed to allow only a single +separator key, with ``&`` as the default. This change also affects +:func:`cgi.parse` and :func:`cgi.parse_multipart` as they use the affected +functions internally. For more details, please see their respective +documentation. +(Contributed by Adam Goldschmidt, Senthil Kumaran and Ken Jin in :issue:`42967`.) + +Notable changes in Python 3.9.3 +=============================== + +A security fix alters the :class:`ftplib.FTP` behavior to not trust the +IPv4 address sent from the remote server when setting up a passive data +channel. We reuse the ftp server IP address instead. For unusual code +requiring the old behavior, set a ``trust_server_pasv_ipv4_address`` +attribute on your FTP instance to ``True``. (See :issue:`43285`) + +Notable changes in Python 3.9.5 +=============================== + +urllib.parse +------------ + +The presence of newline or tab characters in parts of a URL allows for some +forms of attacks. Following the WHATWG specification that updates :rfc:`3986`, +ASCII newline ``\n``, ``\r`` and tab ``\t`` characters are stripped from the +URL by the parser in :mod:`urllib.parse` preventing such attacks. The removal +characters are controlled by a new module level variable +``urllib.parse._UNSAFE_URL_BYTES_TO_REMOVE``. (See :issue:`43882`) + diff --git a/Grammar/python.gram b/Grammar/python.gram index 40e7818d496022..544b4f794e08f6 100644 --- a/Grammar/python.gram +++ b/Grammar/python.gram @@ -1,6 +1,5 @@ -# Simplified grammar for Python +# PEG grammar for Python -@bytecode True @trailer ''' void * _PyPegen_parse(Parser *p) @@ -92,9 +91,9 @@ assignment[stmt_ty]: | a=('(' b=single_target ')' { b } | single_subscript_attribute_target) ':' b=expression c=['=' d=annotated_rhs { d }] { CHECK_VERSION(6, "Variable annotations syntax is", _Py_AnnAssign(a, b, c, 0, EXTRA)) } - | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) tc=[TYPE_COMMENT] { + | a=(z=star_targets '=' { z })+ b=(yield_expr | star_expressions) !'=' tc=[TYPE_COMMENT] { _Py_Assign(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | a=single_target b=augassign c=(yield_expr | star_expressions) { + | a=single_target b=augassign ~ c=(yield_expr | star_expressions) { _Py_AugAssign(a, b->kind, c, EXTRA) } | invalid_assignment @@ -122,7 +121,9 @@ yield_stmt[stmt_ty]: y=yield_expr { _Py_Expr(y, EXTRA) } assert_stmt[stmt_ty]: 'assert' a=expression b=[',' z=expression { z }] { _Py_Assert(a, b, EXTRA) } -del_stmt[stmt_ty]: 'del' a=del_targets { _Py_Delete(a, EXTRA) } +del_stmt[stmt_ty]: + | 'del' a=del_targets &(';' | NEWLINE) { _Py_Delete(a, EXTRA) } + | invalid_del_stmt import_stmt[stmt_ty]: import_name | import_from import_name[stmt_ty]: 'import' a=dotted_as_names { _Py_Import(a, EXTRA) } @@ -134,8 +135,9 @@ import_from[stmt_ty]: _Py_ImportFrom(NULL, b, _PyPegen_seq_count_dots(a), EXTRA) } import_from_targets[asdl_seq*]: | '(' a=import_from_as_names [','] ')' { a } - | import_from_as_names + | import_from_as_names !',' | '*' { _PyPegen_singleton_seq(p, CHECK(_PyPegen_alias_for_star(p))) } + | invalid_import_from_targets import_from_as_names[asdl_seq*]: | a=','.import_from_as_name+ { a } import_from_as_name[alias_ty]: @@ -164,10 +166,11 @@ while_stmt[stmt_ty]: | 'while' a=named_expression ':' b=block c=[else_block] { _Py_While(a, b, c, EXTRA) } for_stmt[stmt_ty]: - | 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { + | 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { _Py_For(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA) } - | ASYNC 'for' t=star_targets 'in' ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { + | ASYNC 'for' t=star_targets 'in' ~ ex=star_expressions ':' tc=[TYPE_COMMENT] b=block el=[else_block] { CHECK_VERSION(5, "Async for loops are", _Py_AsyncFor(t, ex, b, el, NEW_TYPE_COMMENT(p, tc), EXTRA)) } + | invalid_for_target with_stmt[stmt_ty]: | 'with' '(' a=','.with_item+ ','? ')' ':' b=block { @@ -179,7 +182,9 @@ with_stmt[stmt_ty]: | ASYNC 'with' a=','.with_item+ ':' tc=[TYPE_COMMENT] b=block { CHECK_VERSION(5, "Async with statements are", _Py_AsyncWith(a, b, NEW_TYPE_COMMENT(p, tc), EXTRA)) } with_item[withitem_ty]: - | e=expression o=['as' t=target { t }] { _Py_withitem(e, o, p->arena) } + | e=expression 'as' t=star_target &(',' | ')' | ':') { _Py_withitem(e, t, p->arena) } + | invalid_with_item + | e=expression { _Py_withitem(e, NULL, p->arena) } try_stmt[stmt_ty]: | 'try' ':' b=block f=finally_block { _Py_Try(b, NULL, NULL, f, EXTRA) } @@ -296,7 +301,6 @@ block[asdl_seq*] (memo): | simple_stmt | invalid_block -expressions_list[asdl_seq*]: a=','.star_expression+ [','] { a } star_expressions[expr_ty]: | a=star_expression b=(',' c=star_expression { c })+ [','] { _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Load, EXTRA) } @@ -311,7 +315,7 @@ star_named_expression[expr_ty]: | '*' a=bitwise_or { _Py_Starred(a, Load, EXTRA) } | named_expression named_expression[expr_ty]: - | a=NAME ':=' b=expression { _Py_NamedExpr(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } + | a=NAME ':=' ~ b=expression { _Py_NamedExpr(CHECK(_PyPegen_set_expr_context(p, a, Store)), b, EXTRA) } | expression !':=' | invalid_named_expression @@ -328,7 +332,11 @@ expression[expr_ty] (memo): | lambdef lambdef[expr_ty]: - | 'lambda' a=[lambda_parameters] ':' b=expression { _Py_Lambda((a) ? a : CHECK(_PyPegen_empty_arguments(p)), b, EXTRA) } + | 'lambda' a=[lambda_params] ':' b=expression { _Py_Lambda((a) ? a : CHECK(_PyPegen_empty_arguments(p)), b, EXTRA) } + +lambda_params[arguments_ty]: + | invalid_lambda_parameters + | lambda_parameters # lambda_parameters etc. duplicates parameters but without annotations # or type comments, and if there's no comma after a parameter, we expect @@ -404,7 +412,7 @@ compare_op_bitwise_or_pair[CmpopExprPair*]: | is_bitwise_or eq_bitwise_or[CmpopExprPair*]: '==' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Eq, a) } noteq_bitwise_or[CmpopExprPair*]: - | (tok='!=' {_PyPegen_check_barry_as_flufl(p) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } + | (tok='!=' { _PyPegen_check_barry_as_flufl(p, tok) ? NULL : tok}) a=bitwise_or {_PyPegen_cmpop_expr_pair(p, NotEq, a) } lte_bitwise_or[CmpopExprPair*]: '<=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, LtE, a) } lt_bitwise_or[CmpopExprPair*]: '<' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, Lt, a) } gte_bitwise_or[CmpopExprPair*]: '>=' a=bitwise_or { _PyPegen_cmpop_expr_pair(p, GtE, a) } @@ -451,6 +459,7 @@ await_primary[expr_ty] (memo): | AWAIT a=primary { CHECK_VERSION(5, "Await expressions are", _Py_Await(a, EXTRA)) } | primary primary[expr_ty]: + | invalid_primary # must be before 'primay genexp' because of invalid_genexp | a=primary '.' b=NAME { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } | a=primary b=genexp { _Py_Call(a, CHECK(_PyPegen_singleton_seq(p, b)), NULL, EXTRA) } | a=primary '(' b=[arguments] ')' { @@ -472,7 +481,7 @@ atom[expr_ty]: | 'True' { _Py_Constant(Py_True, NULL, EXTRA) } | 'False' { _Py_Constant(Py_False, NULL, EXTRA) } | 'None' { _Py_Constant(Py_None, NULL, EXTRA) } - | '__new_parser__' { RAISE_SYNTAX_ERROR("You found it!") } + | '__peg_parser__' { RAISE_SYNTAX_ERROR("You found it!") } | &STRING strings | NUMBER | &'(' (tuple | group | genexp) @@ -484,35 +493,40 @@ strings[expr_ty] (memo): a=STRING+ { _PyPegen_concatenate_strings(p, a) } list[expr_ty]: | '[' a=[star_named_expressions] ']' { _Py_List(a, Load, EXTRA) } listcomp[expr_ty]: - | '[' a=named_expression b=for_if_clauses ']' { _Py_ListComp(a, b, EXTRA) } + | '[' a=named_expression ~ b=for_if_clauses ']' { _Py_ListComp(a, b, EXTRA) } | invalid_comprehension tuple[expr_ty]: | '(' a=[y=star_named_expression ',' z=[star_named_expressions] { _PyPegen_seq_insert_in_front(p, y, z) } ] ')' { _Py_Tuple(a, Load, EXTRA) } -group[expr_ty]: '(' a=(yield_expr | named_expression) ')' { a } +group[expr_ty]: + | '(' a=(yield_expr | named_expression) ')' { a } + | invalid_group genexp[expr_ty]: - | '(' a=expression b=for_if_clauses ')' { _Py_GeneratorExp(a, b, EXTRA) } + | '(' a=named_expression ~ b=for_if_clauses ')' { _Py_GeneratorExp(a, b, EXTRA) } | invalid_comprehension -set[expr_ty]: '{' a=expressions_list '}' { _Py_Set(a, EXTRA) } +set[expr_ty]: '{' a=star_named_expressions '}' { _Py_Set(a, EXTRA) } setcomp[expr_ty]: - | '{' a=expression b=for_if_clauses '}' { _Py_SetComp(a, b, EXTRA) } + | '{' a=named_expression ~ b=for_if_clauses '}' { _Py_SetComp(a, b, EXTRA) } | invalid_comprehension dict[expr_ty]: - | '{' a=[kvpairs] '}' { _Py_Dict(CHECK(_PyPegen_get_keys(p, a)), - CHECK(_PyPegen_get_values(p, a)), EXTRA) } + | '{' a=[double_starred_kvpairs] '}' { + _Py_Dict(CHECK(_PyPegen_get_keys(p, a)), CHECK(_PyPegen_get_values(p, a)), EXTRA) } dictcomp[expr_ty]: | '{' a=kvpair b=for_if_clauses '}' { _Py_DictComp(a->key, a->value, b, EXTRA) } -kvpairs[asdl_seq*]: a=','.kvpair+ [','] { a } -kvpair[KeyValuePair*]: + | invalid_dict_comprehension +double_starred_kvpairs[asdl_seq*]: a=','.double_starred_kvpair+ [','] { a } +double_starred_kvpair[KeyValuePair*]: | '**' a=bitwise_or { _PyPegen_key_value_pair(p, NULL, a) } - | a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) } + | kvpair +kvpair[KeyValuePair*]: a=expression ':' b=expression { _PyPegen_key_value_pair(p, a, b) } for_if_clauses[asdl_seq*]: | for_if_clause+ for_if_clause[comprehension_ty]: - | ASYNC 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* { + | ASYNC 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* { CHECK_VERSION(6, "Async comprehensions are", _Py_comprehension(a, b, c, 1, p->arena)) } - | 'for' a=star_targets 'in' b=disjunction c=('if' z=disjunction { z })* { + | 'for' a=star_targets 'in' ~ b=disjunction c=('if' z=disjunction { z })* { _Py_comprehension(a, b, c, 0, p->arena) } + | invalid_for_target yield_expr[expr_ty]: | 'yield' 'from' a=expression { _Py_YieldFrom(a, EXTRA) } @@ -520,24 +534,13 @@ yield_expr[expr_ty]: arguments[expr_ty] (memo): | a=args [','] &')' { a } - | incorrect_arguments + | invalid_arguments args[expr_ty]: - | a=starred_expression b=[',' c=args { c }] { - _Py_Call(_PyPegen_dummy_name(p), - (b) ? CHECK(_PyPegen_seq_insert_in_front(p, a, ((expr_ty) b)->v.Call.args)) - : CHECK(_PyPegen_singleton_seq(p, a)), - (b) ? ((expr_ty) b)->v.Call.keywords : NULL, - EXTRA) } + | a=','.(starred_expression | named_expression !'=')+ b=[',' k=kwargs {k}] { _PyPegen_collect_call_seqs(p, a, b, EXTRA) } | a=kwargs { _Py_Call(_PyPegen_dummy_name(p), CHECK_NULL_ALLOWED(_PyPegen_seq_extract_starred_exprs(p, a)), CHECK_NULL_ALLOWED(_PyPegen_seq_delete_starred_exprs(p, a)), EXTRA) } - | a=named_expression b=[',' c=args { c }] { - _Py_Call(_PyPegen_dummy_name(p), - (b) ? CHECK(_PyPegen_seq_insert_in_front(p, a, ((expr_ty) b)->v.Call.args)) - : CHECK(_PyPegen_singleton_seq(p, a)), - (b) ? ((expr_ty) b)->v.Call.keywords : NULL, - EXTRA) } kwargs[asdl_seq*]: | a=','.kwarg_or_starred+ ',' b=','.kwarg_or_double_starred+ { _PyPegen_join_sequences(p, a, b) } | ','.kwarg_or_starred+ @@ -560,18 +563,23 @@ star_targets[expr_ty]: | a=star_target !',' { a } | a=star_target b=(',' c=star_target { c })* [','] { _Py_Tuple(CHECK(_PyPegen_seq_insert_in_front(p, a, b)), Store, EXTRA) } -star_targets_seq[asdl_seq*]: a=','.star_target+ [','] { a } +star_targets_list_seq[asdl_seq*]: a=','.star_target+ [','] { a } +star_targets_tuple_seq[asdl_seq*]: + | a=star_target b=(',' c=star_target { c })+ [','] { _PyPegen_seq_insert_in_front(p, a, b) } + | a=star_target ',' { _PyPegen_singleton_seq(p, a) } star_target[expr_ty] (memo): | '*' a=(!'*' star_target) { _Py_Starred(CHECK(_PyPegen_set_expr_context(p, a, Store)), Store, EXTRA) } + | target_with_star_atom +target_with_star_atom[expr_ty] (memo): | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } | star_atom star_atom[expr_ty]: | a=NAME { _PyPegen_set_expr_context(p, a, Store) } - | '(' a=star_target ')' { _PyPegen_set_expr_context(p, a, Store) } - | '(' a=[star_targets_seq] ')' { _Py_Tuple(a, Store, EXTRA) } - | '[' a=[star_targets_seq] ']' { _Py_List(a, Store, EXTRA) } + | '(' a=target_with_star_atom ')' { _PyPegen_set_expr_context(p, a, Store) } + | '(' a=[star_targets_tuple_seq] ')' { _Py_Tuple(a, Store, EXTRA) } + | '[' a=[star_targets_list_seq] ']' { _Py_List(a, Store, EXTRA) } single_target[expr_ty]: | single_subscript_attribute_target @@ -582,25 +590,16 @@ single_subscript_attribute_target[expr_ty]: | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } del_targets[asdl_seq*]: a=','.del_target+ [','] { a } -# The lookaheads to del_target_end ensure that we don't match expressions where a prefix of the -# expression matches our rule, thereby letting these cases fall through to invalid_del_target. del_target[expr_ty] (memo): - | a=t_primary '.' b=NAME &del_target_end { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) } - | a=t_primary '[' b=slices ']' &del_target_end { _Py_Subscript(a, b, Del, EXTRA) } + | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Del, EXTRA) } + | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Del, EXTRA) } | del_t_atom del_t_atom[expr_ty]: - | a=NAME &del_target_end { _PyPegen_set_expr_context(p, a, Del) } + | a=NAME { _PyPegen_set_expr_context(p, a, Del) } | '(' a=del_target ')' { _PyPegen_set_expr_context(p, a, Del) } | '(' a=[del_targets] ')' { _Py_Tuple(a, Del, EXTRA) } | '[' a=[del_targets] ']' { _Py_List(a, Del, EXTRA) } - | invalid_del_target -del_target_end: ')' | ']' | ',' | ';' | NEWLINE -targets[asdl_seq*]: a=','.target+ [','] { a } -target[expr_ty] (memo): - | a=t_primary '.' b=NAME !t_lookahead { _Py_Attribute(a, b->v.Name.id, Store, EXTRA) } - | a=t_primary '[' b=slices ']' !t_lookahead { _Py_Subscript(a, b, Store, EXTRA) } - | t_atom t_primary[expr_ty]: | a=t_primary '.' b=NAME &t_lookahead { _Py_Attribute(a, b->v.Name.id, Load, EXTRA) } | a=t_primary '[' b=slices ']' &t_lookahead { _Py_Subscript(a, b, Load, EXTRA) } @@ -612,21 +611,18 @@ t_primary[expr_ty]: EXTRA) } | a=atom &t_lookahead { a } t_lookahead: '(' | '[' | '.' -t_atom[expr_ty]: - | a=NAME { _PyPegen_set_expr_context(p, a, Store) } - | '(' a=target ')' { _PyPegen_set_expr_context(p, a, Store) } - | '(' b=[targets] ')' { _Py_Tuple(b, Store, EXTRA) } - | '[' b=[targets] ']' { _Py_List(b, Store, EXTRA) } - # From here on, there are rules for invalid syntax with specialised error messages -incorrect_arguments: +invalid_arguments: | args ',' '*' { RAISE_SYNTAX_ERROR("iterable argument unpacking follows keyword argument unpacking") } | a=expression for_if_clauses ',' [args | expression for_if_clauses] { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } + | a=args for_if_clauses { _PyPegen_nonparen_genexp_in_call(p, a) } + | args ',' a=expression for_if_clauses { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "Generator expression must be parenthesized") } | a=args ',' args { _PyPegen_arguments_parsing_error(p, a) } invalid_kwarg: - | a=expression '=' { + | !(NAME '=') a=expression b='=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "expression cannot contain assignment, perhaps you meant \"==\"?") } invalid_named_expression: @@ -634,31 +630,48 @@ invalid_named_expression: RAISE_SYNTAX_ERROR_KNOWN_LOCATION( a, "cannot use assignment expressions with %s", _PyPegen_get_expr_name(a)) } invalid_assignment: - | a=list ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not list) can be annotated") } - | a=tuple ':' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") } - | a=star_named_expression ',' star_named_expressions* ':' { + | a=invalid_ann_assign_target ':' expression { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION( + a, + "only single target (not %s) can be annotated", + _PyPegen_get_expr_name(a) + )} + | a=star_named_expression ',' star_named_expressions* ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "only single target (not tuple) can be annotated") } - | a=expression ':' expression ['=' annotated_rhs] { + | a=expression ':' expression { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "illegal target for annotation") } - | a=star_expressions '=' (yield_expr | star_expressions) { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - _PyPegen_get_invalid_target(a), - "cannot assign to %s", _PyPegen_get_expr_name(_PyPegen_get_invalid_target(a))) } + | (star_targets '=')* a=star_expressions '=' { + RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } + | (star_targets '=')* a=yield_expr '=' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "assignment to yield expression not possible") } | a=star_expressions augassign (yield_expr | star_expressions) { RAISE_SYNTAX_ERROR_KNOWN_LOCATION( - a, + a, "'%s' is an illegal expression for augmented assignment", _PyPegen_get_expr_name(a) )} - +invalid_ann_assign_target[expr_ty]: + | list + | tuple + | '(' a=invalid_ann_assign_target ')' { a } +invalid_del_stmt: + | 'del' a=star_expressions { + RAISE_SYNTAX_ERROR_INVALID_TARGET(DEL_TARGETS, a) } invalid_block: | NEWLINE !INDENT { RAISE_INDENTATION_ERROR("expected an indented block") } +invalid_primary: + | primary a='{' { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "invalid syntax") } invalid_comprehension: | ('[' | '(' | '{') a=starred_expression for_if_clauses { RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "iterable unpacking cannot be used in comprehension") } +invalid_dict_comprehension: + | '{' a='**' bitwise_or for_if_clauses '}' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "dict unpacking cannot be used in dict comprehension") } invalid_parameters: | param_no_default* (slash_with_default | param_with_default+) param_no_default { RAISE_SYNTAX_ERROR("non-default argument follows default argument") } +invalid_lambda_parameters: + | lambda_param_no_default* (lambda_slash_with_default | lambda_param_with_default+) lambda_param_no_default { + RAISE_SYNTAX_ERROR("non-default argument follows default argument") } invalid_star_etc: | '*' (')' | ',' (')' | '**')) { RAISE_SYNTAX_ERROR("named arguments must follow bare *") } | '*' ',' TYPE_COMMENT { RAISE_SYNTAX_ERROR("bare * has associated type comment") } @@ -667,6 +680,17 @@ invalid_lambda_star_etc: invalid_double_type_comments: | TYPE_COMMENT NEWLINE TYPE_COMMENT NEWLINE INDENT { RAISE_SYNTAX_ERROR("Cannot have two type comments on def") } -invalid_del_target: - | a=star_expression &del_target_end { - RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "cannot delete %s", _PyPegen_get_expr_name(a)) } +invalid_with_item: + | expression 'as' a=expression { + RAISE_SYNTAX_ERROR_INVALID_TARGET(STAR_TARGETS, a) } + +invalid_for_target: + | ASYNC? 'for' a=star_expressions { + RAISE_SYNTAX_ERROR_INVALID_TARGET(FOR_TARGETS, a) } + +invalid_group: + | '(' a=starred_expression ')' { + RAISE_SYNTAX_ERROR_KNOWN_LOCATION(a, "can't use starred expression here") } +invalid_import_from_targets: + | import_from_as_names ',' { + RAISE_SYNTAX_ERROR("trailing comma not allowed without surrounding parentheses") } diff --git a/Include/Python.h b/Include/Python.h index dcd0a57ac1f03f..acee38c41539d6 100644 --- a/Include/Python.h +++ b/Include/Python.h @@ -63,13 +63,22 @@ #include "pyport.h" #include "pymacro.h" -/* A convenient way for code to know if clang's memory sanitizer is enabled. */ +/* A convenient way for code to know if sanitizers are enabled. */ #if defined(__has_feature) # if __has_feature(memory_sanitizer) # if !defined(_Py_MEMORY_SANITIZER) # define _Py_MEMORY_SANITIZER # endif # endif +# if __has_feature(address_sanitizer) +# if !defined(_Py_ADDRESS_SANITIZER) +# define _Py_ADDRESS_SANITIZER +# endif +# endif +#elif defined(__GNUC__) +# if defined(__SANITIZE_ADDRESS__) +# define _Py_ADDRESS_SANITIZER +# endif #endif /* Debug-mode build with pymalloc implies PYMALLOC_DEBUG. diff --git a/Include/ceval.h b/Include/ceval.h index df5253900eea71..0f372e2044a1c8 100644 --- a/Include/ceval.h +++ b/Include/ceval.h @@ -128,8 +128,12 @@ PyAPI_FUNC(void) PyEval_RestoreThread(PyThreadState *); Py_DEPRECATED(3.9) PyAPI_FUNC(int) PyEval_ThreadsInitialized(void); Py_DEPRECATED(3.9) PyAPI_FUNC(void) PyEval_InitThreads(void); +/* PyEval_AcquireLock() and PyEval_ReleaseLock() are part of stable ABI. + * They will be removed from this header file in the future version. + * But they will be remained in ABI until Python 4.0. + */ Py_DEPRECATED(3.2) PyAPI_FUNC(void) PyEval_AcquireLock(void); -/* Py_DEPRECATED(3.2) */ PyAPI_FUNC(void) PyEval_ReleaseLock(void); +Py_DEPRECATED(3.2) PyAPI_FUNC(void) PyEval_ReleaseLock(void); PyAPI_FUNC(void) PyEval_AcquireThread(PyThreadState *tstate); PyAPI_FUNC(void) PyEval_ReleaseThread(PyThreadState *tstate); diff --git a/Include/compile.h b/Include/compile.h index 12417ce805464b..98adee3d191204 100644 --- a/Include/compile.h +++ b/Include/compile.h @@ -9,6 +9,9 @@ extern "C" { /* Public interface */ struct _node; /* Declare the existence of this type */ +#ifndef Py_BUILD_CORE +Py_DEPRECATED(3.9) +#endif PyAPI_FUNC(PyCodeObject *) PyNode_Compile(struct _node *, const char *); /* XXX (ncoghlan): Unprefixed type name in a public API! */ diff --git a/Include/cpython/abstract.h b/Include/cpython/abstract.h index 7bc80833a746ef..0f1304d26af335 100644 --- a/Include/cpython/abstract.h +++ b/Include/cpython/abstract.h @@ -67,7 +67,7 @@ PyVectorcall_Function(PyObject *callable) { PyTypeObject *tp; Py_ssize_t offset; - vectorcallfunc *ptr; + vectorcallfunc ptr; assert(callable != NULL); tp = Py_TYPE(callable); @@ -77,8 +77,8 @@ PyVectorcall_Function(PyObject *callable) assert(PyCallable_Check(callable)); offset = tp->tp_vectorcall_offset; assert(offset > 0); - ptr = (vectorcallfunc *)(((char *)callable) + offset); - return *ptr; + memcpy(&ptr, (char *) callable + offset, sizeof(ptr)); + return ptr; } /* Call the callable object 'callable' with the "vectorcall" calling diff --git a/Include/cpython/fileobject.h b/Include/cpython/fileobject.h index 57eac13c064c2e..3005ce1f00f9d5 100644 --- a/Include/cpython/fileobject.h +++ b/Include/cpython/fileobject.h @@ -8,14 +8,6 @@ extern "C" { PyAPI_FUNC(char *) Py_UniversalNewlineFgets(char *, int, FILE*, PyObject *); -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 -PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; -#endif - -#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 -PyAPI_DATA(int) Py_UTF8Mode; -#endif - /* The std printer acts as a preliminary sys.stderr until the new io infrastructure is in place. */ PyAPI_FUNC(PyObject *) PyFile_NewStdPrinter(int); diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index df93a5539d48bd..0a256d4b5b0b82 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -388,6 +388,7 @@ typedef struct { wchar_t *base_prefix; /* sys.base_prefix */ wchar_t *exec_prefix; /* sys.exec_prefix */ wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ + wchar_t *platlibdir; /* sys.platlibdir */ /* --- Parameter only used by Py_Main() ---------- */ @@ -413,6 +414,14 @@ typedef struct { /* If non-zero, disallow threads, subprocesses, and fork. Default: 0. */ int _isolated_interpreter; + + /* Original command line arguments. If _orig_argv is empty and _argv is + not equal to [''], PyConfig_Read() copies the configuration 'argv' list + into '_orig_argv' list before modifying 'argv' list (if parse_argv + is non-zero). + + _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */ + PyWideStringList _orig_argv; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); @@ -438,6 +447,14 @@ PyAPI_FUNC(PyStatus) PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items); + +/* --- Helper functions --------------------------------------- */ + +/* Get the original command line arguments, before Python modified them. + + See also PyConfig._orig_argv. */ +PyAPI_FUNC(void) Py_GetArgcArgv(int *argc, wchar_t ***argv); + #ifdef __cplusplus } #endif diff --git a/Include/cpython/pyerrors.h b/Include/cpython/pyerrors.h index dd3c2caa0cc043..9c87b53979024d 100644 --- a/Include/cpython/pyerrors.h +++ b/Include/cpython/pyerrors.h @@ -149,7 +149,10 @@ PyAPI_FUNC(PyObject *) PyErr_ProgramTextObject( PyObject *filename, int lineno); -/* Create a UnicodeEncodeError object */ +/* Create a UnicodeEncodeError object. + * + * TODO: This API will be removed in Python 3.11. + */ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject *) PyUnicodeEncodeError_Create( const char *encoding, /* UTF-8 encoded string */ const Py_UNICODE *object, @@ -159,7 +162,10 @@ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject *) PyUnicodeEncodeError_Create( const char *reason /* UTF-8 encoded string */ ); -/* Create a UnicodeTranslateError object */ +/* Create a UnicodeTranslateError object. + * + * TODO: This API will be removed in Python 3.11. + */ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject *) PyUnicodeTranslateError_Create( const Py_UNICODE *object, Py_ssize_t length, diff --git a/Include/cpython/unicodeobject.h b/Include/cpython/unicodeobject.h index 4fd674ffea36ea..17db79cffbc547 100644 --- a/Include/cpython/unicodeobject.h +++ b/Include/cpython/unicodeobject.h @@ -22,7 +22,7 @@ extern "C" { */ #define Py_UNICODE_ISSPACE(ch) \ - ((ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch)) + ((Py_UCS4)(ch) < 128U ? _Py_ascii_whitespace[(ch)] : _PyUnicode_IsWhitespace(ch)) #define Py_UNICODE_ISLOWER(ch) _PyUnicode_IsLowercase(ch) #define Py_UNICODE_ISUPPER(ch) _PyUnicode_IsUppercase(ch) @@ -50,13 +50,18 @@ extern "C" { Py_UNICODE_ISDIGIT(ch) || \ Py_UNICODE_ISNUMERIC(ch)) -#define Py_UNICODE_COPY(target, source, length) \ - memcpy((target), (source), (length)*sizeof(Py_UNICODE)) +Py_DEPRECATED(3.3) static inline void +Py_UNICODE_COPY(Py_UNICODE *target, const Py_UNICODE *source, Py_ssize_t length) { + memcpy(target, source, (size_t)(length) * sizeof(Py_UNICODE)); +} -#define Py_UNICODE_FILL(target, value, length) \ - do {Py_ssize_t i_; Py_UNICODE *t_ = (target); Py_UNICODE v_ = (value);\ - for (i_ = 0; i_ < (length); i_++) t_[i_] = v_;\ - } while (0) +Py_DEPRECATED(3.3) static inline void +Py_UNICODE_FILL(Py_UNICODE *target, Py_UNICODE value, Py_ssize_t length) { + Py_ssize_t i; + for (i = 0; i < length; i++) { + target[i] = value; + } +} /* macros to work with surrogates */ #define Py_UNICODE_IS_SURROGATE(ch) (0xD800 <= (ch) && (ch) <= 0xDFFF) @@ -71,14 +76,6 @@ extern "C" { /* low surrogate = bottom 10 bits added to DC00 */ #define Py_UNICODE_LOW_SURROGATE(ch) (0xDC00 + ((ch) & 0x3FF)) -/* Check if substring matches at given offset. The offset must be - valid, and the substring must not be empty. */ - -#define Py_UNICODE_MATCH(string, offset, substring) \ - ((*((string)->wstr + (offset)) == *((substring)->wstr)) && \ - ((*((string)->wstr + (offset) + (substring)->wstr_length-1) == *((substring)->wstr + (substring)->wstr_length-1))) && \ - !memcmp((string)->wstr + (offset), (substring)->wstr, (substring)->wstr_length*sizeof(Py_UNICODE))) - /* --- Unicode Type ------------------------------------------------------- */ /* ASCII-only strings created through PyUnicode_New use the PyASCIIObject @@ -251,10 +248,6 @@ PyAPI_FUNC(int) _PyUnicode_CheckConsistency( int check_content); /* Fast access macros */ -#define PyUnicode_WSTR_LENGTH(op) \ - (PyUnicode_IS_COMPACT_ASCII(op) ? \ - ((PyASCIIObject*)op)->length : \ - ((PyCompactUnicodeObject*)op)->wstr_length) /* Returns the deprecated Py_UNICODE representation's size in code units (this includes surrogate pairs as 2 units). @@ -449,6 +442,14 @@ enum PyUnicode_Kind { (0xffffU) : \ (0x10ffffU))))) +Py_DEPRECATED(3.3) +static inline Py_ssize_t _PyUnicode_get_wstr_length(PyObject *op) { + return PyUnicode_IS_COMPACT_ASCII(op) ? + ((PyASCIIObject*)op)->length : + ((PyCompactUnicodeObject*)op)->wstr_length; +} +#define PyUnicode_WSTR_LENGTH(op) _PyUnicode_get_wstr_length((PyObject*)op) + /* === Public API ========================================================= */ /* --- Plain Py_UNICODE --------------------------------------------------- */ @@ -547,7 +548,7 @@ PyAPI_FUNC(void) _PyUnicode_FastFill( only allowed if u was set to NULL. The buffer is copied into the new object. */ -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode( +Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject*) PyUnicode_FromUnicode( const Py_UNICODE *u, /* Unicode buffer */ Py_ssize_t size /* size of buffer */ ); @@ -576,13 +577,13 @@ PyAPI_FUNC(Py_UCS4) _PyUnicode_FindMaxChar ( Py_UNICODE buffer. If the wchar_t/Py_UNICODE representation is not yet available, this function will calculate it. */ -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( +Py_DEPRECATED(3.3) PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicode( PyObject *unicode /* Unicode object */ ); /* Similar to PyUnicode_AsUnicode(), but raises a ValueError if the string contains null characters. */ -PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode( +Py_DEPRECATED(3.3) PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode( PyObject *unicode /* Unicode object */ ); @@ -591,7 +592,7 @@ PyAPI_FUNC(const Py_UNICODE *) _PyUnicode_AsUnicode( If the wchar_t/Py_UNICODE representation is not yet available, this function will calculate it. */ -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( +Py_DEPRECATED(3.3) PyAPI_FUNC(Py_UNICODE *) PyUnicode_AsUnicodeAndSize( PyObject *unicode, /* Unicode object */ Py_ssize_t *size /* location where to save the length */ ); @@ -759,13 +760,6 @@ PyAPI_FUNC(const char *) PyUnicode_AsUTF8AndSize( Use of this API is DEPRECATED since no size information can be extracted from the returned data. - - *** This API is for interpreter INTERNAL USE ONLY and will likely - *** be removed or changed for Python 3.1. - - *** If you need to access the Unicode object as UTF-8 bytes string, - *** please use PyUnicode_AsUTF8String() instead. - */ PyAPI_FUNC(const char *) PyUnicode_AsUTF8(PyObject *unicode); @@ -978,7 +972,7 @@ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS( */ -/* Py_DEPRECATED(3.3) */ PyAPI_FUNC(int) PyUnicode_EncodeDecimal( +Py_DEPRECATED(3.3) PyAPI_FUNC(int) PyUnicode_EncodeDecimal( Py_UNICODE *s, /* Unicode buffer */ Py_ssize_t length, /* Number of Py_UNICODE chars to encode */ char *output, /* Output buffer; must have size >= length */ @@ -991,7 +985,7 @@ Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject*) PyUnicode_EncodeMBCS( Returns a new Unicode string on success, NULL on failure. */ -/* Py_DEPRECATED(3.3) */ +Py_DEPRECATED(3.3) PyAPI_FUNC(PyObject*) PyUnicode_TransformDecimalToASCII( Py_UNICODE *s, /* Unicode buffer */ Py_ssize_t length /* Number of Py_UNICODE chars to transform */ diff --git a/Include/fileobject.h b/Include/fileobject.h index 456887ef9d045d..6ec2994aa859b6 100644 --- a/Include/fileobject.h +++ b/Include/fileobject.h @@ -20,8 +20,15 @@ PyAPI_FUNC(int) PyObject_AsFileDescriptor(PyObject *); If non-NULL, this is different than the default encoding for strings */ PyAPI_DATA(const char *) Py_FileSystemDefaultEncoding; +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03060000 +PyAPI_DATA(const char *) Py_FileSystemDefaultEncodeErrors; +#endif PyAPI_DATA(int) Py_HasFileSystemDefaultEncoding; +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03070000 +PyAPI_DATA(int) Py_UTF8Mode; +#endif + /* A routine to check if a file descriptor can be select()-ed. */ #ifdef _MSC_VER /* On Windows, any socket fd can be select()-ed, no matter how high */ diff --git a/Include/internal/pycore_byteswap.h b/Include/internal/pycore_byteswap.h index 5e64704a004c82..2b20fc6c7d3136 100644 --- a/Include/internal/pycore_byteswap.h +++ b/Include/internal/pycore_byteswap.h @@ -15,9 +15,8 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8))) +#if defined(__GNUC__) \ + && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 8)) /* __builtin_bswap16() is available since GCC 4.8, __builtin_bswap32() is available since GCC 4.3, __builtin_bswap64() is available since GCC 4.3. */ @@ -32,7 +31,7 @@ extern "C" { static inline uint16_t _Py_bswap16(uint16_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap16) return __builtin_bswap16(word); #elif defined(_MSC_VER) Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned short)); @@ -47,7 +46,7 @@ _Py_bswap16(uint16_t word) static inline uint32_t _Py_bswap32(uint32_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap32) return __builtin_bswap32(word); #elif defined(_MSC_VER) Py_BUILD_ASSERT(sizeof(word) == sizeof(unsigned long)); @@ -64,7 +63,7 @@ _Py_bswap32(uint32_t word) static inline uint64_t _Py_bswap64(uint64_t word) { -#ifdef _PY_HAVE_BUILTIN_BSWAP +#if defined(_PY_HAVE_BUILTIN_BSWAP) || _Py__has_builtin(__builtin_bswap64) return __builtin_bswap64(word); #elif defined(_MSC_VER) return _byteswap_uint64(word); diff --git a/Include/internal/pycore_ceval.h b/Include/internal/pycore_ceval.h index 368990099089fe..18c8f027af16e6 100644 --- a/Include/internal/pycore_ceval.h +++ b/Include/internal/pycore_ceval.h @@ -50,11 +50,7 @@ extern PyObject *_PyEval_EvalCode( PyObject *kwdefs, PyObject *closure, PyObject *name, PyObject *qualname); -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -extern int _PyEval_ThreadsInitialized(PyInterpreterState *interp); -#else extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime); -#endif extern PyStatus _PyEval_InitGIL(PyThreadState *tstate); extern void _PyEval_FiniGIL(PyThreadState *tstate); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index bbee58617fd05e..8cf137bb4bdf9d 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -48,6 +48,18 @@ PyAPI_FUNC(int) _Py_GetLocaleconvNumeric( PyObject **decimal_point, PyObject **thousands_sep); +#ifdef HAVE_NON_UNICODE_WCHAR_T_REPRESENTATION +extern int _Py_LocaleUsesNonUnicodeWchar(void); + +extern wchar_t* _Py_DecodeNonUnicodeWchar( + const wchar_t* native, + Py_ssize_t size); + +extern int _Py_EncodeNonUnicodeWchar_InPlace( + wchar_t* unicode, + Py_ssize_t size); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 8c6706c95cbd1e..457a005860b202 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -150,7 +150,7 @@ extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); extern PyStatus _PyConfig_InitPathConfig(PyConfig *config); -extern void _PyConfig_Write(const PyConfig *config, +extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index f04ea330d04571..551ad833bb6927 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -46,9 +46,6 @@ struct _ceval_state { /* Request for dropping the GIL */ _Py_atomic_int gil_drop_request; struct _pending_calls pending; -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - struct _gil_runtime_state gil; -#endif }; /* fs_codec.encoding is initialized to NULL. diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index 3290a37051e0f2..2cf1160afc0149 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -51,7 +51,7 @@ PyAPI_FUNC(void) _PyErr_SetObject( PyObject *value); PyAPI_FUNC(void) _PyErr_ChainStackItem( - _PyErr_StackItem *exc_state); + _PyErr_StackItem *exc_info); PyAPI_FUNC(void) _PyErr_Clear(PyThreadState *tstate); diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 77ea3f27454da0..50ab645fc74859 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -68,6 +68,7 @@ extern void _PyFloat_Fini(void); extern void _PySlice_Fini(void); extern void _PyAsyncGen_Fini(void); +extern int _PySignal_Init(int install_signal_handlers); extern void PyOS_FiniInterrupts(void); extern void _PyExc_Fini(void); @@ -82,6 +83,7 @@ extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern void _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); +extern void _PyAST_Fini(void); extern PyStatus _PyGILState_Init(PyThreadState *tstate); extern void _PyGILState_Fini(PyThreadState *tstate); diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index d96ba31207001a..835d6e029c4498 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -49,18 +49,10 @@ _Py_ThreadCanHandlePendingCalls(void) /* Variable and macro for in-line access to current thread and interpreter state */ -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS -PyAPI_FUNC(PyThreadState*) _PyThreadState_GetTSS(void); -#endif - static inline PyThreadState* _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return (PyThreadState*)_Py_atomic_load_relaxed(&runtime->gilstate.tstate_current); -#endif } /* Get the current Python thread state. @@ -75,17 +67,28 @@ _PyRuntimeState_GetThreadState(_PyRuntimeState *runtime) static inline PyThreadState* _PyThreadState_GET(void) { -#ifdef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS - return _PyThreadState_GetTSS(); -#else return _PyRuntimeState_GetThreadState(&_PyRuntime); -#endif } /* Redefine PyThreadState_GET() as an alias to _PyThreadState_GET() */ #undef PyThreadState_GET #define PyThreadState_GET() _PyThreadState_GET() +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalError_TstateNULL(const char *func); + +static inline void +_Py_EnsureFuncTstateNotNULL(const char *func, PyThreadState *tstate) +{ + if (tstate == NULL) { + _Py_FatalError_TstateNULL(func); + } +} + +// Call Py_FatalError() if tstate is NULL +#define _Py_EnsureTstateNotNULL(tstate) \ + _Py_EnsureFuncTstateNotNULL(__func__, tstate) + + /* Get the current interpreter state. The macro is unsafe: it does not check for error and it can return NULL. @@ -96,7 +99,9 @@ _PyThreadState_GET(void) and _PyGILState_GetInterpreterStateUnsafe(). */ static inline PyInterpreterState* _PyInterpreterState_GET(void) { PyThreadState *tstate = _PyThreadState_GET(); - assert(tstate != NULL); +#ifdef Py_DEBUG + _Py_EnsureTstateNotNULL(tstate); +#endif return tstate->interp; } @@ -124,6 +129,9 @@ PyAPI_FUNC(int) _PyState_AddModule( PyObject* module, struct PyModuleDef* def); + +PyAPI_FUNC(int) _PyOS_InterruptOccurred(PyThreadState *tstate); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index ebdc12b23a9ca6..34eb492b9f254f 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -19,9 +19,7 @@ struct _ceval_runtime_state { the main thread of the main interpreter can handle signals: see _Py_ThreadCanHandleSignals(). */ _Py_atomic_int signals_pending; -#ifndef EXPERIMENTAL_ISOLATED_SUBINTERPRETERS struct _gil_runtime_state gil; -#endif }; /* GIL state */ diff --git a/Include/object.h b/Include/object.h index 514d934196f571..9c1a7f479e4490 100644 --- a/Include/object.h +++ b/Include/object.h @@ -618,8 +618,16 @@ times. static inline int -PyType_HasFeature(PyTypeObject *type, unsigned long feature) { - return ((PyType_GetFlags(type) & feature) != 0); +PyType_HasFeature(PyTypeObject *type, unsigned long feature) +{ + unsigned long flags; +#ifdef Py_LIMITED_API + // PyTypeObject is opaque in the limited C API + flags = PyType_GetFlags(type); +#else + flags = type->tp_flags; +#endif + return ((flags & feature) != 0); } #define PyType_FastSubclass(type, flag) PyType_HasFeature(type, flag) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index 3cbd3db76b2d86..999b7fa057e475 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -18,12 +18,12 @@ /*--start constants--*/ #define PY_MAJOR_VERSION 3 #define PY_MINOR_VERSION 9 -#define PY_MICRO_VERSION 0 -#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 6 +#define PY_MICRO_VERSION 6 +#define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_FINAL +#define PY_RELEASE_SERIAL 0 /* Version as a string */ -#define PY_VERSION "3.9.0a6+" +#define PY_VERSION "3.9.6+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Include/pyctype.h b/Include/pyctype.h index 6bce63eeb63b82..729d93275e6c53 100644 --- a/Include/pyctype.h +++ b/Include/pyctype.h @@ -1,6 +1,9 @@ #ifndef Py_LIMITED_API #ifndef PYCTYPE_H #define PYCTYPE_H +#ifdef __cplusplus +extern "C" { +#endif #define PY_CTF_LOWER 0x01 #define PY_CTF_UPPER 0x02 @@ -29,5 +32,8 @@ PyAPI_DATA(const unsigned char) _Py_ctype_toupper[256]; #define Py_TOLOWER(c) (_Py_ctype_tolower[Py_CHARMASK(c)]) #define Py_TOUPPER(c) (_Py_ctype_toupper[Py_CHARMASK(c)]) +#ifdef __cplusplus +} +#endif #endif /* !PYCTYPE_H */ #endif /* !Py_LIMITED_API */ diff --git a/Include/pyerrors.h b/Include/pyerrors.h index 399bb7c3a6fac0..979a26ba68a033 100644 --- a/Include/pyerrors.h +++ b/Include/pyerrors.h @@ -4,6 +4,8 @@ extern "C" { #endif +#include // va_list + /* Error handling definitions */ PyAPI_FUNC(void) PyErr_SetNone(PyObject *); @@ -307,21 +309,6 @@ PyAPI_FUNC(int) PyUnicodeTranslateError_SetReason( const char *reason /* UTF-8 encoded string */ ); -/* These APIs aren't really part of the error implementation, but - often needed to format error messages; the native C lib APIs are - not available on all platforms, which is why we provide emulations - for those platforms in Python/mysnprintf.c, - WARNING: The return value of snprintf varies across platforms; do - not rely on any particular behavior; eventually the C99 defn may - be reliable. -*/ -#if defined(MS_WIN32) && !defined(HAVE_SNPRINTF) -# define HAVE_SNPRINTF -# define snprintf _snprintf -# define vsnprintf _vsnprintf -#endif - -#include PyAPI_FUNC(int) PyOS_snprintf(char *str, size_t size, const char *format, ...) Py_GCC_ATTRIBUTE((format(printf, 3, 4))); PyAPI_FUNC(int) PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index c5368b3c5edaa0..783fcb455eb528 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -32,6 +32,8 @@ PyAPI_FUNC(void) _Py_NO_RETURN Py_Exit(int); /* Bootstrap __main__ (defined in Modules/main.c) */ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); +PyAPI_FUNC(int) Py_FrozenMain(int argc, char **argv); + PyAPI_FUNC(int) Py_BytesMain(int argc, char **argv); /* In pathconfig.c */ diff --git a/Include/pymacro.h b/Include/pymacro.h index 856cae774d61c7..202b936d964f00 100644 --- a/Include/pymacro.h +++ b/Include/pymacro.h @@ -118,7 +118,9 @@ "We've reached an unreachable state. Anything is possible.\n" \ "The limits were in our heads all along. Follow your dreams.\n" \ "https://xkcd.com/2200") -#elif defined(__GNUC__) || defined(__clang__) || defined(__INTEL_COMPILER) +#elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5)) +# define Py_UNREACHABLE() __builtin_unreachable() +#elif defined(__clang__) || defined(__INTEL_COMPILER) # define Py_UNREACHABLE() __builtin_unreachable() #elif defined(_MSC_VER) # define Py_UNREACHABLE() __assume(0) diff --git a/Include/pyport.h b/Include/pyport.h index 63d3b81de5d232..4bd4eb4751b952 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -513,6 +513,26 @@ extern "C" { #define Py_DEPRECATED(VERSION_UNUSED) #endif +#if defined(__clang__) +#define _Py_COMP_DIAG_PUSH _Pragma("clang diagnostic push") +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \ + _Pragma("clang diagnostic ignored \"-Wdeprecated-declarations\"") +#define _Py_COMP_DIAG_POP _Pragma("clang diagnostic pop") +#elif defined(__GNUC__) \ + && ((__GNUC__ >= 5) || (__GNUC__ == 4) && (__GNUC_MINOR__ >= 6)) +#define _Py_COMP_DIAG_PUSH _Pragma("GCC diagnostic push") +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS \ + _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") +#define _Py_COMP_DIAG_POP _Pragma("GCC diagnostic pop") +#elif defined(_MSC_VER) +#define _Py_COMP_DIAG_PUSH __pragma(warning(push)) +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS __pragma(warning(disable: 4996)) +#define _Py_COMP_DIAG_POP __pragma(warning(pop)) +#else +#define _Py_COMP_DIAG_PUSH +#define _Py_COMP_DIAG_IGNORE_DEPR_DECLS +#define _Py_COMP_DIAG_POP +#endif /* _Py_HOT_FUNCTION * The hot attribute on a function is used to inform the compiler that the @@ -829,10 +849,10 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; #endif /* Mark a function which cannot return. Example: + PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); - PyAPI_FUNC(void) _Py_NO_RETURN PyThread_exit_thread(void); */ + XLC support is intentionally omitted due to bpo-40244 */ #if defined(__clang__) || \ - defined(__xlc__) || \ (defined(__GNUC__) && \ ((__GNUC__ >= 3) || \ (__GNUC__ == 2) && (__GNUC_MINOR__ >= 5))) @@ -843,4 +863,16 @@ extern _invalid_parameter_handler _Py_silent_invalid_parameter_handler; # define _Py_NO_RETURN #endif + +// Preprocessor check for a builtin preprocessor function. Always return 0 +// if __has_builtin() macro is not defined. +// +// __has_builtin() is available on clang and GCC 10. +#ifdef __has_builtin +# define _Py__has_builtin(x) __has_builtin(x) +#else +# define _Py__has_builtin(x) 0 +#endif + + #endif /* Py_PYPORT_H */ diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 46091e09216330..57529072432ea7 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -72,16 +72,23 @@ PyAPI_FUNC(struct _mod *) PyParser_ASTFromFileObject( #define PyParser_SimpleParseFile(FP, S, B) \ PyParser_SimpleParseFileFlags(FP, S, B, 0) #endif -PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, - int); + +#ifndef Py_BUILD_CORE +Py_DEPRECATED(3.9) +#endif +PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlags(const char *, int, int); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03030000 +#ifndef Py_BUILD_CORE +Py_DEPRECATED(3.9) +#endif PyAPI_FUNC(struct _node *) PyParser_SimpleParseStringFlagsFilename(const char *, const char *, int, int); #endif -PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, - int, int); - +#ifndef Py_BUILD_CORE +Py_DEPRECATED(3.9) +#endif +PyAPI_FUNC(struct _node *) PyParser_SimpleParseFileFlags(FILE *, const char *, int, int); #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) PyRun_StringFlags(const char *, int, PyObject *, PyObject *, PyCompilerFlags *); diff --git a/Include/typeslots.h b/Include/typeslots.h index 0ce6a377dcfbd6..64f6fff5144493 100644 --- a/Include/typeslots.h +++ b/Include/typeslots.h @@ -1,7 +1,12 @@ /* Do not renumber the file; these numbers are part of the stable ABI. */ +#if defined(Py_LIMITED_API) /* Disabled, see #10181 */ #undef Py_bf_getbuffer #undef Py_bf_releasebuffer +#else +#define Py_bf_getbuffer 1 +#define Py_bf_releasebuffer 2 +#endif #define Py_mp_ass_subscript 3 #define Py_mp_length 4 #define Py_mp_subscript 5 diff --git a/LICENSE b/LICENSE index 66a3ac80d729a3..473861da1be7c5 100644 --- a/LICENSE +++ b/LICENSE @@ -59,6 +59,17 @@ direction to make these releases possible. B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON =============================================================== +Python software and documentation are licensed under the +Python Software Foundation License Version 2. + +Starting with Python 3.8.6, examples, recipes, and other code in +the documentation are dual licensed under the PSF License Version 2 +and the Zero-Clause BSD license. + +Some software incorporated into Python is under different licenses. +The licenses are listed with code falling under that license. + + PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 -------------------------------------------- @@ -73,7 +84,7 @@ analyze, test, perform and/or display publicly, prepare derivative works, distribute, and otherwise use Python alone or in any derivative version, provided, however, that PSF's License Agreement and PSF's notice of copyright, i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020 Python Software Foundation; +2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021 Python Software Foundation; All Rights Reserved" are retained in Python alone or in any derivative version prepared by Licensee. @@ -252,3 +263,17 @@ FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION +---------------------------------------------------------------------- + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH +REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY +AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, +INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR +OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. diff --git a/Lib/__future__.py b/Lib/__future__.py index d7cb8ac5f49745..0e7b5552343356 100644 --- a/Lib/__future__.py +++ b/Lib/__future__.py @@ -66,18 +66,20 @@ # code.h and used by compile.h, so that an editor search will find them here. # However, they're not exported in __all__, because they don't really belong to # this module. -CO_NESTED = 0x0010 # nested_scopes -CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) -CO_FUTURE_DIVISION = 0x20000 # division -CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default -CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement -CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function -CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals +CO_NESTED = 0x0010 # nested_scopes +CO_GENERATOR_ALLOWED = 0 # generators (obsolete, was 0x1000) +CO_FUTURE_DIVISION = 0x20000 # division +CO_FUTURE_ABSOLUTE_IMPORT = 0x40000 # perform absolute imports by default +CO_FUTURE_WITH_STATEMENT = 0x80000 # with statement +CO_FUTURE_PRINT_FUNCTION = 0x100000 # print function +CO_FUTURE_UNICODE_LITERALS = 0x200000 # unicode string literals CO_FUTURE_BARRY_AS_BDFL = 0x400000 -CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators -CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime +CO_FUTURE_GENERATOR_STOP = 0x800000 # StopIteration becomes RuntimeError in generators +CO_FUTURE_ANNOTATIONS = 0x1000000 # annotations become strings at runtime + class _Feature: + def __init__(self, optionalRelease, mandatoryRelease, compiler_flag): self.optional = optionalRelease self.mandatory = mandatoryRelease @@ -88,7 +90,6 @@ def getOptionalRelease(self): This is a 5-tuple, of the same form as sys.version_info. """ - return self.optional def getMandatoryRelease(self): @@ -97,7 +98,6 @@ def getMandatoryRelease(self): This is a 5-tuple, of the same form as sys.version_info, or, if the feature was dropped, is None. """ - return self.mandatory def __repr__(self): @@ -105,6 +105,7 @@ def __repr__(self): self.mandatory, self.compiler_flag)) + nested_scopes = _Feature((2, 1, 0, "beta", 1), (2, 2, 0, "alpha", 0), CO_NESTED) @@ -142,5 +143,5 @@ def __repr__(self): CO_FUTURE_GENERATOR_STOP) annotations = _Feature((3, 7, 0, "beta", 1), - (4, 0, 0, "alpha", 0), + (3, 10, 0, "alpha", 0), CO_FUTURE_ANNOTATIONS) diff --git a/Lib/_aix_support.py b/Lib/_aix_support.py index 45504934063df8..d27a1e8735de30 100644 --- a/Lib/_aix_support.py +++ b/Lib/_aix_support.py @@ -15,8 +15,9 @@ def _aix_tag(vrtl, bd): # type: (List[int], int) -> str # Infer the ABI bitwidth from maxsize (assuming 64 bit as the default) _sz = 32 if sys.maxsize == (2**31-1) else 64 + _bd = bd if bd != 0 else 9988 # vrtl[version, release, technology_level] - return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], bd, _sz) + return "aix-{:1x}{:1d}{:02d}-{:04d}-{}".format(vrtl[0], vrtl[1], vrtl[2], _bd, _sz) # extract version, release and technology level from a VRMF string @@ -26,19 +27,20 @@ def _aix_vrtl(vrmf): return [int(v[-1]), int(r), int(tl)] -def _aix_bosmp64(): +def _aix_bos_rte(): # type: () -> Tuple[str, int] """ Return a Tuple[str, int] e.g., ['7.1.4.34', 1806] - The fileset bos.mp64 is the AIX kernel. It's VRMF and builddate - reflect the current ABI levels of the runtime environment. + The fileset bos.rte represents the current AIX run-time level. It's VRMF and + builddate reflect the current ABI levels of the runtime environment. + If no builddate is found give a value that will satisfy pep425 related queries """ - # We expect all AIX systems to have lslpp installed in this location - out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.mp64"]) + # All AIX systems to have lslpp installed in this location + out = subprocess.check_output(["/usr/bin/lslpp", "-Lqc", "bos.rte"]) out = out.decode("utf-8") out = out.strip().split(":") # type: ignore - # Use str() and int() to help mypy see types - return (str(out[2]), int(out[-1])) + _bd = int(out[-1]) if out[-1] != '' else 9988 + return (str(out[2]), _bd) def aix_platform(): @@ -47,11 +49,11 @@ def aix_platform(): AIX filesets are identified by four decimal values: V.R.M.F. V (version) and R (release) can be retreived using ``uname`` Since 2007, starting with AIX 5.3 TL7, the M value has been - included with the fileset bos.mp64 and represents the Technology + included with the fileset bos.rte and represents the Technology Level (TL) of AIX. The F (Fix) value also increases, but is not relevant for comparing releases and binary compatibility. For binary compatibility the so-called builddate is needed. - Again, the builddate of an AIX release is associated with bos.mp64. + Again, the builddate of an AIX release is associated with bos.rte. AIX ABI compatibility is described as guaranteed at: https://www.ibm.com/\ support/knowledgecenter/en/ssw_aix_72/install/binary_compatability.html @@ -60,7 +62,7 @@ def aix_platform(): e.g., "aix-6107-1415-32" for AIX 6.1 TL7 bd 1415, 32-bit and, "aix-6107-1415-64" for AIX 6.1 TL7 bd 1415, 64-bit """ - vrmf, bd = _aix_bosmp64() + vrmf, bd = _aix_bos_rte() return _aix_tag(_aix_vrtl(vrmf), bd) @@ -79,7 +81,7 @@ def aix_buildtag(): Return the platform_tag of the system Python was built on. """ # AIX_BUILDDATE is defined by configure with: - # lslpp -Lcq bos.mp64 | awk -F: '{ print $NF }' + # lslpp -Lcq bos.rte | awk -F: '{ print $NF }' build_date = sysconfig.get_config_var("AIX_BUILDDATE") try: build_date = int(build_date) diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index 36cd9930003475..b6ecf8eac6368b 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -10,6 +10,10 @@ import sys GenericAlias = type(list[int]) +EllipsisType = type(...) +def _f(): pass +FunctionType = type(_f) +del _f __all__ = ["Awaitable", "Coroutine", "AsyncIterable", "AsyncIterator", "AsyncGenerator", @@ -409,6 +413,86 @@ def __subclasshook__(cls, C): return NotImplemented +class _CallableGenericAlias(GenericAlias): + """ Represent `Callable[argtypes, resulttype]`. + + This sets ``__args__`` to a tuple containing the flattened``argtypes`` + followed by ``resulttype``. + + Example: ``Callable[[int, str], float]`` sets ``__args__`` to + ``(int, str, float)``. + """ + + __slots__ = () + + def __new__(cls, origin, args): + try: + return cls.__create_ga(origin, args) + except TypeError as exc: + import warnings + warnings.warn(f'{str(exc)} ' + f'(This will raise a TypeError in Python 3.10.)', + DeprecationWarning) + return GenericAlias(origin, args) + + @classmethod + def __create_ga(cls, origin, args): + if not isinstance(args, tuple) or len(args) != 2: + raise TypeError( + "Callable must be used as Callable[[arg, ...], result].") + t_args, t_result = args + if isinstance(t_args, (list, tuple)): + ga_args = tuple(t_args) + (t_result,) + # This relaxes what t_args can be on purpose to allow things like + # PEP 612 ParamSpec. Responsibility for whether a user is using + # Callable[...] properly is deferred to static type checkers. + else: + ga_args = args + return super().__new__(cls, origin, ga_args) + + def __repr__(self): + if len(self.__args__) == 2 and self.__args__[0] is Ellipsis: + return super().__repr__() + return (f'collections.abc.Callable' + f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], ' + f'{_type_repr(self.__args__[-1])}]') + + def __reduce__(self): + args = self.__args__ + if not (len(args) == 2 and args[0] is Ellipsis): + args = list(args[:-1]), args[-1] + return _CallableGenericAlias, (Callable, args) + + def __getitem__(self, item): + # Called during TypeVar substitution, returns the custom subclass + # rather than the default types.GenericAlias object. + ga = super().__getitem__(item) + args = ga.__args__ + t_result = args[-1] + t_args = args[:-1] + args = (t_args, t_result) + return _CallableGenericAlias(Callable, args) + + +def _type_repr(obj): + """Return the repr() of an object, special-casing types (internal helper). + + Copied from :mod:`typing` since collections.abc + shouldn't depend on that module. + """ + if isinstance(obj, GenericAlias): + return repr(obj) + if isinstance(obj, type): + if obj.__module__ == 'builtins': + return obj.__qualname__ + return f'{obj.__module__}.{obj.__qualname__}' + if obj is Ellipsis: + return '...' + if isinstance(obj, FunctionType): + return obj.__name__ + return repr(obj) + + class Callable(metaclass=ABCMeta): __slots__ = () @@ -423,7 +507,7 @@ def __subclasshook__(cls, C): return _check_methods(C, "__call__") return NotImplemented - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(_CallableGenericAlias) ### SETS ### diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed5bd..37975fe8a3eefa 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -52,7 +52,7 @@ def _find_executable(executable, path=None): return executable -def _read_output(commandstring): +def _read_output(commandstring, capture_stderr=False): """Output from successful command execution or None""" # Similar to os.popen(commandstring, "r").read(), # but without actually using os.popen because that @@ -67,7 +67,10 @@ def _read_output(commandstring): os.getpid(),), "w+b") with contextlib.closing(fp) as fp: - cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) + if capture_stderr: + cmd = "%s >'%s' 2>&1" % (commandstring, fp.name) + else: + cmd = "%s 2>/dev/null >'%s'" % (commandstring, fp.name) return fp.read().decode('utf-8').strip() if not os.system(cmd) else None @@ -110,6 +113,26 @@ def _get_system_version(): return _SYSTEM_VERSION +_SYSTEM_VERSION_TUPLE = None +def _get_system_version_tuple(): + """ + Return the macOS system version as a tuple + + The return value is safe to use to compare + two version numbers. + """ + global _SYSTEM_VERSION_TUPLE + if _SYSTEM_VERSION_TUPLE is None: + osx_version = _get_system_version() + if osx_version: + try: + _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + _SYSTEM_VERSION_TUPLE = () + + return _SYSTEM_VERSION_TUPLE + + def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. @@ -125,6 +148,33 @@ def _save_modified_value(_config_vars, cv, newvalue): _config_vars[_INITPRE + cv] = oldvalue _config_vars[cv] = newvalue + +_cache_default_sysroot = None +def _default_sysroot(cc): + """ Returns the root of the default SDK for this system, or '/' """ + global _cache_default_sysroot + + if _cache_default_sysroot is not None: + return _cache_default_sysroot + + contents = _read_output('%s -c -E -v - "): + in_incdirs = True + elif line.startswith("End of search list"): + in_incdirs = False + elif in_incdirs: + line = line.strip() + if line == '/usr/include': + _cache_default_sysroot = '/' + elif line.endswith(".sdk/usr/include"): + _cache_default_sysroot = line[:-12] + if _cache_default_sysroot is None: + _cache_default_sysroot = '/' + + return _cache_default_sysroot + def _supports_universal_builds(): """Returns True if universal builds are supported on this system""" # As an approximation, we assume that if we are running on 10.4 or above, @@ -132,14 +182,18 @@ def _supports_universal_builds(): # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return bool(osx_version >= (10, 4)) if osx_version else False +def _supports_arm64_builds(): + """Returns True if arm64 builds are supported on this system""" + # There are two sets of systems supporting macOS/arm64 builds: + # 1. macOS 11 and later, unconditionally + # 2. macOS 10.15 with Xcode 12.2 or later + # For now the second category is ignored. + osx_version = _get_system_version_tuple() + return osx_version >= (11, 0) if osx_version else False + def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" @@ -331,6 +385,12 @@ def compiler_fixup(compiler_so, cc_args): except ValueError: break + elif not _supports_arm64_builds(): + # Look for "-arch arm64" and drop that + for idx in reversed(range(len(compiler_so))): + if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": + del compiler_so[idx:idx+2] + if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig @@ -481,6 +541,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 5df37f5f4b89d5..b97dfcce1e8e4d 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -201,7 +201,7 @@ def __init__(self, locale_time=None): #XXX: Does 'Y' need to worry about having less or more than # 4 digits? 'Y': r"(?P\d\d\d\d)", - 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|Z)", + 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), diff --git a/Lib/abc.py b/Lib/abc.py index 431b64040a66e8..9de128e2362195 100644 --- a/Lib/abc.py +++ b/Lib/abc.py @@ -28,7 +28,14 @@ def my_abstract_method(self, ...): class abstractclassmethod(classmethod): """A decorator indicating abstract classmethods. - Deprecated, use 'classmethod' with 'abstractmethod' instead. + Deprecated, use 'classmethod' with 'abstractmethod' instead: + + class C(ABC): + @classmethod + @abstractmethod + def my_abstract_classmethod(cls, ...): + ... + """ __isabstractmethod__ = True @@ -41,7 +48,14 @@ def __init__(self, callable): class abstractstaticmethod(staticmethod): """A decorator indicating abstract staticmethods. - Deprecated, use 'staticmethod' with 'abstractmethod' instead. + Deprecated, use 'staticmethod' with 'abstractmethod' instead: + + class C(ABC): + @staticmethod + @abstractmethod + def my_abstract_staticmethod(...): + ... + """ __isabstractmethod__ = True @@ -54,7 +68,14 @@ def __init__(self, callable): class abstractproperty(property): """A decorator indicating abstract properties. - Deprecated, use 'property' with 'abstractmethod' instead. + Deprecated, use 'property' with 'abstractmethod' instead: + + class C(ABC): + @property + @abstractmethod + def my_abstract_property(self): + ... + """ __isabstractmethod__ = True diff --git a/Lib/argparse.py b/Lib/argparse.py index 2677ef63e9e541..2fb1da59f942cf 100644 --- a/Lib/argparse.py +++ b/Lib/argparse.py @@ -857,7 +857,6 @@ class BooleanOptionalAction(Action): def __init__(self, option_strings, dest, - const=None, default=None, type=None, choices=None, diff --git a/Lib/ast.py b/Lib/ast.py index 52e51b48587747..396eea18303ceb 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -180,7 +180,11 @@ def copy_location(new_node, old_node): for attr in 'lineno', 'col_offset', 'end_lineno', 'end_col_offset': if attr in old_node._attributes and attr in new_node._attributes: value = getattr(old_node, attr, None) - if value is not None: + # end_lineno and end_col_offset are optional attributes, and they + # should be copied whether the value is None or not. + if value is not None or ( + hasattr(old_node, attr) and attr.startswith("end_") + ): setattr(new_node, attr, value) return new_node @@ -229,8 +233,11 @@ def increment_lineno(node, n=1): for child in walk(node): if 'lineno' in child._attributes: child.lineno = getattr(child, 'lineno', 0) + n - if 'end_lineno' in child._attributes: - child.end_lineno = getattr(child, 'end_lineno', 0) + n + if ( + "end_lineno" in child._attributes + and (end_lineno := getattr(child, "end_lineno", 0)) is not None + ): + child.end_lineno = end_lineno + n return node @@ -490,18 +497,20 @@ def generic_visit(self, node): return node -# The following code is for backward compatibility. -# It will be removed in future. +# If the ast module is loaded more than once, only add deprecated methods once +if not hasattr(Constant, 'n'): + # The following code is for backward compatibility. + # It will be removed in future. -def _getter(self): - """Deprecated. Use value instead.""" - return self.value + def _getter(self): + """Deprecated. Use value instead.""" + return self.value -def _setter(self, value): - self.value = value + def _setter(self, value): + self.value = value -Constant.n = property(_getter, _setter) -Constant.s = property(_getter, _setter) + Constant.n = property(_getter, _setter) + Constant.s = property(_getter, _setter) class _ABC(type): @@ -524,6 +533,13 @@ def __instancecheck__(cls, inst): return type.__instancecheck__(cls, inst) def _new(cls, *args, **kwargs): + for key in kwargs: + if key not in cls._fields: + # arbitrary keyword arguments are accepted + continue + pos = cls._fields.index(key) + if pos < len(args): + raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}") if cls in _const_types: return Constant(*args, **kwargs) return Constant.__new__(cls, *args, **kwargs) @@ -586,14 +602,19 @@ class ExtSlice(slice): def __new__(cls, dims=(), **kwargs): return Tuple(list(dims), Load(), **kwargs) -def _dims_getter(self): - """Deprecated. Use elts instead.""" - return self.elts +# If the ast module is loaded more than once, only add deprecated methods once +if not hasattr(Tuple, 'dims'): + # The following code is for backward compatibility. + # It will be removed in future. -def _dims_setter(self, value): - self.elts = value + def _dims_getter(self): + """Deprecated. Use elts instead.""" + return self.elts -Tuple.dims = property(_dims_getter, _dims_setter) + def _dims_setter(self, value): + self.elts = value + + Tuple.dims = property(_dims_getter, _dims_setter) class Suite(mod): """Deprecated AST node class. Unused in Python 3.""" @@ -641,17 +662,23 @@ def next(self): except ValueError: return self + +_SINGLE_QUOTES = ("'", '"') +_MULTI_QUOTES = ('"""', "'''") +_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES) + class _Unparser(NodeVisitor): """Methods in this class recursively traverse an AST and output source code for the abstract syntax; original formatting is disregarded.""" - def __init__(self): + def __init__(self, *, _avoid_backslashes=False): self._source = [] self._buffer = [] self._precedences = {} self._type_ignores = {} self._indent = 0 + self._avoid_backslashes = _avoid_backslashes def interleave(self, inter, f, seq): """Call f on each item in seq, calling inter() in between.""" @@ -1046,15 +1073,85 @@ def visit_AsyncWith(self, node): with self.block(extra=self.get_type_comment(node)): self.traverse(node.body) + def _str_literal_helper( + self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False + ): + """Helper for writing string literals, minimizing escapes. + Returns the tuple (string literal to write, possible quote types). + """ + def escape_char(c): + # \n and \t are non-printable, but we only escape them if + # escape_special_whitespace is True + if not escape_special_whitespace and c in "\n\t": + return c + # Always escape backslashes and other non-printable characters + if c == "\\" or not c.isprintable(): + return c.encode("unicode_escape").decode("ascii") + return c + + escaped_string = "".join(map(escape_char, string)) + possible_quotes = quote_types + if "\n" in escaped_string: + possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES] + possible_quotes = [q for q in possible_quotes if q not in escaped_string] + if not possible_quotes: + # If there aren't any possible_quotes, fallback to using repr + # on the original string. Try to use a quote from quote_types, + # e.g., so that we use triple quotes for docstrings. + string = repr(string) + quote = next((q for q in quote_types if string[0] in q), string[0]) + return string[1:-1], [quote] + if escaped_string: + # Sort so that we prefer '''"''' over """\"""" + possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1]) + # If we're using triple quotes and we'd need to escape a final + # quote, escape it + if possible_quotes[0][0] == escaped_string[-1]: + assert len(possible_quotes[0]) == 3 + escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1] + return escaped_string, possible_quotes + + def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES): + """Write string literal value with a best effort attempt to avoid backslashes.""" + string, quote_types = self._str_literal_helper(string, quote_types=quote_types) + quote_type = quote_types[0] + self.write(f"{quote_type}{string}{quote_type}") + def visit_JoinedStr(self, node): self.write("f") - self._fstring_JoinedStr(node, self.buffer_writer) - self.write(repr(self.buffer)) + if self._avoid_backslashes: + self._fstring_JoinedStr(node, self.buffer_writer) + self._write_str_avoiding_backslashes(self.buffer) + return + + # If we don't need to avoid backslashes globally (i.e., we only need + # to avoid them inside FormattedValues), it's cosmetically preferred + # to use escaped whitespace. That is, it's preferred to use backslashes + # for cases like: f"{x}\n". To accomplish this, we keep track of what + # in our buffer corresponds to FormattedValues and what corresponds to + # Constant parts of the f-string, and allow escapes accordingly. + buffer = [] + for value in node.values: + meth = getattr(self, "_fstring_" + type(value).__name__) + meth(value, self.buffer_writer) + buffer.append((self.buffer, isinstance(value, Constant))) + new_buffer = [] + quote_types = _ALL_QUOTES + for value, is_constant in buffer: + # Repeatedly narrow down the list of possible quote_types + value, quote_types = self._str_literal_helper( + value, quote_types=quote_types, + escape_special_whitespace=is_constant + ) + new_buffer.append(value) + value = "".join(new_buffer) + quote_type = quote_types[0] + self.write(f"{quote_type}{value}{quote_type}") def visit_FormattedValue(self, node): self.write("f") self._fstring_FormattedValue(node, self.buffer_writer) - self.write(repr(self.buffer)) + self._write_str_avoiding_backslashes(self.buffer) def _fstring_JoinedStr(self, node, write): for value in node.values: @@ -1069,11 +1166,13 @@ def _fstring_Constant(self, node, write): def _fstring_FormattedValue(self, node, write): write("{") - unparser = type(self)() + unparser = type(self)(_avoid_backslashes=True) unparser.set_precedence(_Precedence.TEST.next(), node.value) expr = unparser.visit(node.value) if expr.startswith("{"): write(" ") # Separate pair of opening brackets as "{ {" + if "\\" in expr: + raise ValueError("Unable to avoid backslash in f-string expression part") write(expr) if node.conversion != -1: conversion = chr(node.conversion) @@ -1090,33 +1189,22 @@ def visit_Name(self, node): self.write(node.id) def _write_docstring(self, node): - def esc_char(c): - if c in ("\n", "\t"): - # In the AST form, we don't know the author's intentation - # about how this should be displayed. We'll only escape - # \n and \t, because they are more likely to be unescaped - # in the source - return c - return c.encode('unicode_escape').decode('ascii') - self.fill() if node.kind == "u": self.write("u") - - value = node.value - if value: - # Preserve quotes in the docstring by escaping them - value = "".join(map(esc_char, value)) - if value[-1] == '"': - value = value.replace('"', '\\"', -1) - value = value.replace('"""', '""\\"') - - self.write(f'"""{value}"""') + self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES) def _write_constant(self, value): if isinstance(value, (float, complex)): - # Substitute overflowing decimal literal for AST infinities. - self.write(repr(value).replace("inf", _INFSTR)) + # Substitute overflowing decimal literal for AST infinities, + # and inf - inf for NaNs. + self.write( + repr(value) + .replace("inf", _INFSTR) + .replace("nan", f"({_INFSTR}-{_INFSTR})") + ) + elif self._avoid_backslashes and isinstance(value, str): + self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) @@ -1187,10 +1275,13 @@ def visit_IfExp(self, node): self.traverse(node.orelse) def visit_Set(self, node): - if not node.elts: - raise ValueError("Set node should have at least one item") - with self.delimit("{", "}"): - self.interleave(lambda: self.write(", "), self.traverse, node.elts) + if node.elts: + with self.delimit("{", "}"): + self.interleave(lambda: self.write(", "), self.traverse, node.elts) + else: + # `{}` would be interpreted as a dictionary literal, and + # `set` might be shadowed. Thus: + self.write('{*()}') def visit_Dict(self, node): def write_key_value_pair(k, v): @@ -1357,9 +1448,9 @@ def visit_Call(self, node): def visit_Subscript(self, node): def is_simple_tuple(slice_value): - # when unparsing a non-empty tuple, the parantheses can be safely + # when unparsing a non-empty tuple, the parentheses can be safely # omitted if there aren't any elements that explicitly requires - # parantheses (such as starred expressions). + # parentheses (such as starred expressions). return ( isinstance(slice_value, Tuple) and slice_value.elts diff --git a/Lib/asyncio/__init__.py b/Lib/asyncio/__init__.py index 28c2e2c429f34a..eb84bfb189ccf3 100644 --- a/Lib/asyncio/__init__.py +++ b/Lib/asyncio/__init__.py @@ -17,6 +17,7 @@ from .streams import * from .subprocess import * from .tasks import * +from .threads import * from .transports import * # Exposed for _asynciomodule.c to implement now deprecated @@ -35,6 +36,7 @@ streams.__all__ + subprocess.__all__ + tasks.__all__ + + threads.__all__ + transports.__all__) if sys.platform == 'win32': # pragma: no cover diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 22f298069c505e..2c01ac98e10fca 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -1,6 +1,7 @@ __all__ = () import reprlib +from _thread import get_ident from . import format_helpers @@ -41,6 +42,16 @@ def format_cb(callback): return f'cb=[{cb}]' +# bpo-42183: _repr_running is needed for repr protection +# when a Future or Task result contains itself directly or indirectly. +# The logic is borrowed from @reprlib.recursive_repr decorator. +# Unfortunately, the direct decorator usage is impossible because of +# AttributeError: '_asyncio.Task' object has no attribute '__module__' error. +# +# After fixing this thing we can return to the decorator based approach. +_repr_running = set() + + def _future_repr_info(future): # (Future) -> str """helper function for Future.__repr__""" @@ -49,9 +60,17 @@ def _future_repr_info(future): if future._exception is not None: info.append(f'exception={future._exception!r}') else: - # use reprlib to limit the length of the output, especially - # for very long strings - result = reprlib.repr(future._result) + key = id(future), get_ident() + if key in _repr_running: + result = '...' + else: + _repr_running.add(key) + try: + # use reprlib to limit the length of the output, especially + # for very long strings + result = reprlib.repr(future._result) + finally: + _repr_running.discard(key) info.append(f'result={result}') if future._callbacks: info.append(_format_callbacks(future._callbacks)) diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index 70017cb86a0596..0dce87b8ecc586 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -283,7 +283,7 @@ def create_task(self, coro, *, name=None): def call_soon_threadsafe(self, callback, *args): raise NotImplementedError - async def run_in_executor(self, executor, func, *args): + def run_in_executor(self, executor, func, *args): raise NotImplementedError def set_default_executor(self, executor): diff --git a/Lib/asyncio/exceptions.py b/Lib/asyncio/exceptions.py index e03602ef576234..f07e4486577381 100644 --- a/Lib/asyncio/exceptions.py +++ b/Lib/asyncio/exceptions.py @@ -34,8 +34,9 @@ class IncompleteReadError(EOFError): - expected: total number of expected bytes (or None if unknown) """ def __init__(self, partial, expected): + r_expected = 'undefined' if expected is None else repr(expected) super().__init__(f'{len(partial)} bytes read on a total of ' - f'{expected!r} expected bytes') + f'{r_expected} expected bytes') self.partial = partial self.expected = expected diff --git a/Lib/asyncio/proactor_events.py b/Lib/asyncio/proactor_events.py index 8338449aaa0a3e..b4cd414b82d727 100644 --- a/Lib/asyncio/proactor_events.py +++ b/Lib/asyncio/proactor_events.py @@ -766,6 +766,14 @@ def _loop_self_reading(self, f=None): try: if f is not None: f.result() # may raise + if self._self_reading_future is not f: + # When we scheduled this Future, we assigned it to + # _self_reading_future. If it's not there now, something has + # tried to cancel the loop while this callback was still in the + # queue (see windows_events.ProactorEventLoop.run_forever). In + # that case stop here instead of continuing to schedule a new + # iteration. + return f = self._proactor.recv(self._ssock, 4096) except exceptions.CancelledError: # _close_self_pipe() has been called, stop waiting for data @@ -783,8 +791,17 @@ def _loop_self_reading(self, f=None): f.add_done_callback(self._loop_self_reading) def _write_to_self(self): + # This may be called from a different thread, possibly after + # _close_self_pipe() has been called or even while it is + # running. Guard for self._csock being None or closed. When + # a socket is closed, send() raises OSError (with errno set to + # EBADF, but let's not rely on the exact error code). + csock = self._csock + if csock is None: + return + try: - self._csock.send(b'\0') + csock.send(b'\0') except OSError: if self._debug: logger.debug("Fail to write a null byte into the " diff --git a/Lib/asyncio/runners.py b/Lib/asyncio/runners.py index 03ce33300eba83..268635d68fb0c0 100644 --- a/Lib/asyncio/runners.py +++ b/Lib/asyncio/runners.py @@ -5,7 +5,7 @@ from . import tasks -def run(main, *, debug=False): +def run(main, *, debug=None): """Execute the coroutine and return the result. This function runs the passed coroutine, taking care of @@ -39,7 +39,8 @@ async def main(): loop = events.new_event_loop() try: events.set_event_loop(loop) - loop.set_debug(debug) + if debug is not None: + loop.set_debug(debug) return loop.run_until_complete(main) finally: try: diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index a05cbb6bdd69dc..59cb6b1babec54 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -133,14 +133,16 @@ def _write_to_self(self): # a socket is closed, send() raises OSError (with errno set to # EBADF, but let's not rely on the exact error code). csock = self._csock - if csock is not None: - try: - csock.send(b'\0') - except OSError: - if self._debug: - logger.debug("Fail to write a null byte into the " - "self-pipe socket", - exc_info=True) + if csock is None: + return + + try: + csock.send(b'\0') + except OSError: + if self._debug: + logger.debug("Fail to write a null byte into the " + "self-pipe socket", + exc_info=True) def _start_serving(self, protocol_factory, sock, sslcontext=None, server=None, backlog=100, @@ -266,6 +268,7 @@ def _add_reader(self, fd, callback, *args): (handle, writer)) if reader is not None: reader.cancel() + return handle def _remove_reader(self, fd): if self.is_closed(): @@ -302,6 +305,7 @@ def _add_writer(self, fd, callback, *args): (reader, handle)) if writer is not None: writer.cancel() + return handle def _remove_writer(self, fd): """Remove a writer callback.""" @@ -329,7 +333,7 @@ def _remove_writer(self, fd): def add_reader(self, fd, callback, *args): """Add a reader callback.""" self._ensure_fd_no_transport(fd) - return self._add_reader(fd, callback, *args) + self._add_reader(fd, callback, *args) def remove_reader(self, fd): """Remove a reader callback.""" @@ -339,7 +343,7 @@ def remove_reader(self, fd): def add_writer(self, fd, callback, *args): """Add a writer callback..""" self._ensure_fd_no_transport(fd) - return self._add_writer(fd, callback, *args) + self._add_writer(fd, callback, *args) def remove_writer(self, fd): """Remove a writer callback.""" @@ -362,13 +366,15 @@ async def sock_recv(self, sock, n): pass fut = self.create_future() fd = sock.fileno() - self.add_reader(fd, self._sock_recv, fut, sock, n) + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recv, fut, sock, n) fut.add_done_callback( - functools.partial(self._sock_read_done, fd)) + functools.partial(self._sock_read_done, fd, handle=handle)) return await fut - def _sock_read_done(self, fd, fut): - self.remove_reader(fd) + def _sock_read_done(self, fd, fut, handle=None): + if handle is None or not handle.cancelled(): + self.remove_reader(fd) def _sock_recv(self, fut, sock, n): # _sock_recv() can add itself as an I/O callback if the operation can't @@ -401,9 +407,10 @@ async def sock_recv_into(self, sock, buf): pass fut = self.create_future() fd = sock.fileno() - self.add_reader(fd, self._sock_recv_into, fut, sock, buf) + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_recv_into, fut, sock, buf) fut.add_done_callback( - functools.partial(self._sock_read_done, fd)) + functools.partial(self._sock_read_done, fd, handle=handle)) return await fut def _sock_recv_into(self, fut, sock, buf): @@ -446,11 +453,12 @@ async def sock_sendall(self, sock, data): fut = self.create_future() fd = sock.fileno() - fut.add_done_callback( - functools.partial(self._sock_write_done, fd)) + self._ensure_fd_no_transport(fd) # use a trick with a list in closure to store a mutable state - self.add_writer(fd, self._sock_sendall, fut, sock, - memoryview(data), [n]) + handle = self._add_writer(fd, self._sock_sendall, fut, sock, + memoryview(data), [n]) + fut.add_done_callback( + functools.partial(self._sock_write_done, fd, handle=handle)) return await fut def _sock_sendall(self, fut, sock, view, pos): @@ -502,9 +510,11 @@ def _sock_connect(self, fut, sock, address): # connection runs in background. We have to wait until the socket # becomes writable to be notified when the connection succeed or # fails. + self._ensure_fd_no_transport(fd) + handle = self._add_writer( + fd, self._sock_connect_cb, fut, sock, address) fut.add_done_callback( - functools.partial(self._sock_write_done, fd)) - self.add_writer(fd, self._sock_connect_cb, fut, sock, address) + functools.partial(self._sock_write_done, fd, handle=handle)) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: @@ -512,8 +522,9 @@ def _sock_connect(self, fut, sock, address): else: fut.set_result(None) - def _sock_write_done(self, fd, fut): - self.remove_writer(fd) + def _sock_write_done(self, fd, fut, handle=None): + if handle is None or not handle.cancelled(): + self.remove_writer(fd) def _sock_connect_cb(self, fut, sock, address): if fut.done(): @@ -546,20 +557,19 @@ async def sock_accept(self, sock): if self._debug and sock.gettimeout() != 0: raise ValueError("the socket must be non-blocking") fut = self.create_future() - self._sock_accept(fut, False, sock) + self._sock_accept(fut, sock) return await fut - def _sock_accept(self, fut, registered, sock): + def _sock_accept(self, fut, sock): fd = sock.fileno() - if registered: - self.remove_reader(fd) - if fut.done(): - return try: conn, address = sock.accept() conn.setblocking(False) except (BlockingIOError, InterruptedError): - self.add_reader(fd, self._sock_accept, fut, True, sock) + self._ensure_fd_no_transport(fd) + handle = self._add_reader(fd, self._sock_accept, fut, sock) + fut.add_done_callback( + functools.partial(self._sock_read_done, fd, handle=handle)) except (SystemExit, KeyboardInterrupt): raise except BaseException as exc: diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index 21b98b6647bd90..d6262ae75e6569 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -113,34 +113,6 @@ class Task(futures._PyFuture): # Inherit Python Task implementation # status is still pending _log_destroy_pending = True - @classmethod - def current_task(cls, loop=None): - """Return the currently running task in an event loop or None. - - By default the current task for the current event loop is returned. - - None is returned when called not in the context of a Task. - """ - warnings.warn("Task.current_task() is deprecated since Python 3.7, " - "use asyncio.current_task() instead", - DeprecationWarning, - stacklevel=2) - if loop is None: - loop = events.get_event_loop() - return current_task(loop) - - @classmethod - def all_tasks(cls, loop=None): - """Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - """ - warnings.warn("Task.all_tasks() is deprecated since Python 3.7, " - "use asyncio.all_tasks() instead", - DeprecationWarning, - stacklevel=2) - return _all_tasks_compat(loop) - def __init__(self, coro, *, loop=None, name=None): super().__init__(loop=loop) if self._source_traceback: @@ -401,7 +373,7 @@ def create_task(coro, *, name=None): async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): """Wait for the Futures and coroutines given by fs to complete. - The sequence futures must not be empty. + The fs iterable must not be empty. Coroutines will be wrapped in Tasks. @@ -428,13 +400,15 @@ async def wait(fs, *, loop=None, timeout=None, return_when=ALL_COMPLETED): "and scheduled for removal in Python 3.10.", DeprecationWarning, stacklevel=2) - if any(coroutines.iscoroutine(f) for f in set(fs)): + fs = set(fs) + + if any(coroutines.iscoroutine(f) for f in fs): warnings.warn("The explicit passing of coroutine objects to " "asyncio.wait() is deprecated since Python 3.8, and " "scheduled for removal in Python 3.11.", DeprecationWarning, stacklevel=2) - fs = {ensure_future(f, loop=loop) for f in set(fs)} + fs = {ensure_future(f, loop=loop) for f in fs} return await _wait(fs, timeout, return_when, loop) @@ -473,8 +447,13 @@ async def wait_for(fut, timeout, *, loop=None): if fut.done(): return fut.result() - fut.cancel() - raise exceptions.TimeoutError() + await _cancel_and_wait(fut, loop=loop) + try: + fut.result() + except exceptions.CancelledError as exc: + raise exceptions.TimeoutError() from exc + else: + raise exceptions.TimeoutError() waiter = loop.create_future() timeout_handle = loop.call_later(timeout, _release_waiter, waiter) @@ -488,9 +467,15 @@ async def wait_for(fut, timeout, *, loop=None): try: await waiter except exceptions.CancelledError: - fut.remove_done_callback(cb) - fut.cancel() - raise + if fut.done(): + return fut.result() + else: + fut.remove_done_callback(cb) + # We must ensure that the task is not running + # after wait_for() returns. + # See https://bugs.python.org/issue32751 + await _cancel_and_wait(fut, loop=loop) + raise if fut.done(): return fut.result() @@ -593,7 +578,7 @@ def as_completed(fs, *, loop=None, timeout=None): Note: The futures 'f' are not necessarily members of fs. """ if futures.isfuture(fs) or coroutines.iscoroutine(fs): - raise TypeError(f"expect a list of futures, not {type(fs).__name__}") + raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}") from .queues import Queue # Import here to avoid circular import problem. done = Queue(loop=loop) @@ -757,6 +742,13 @@ def gather(*coros_or_futures, loop=None, return_exceptions=False): the outer Future is *not* cancelled in this case. (This is to prevent the cancellation of one child to cause other children to be cancelled.) + + If *return_exceptions* is False, cancelling gather() after it + has been marked done won't cancel any submitted awaitables. + For instance, gather can be marked done after propagating an + exception to the caller, therefore, calling ``gather.cancel()`` + after catching an exception (raised by one of the awaitables) from + gather won't cancel any other awaitables. """ if not coros_or_futures: if loop is None: diff --git a/Lib/asyncio/threads.py b/Lib/asyncio/threads.py new file mode 100644 index 00000000000000..34b7513a420902 --- /dev/null +++ b/Lib/asyncio/threads.py @@ -0,0 +1,25 @@ +"""High-level support for working with threads in asyncio""" + +import functools +import contextvars + +from . import events + + +__all__ = "to_thread", + + +async def to_thread(func, /, *args, **kwargs): + """Asynchronously run function *func* in a separate thread. + + Any *args and **kwargs supplied for this function are directly passed + to *func*. Also, the current :class:`contextvars.Context` is propogated, + allowing context variables from the main thread to be accessed in the + separate thread. + + Return a coroutine that can be awaited to get the eventual result of *func*. + """ + loop = events.get_running_loop() + ctx = contextvars.copy_context() + func_call = functools.partial(ctx.run, func, *args, **kwargs) + return await loop.run_in_executor(None, func_call) diff --git a/Lib/asyncio/transports.py b/Lib/asyncio/transports.py index 513b1c024a4d6d..45e155c94cad1b 100644 --- a/Lib/asyncio/transports.py +++ b/Lib/asyncio/transports.py @@ -29,8 +29,8 @@ def close(self): Buffered data will be flushed asynchronously. No more data will be received. After all buffered data is flushed, the - protocol's connection_lost() method will (eventually) called - with None as its argument. + protocol's connection_lost() method will (eventually) be + called with None as its argument. """ raise NotImplementedError diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py index 19d713545e4cd1..3efa6698b89ced 100644 --- a/Lib/asyncio/unix_events.py +++ b/Lib/asyncio/unix_events.py @@ -101,7 +101,7 @@ def add_signal_handler(self, sig, callback, *args): try: # Register a dummy signal handler to ask Python to write the signal - # number in the wakup file descriptor. _process_self_data() will + # number in the wakeup file descriptor. _process_self_data() will # read signal numbers from this file descriptor to handle signals. signal.signal(sig, _sighandler_noop) @@ -1230,13 +1230,15 @@ def is_active(self): def close(self): self._callbacks.clear() - if self._saved_sighandler is not None: - handler = signal.getsignal(signal.SIGCHLD) - if handler != self._sig_chld: - logger.warning("SIGCHLD handler was changed by outside code") - else: - signal.signal(signal.SIGCHLD, self._saved_sighandler) - self._saved_sighandler = None + if self._saved_sighandler is None: + return + + handler = signal.getsignal(signal.SIGCHLD) + if handler != self._sig_chld: + logger.warning("SIGCHLD handler was changed by outside code") + else: + signal.signal(signal.SIGCHLD, self._saved_sighandler) + self._saved_sighandler = None def __enter__(self): return self @@ -1263,15 +1265,17 @@ def attach_loop(self, loop): # The reason to do it here is that attach_loop() is called from # unix policy only for the main thread. # Main thread is required for subscription on SIGCHLD signal + if self._saved_sighandler is not None: + return + + self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) if self._saved_sighandler is None: - self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld) - if self._saved_sighandler is None: - logger.warning("Previous SIGCHLD handler was set by non-Python code, " - "restore to default handler on watcher close.") - self._saved_sighandler = signal.SIG_DFL + logger.warning("Previous SIGCHLD handler was set by non-Python code, " + "restore to default handler on watcher close.") + self._saved_sighandler = signal.SIG_DFL - # Set SA_RESTART to limit EINTR occurrences. - signal.siginterrupt(signal.SIGCHLD, False) + # Set SA_RESTART to limit EINTR occurrences. + signal.siginterrupt(signal.SIGCHLD, False) def _do_waitpid_all(self): for pid in list(self._callbacks): diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py index c07fe3241c569a..5e7cd795895d65 100644 --- a/Lib/asyncio/windows_events.py +++ b/Lib/asyncio/windows_events.py @@ -318,8 +318,12 @@ def run_forever(self): if self._self_reading_future is not None: ov = self._self_reading_future._ov self._self_reading_future.cancel() - # self_reading_future was just cancelled so it will never be signalled - # Unregister it otherwise IocpProactor.close will wait for it forever + # self_reading_future was just cancelled so if it hasn't been + # finished yet, it never will be (it's possible that it has + # already finished and its callback is waiting in the queue, + # where it could still happen if the event loop is restarted). + # Unregister it otherwise IocpProactor.close will wait for it + # forever if ov is not None: self._proactor._unregister(ov) self._self_reading_future = None @@ -469,7 +473,7 @@ def recv_into(self, conn, buf, flags=0): else: ov.ReadFileInto(conn.fileno(), buf) except BrokenPipeError: - return self._result(b'') + return self._result(0) def finish_recv(trans, key, ov): try: diff --git a/Lib/base64.py b/Lib/base64.py index a28109f8a7f9c3..ec3823b724a37e 100755 --- a/Lib/base64.py +++ b/Lib/base64.py @@ -320,7 +320,7 @@ def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): global _a85chars, _a85chars2 # Delay the initialization of tables to not waste memory # if the function is never called - if _a85chars is None: + if _a85chars2 is None: _a85chars = [bytes((i,)) for i in range(33, 118)] _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] @@ -428,7 +428,7 @@ def b85encode(b, pad=False): global _b85chars, _b85chars2 # Delay the initialization of tables to not waste memory # if the function is never called - if _b85chars is None: + if _b85chars2 is None: _b85chars = [bytes((i,)) for i in _b85alphabet] _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] return _85encode(b, _b85chars, _b85chars2, pad) diff --git a/Lib/bdb.py b/Lib/bdb.py index b18a0612d8c789..384a272a85217b 100644 --- a/Lib/bdb.py +++ b/Lib/bdb.py @@ -117,7 +117,7 @@ def dispatch_call(self, frame, arg): """Invoke user function and return trace function for call event. If the debugger stops on this function call, invoke - self.user_call(). Raise BbdQuit if self.quitting is set. + self.user_call(). Raise BdbQuit if self.quitting is set. Return self.trace_dispatch to continue tracing in this scope. """ # XXX 'arg' is no longer used diff --git a/Lib/binhex.py b/Lib/binhex.py index 9559f46d5a2882..ace5217d271392 100644 --- a/Lib/binhex.py +++ b/Lib/binhex.py @@ -117,12 +117,12 @@ def _flush(self, force): first = 0 while first <= len(self.hqxdata) - self.linelen: last = first + self.linelen - self.ofp.write(self.hqxdata[first:last] + b'\n') + self.ofp.write(self.hqxdata[first:last] + b'\r') self.linelen = LINELEN first = last self.hqxdata = self.hqxdata[first:] if force: - self.ofp.write(self.hqxdata + b':\n') + self.ofp.write(self.hqxdata + b':\r') def close(self): if self.data: diff --git a/Lib/bz2.py b/Lib/bz2.py index ce07ebeb142d92..7447d12fc4b8c8 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -226,15 +226,23 @@ def write(self, data): """Write a byte string to the file. Returns the number of uncompressed bytes written, which is - always len(data). Note that due to buffering, the file on disk - may not reflect the data written until close() is called. + always the length of data in bytes. Note that due to buffering, + the file on disk may not reflect the data written until close() + is called. """ with self._lock: self._check_can_write() + if isinstance(data, (bytes, bytearray)): + length = len(data) + else: + # accept any data that supports the buffer protocol + data = memoryview(data) + length = data.nbytes + compressed = self._compressor.compress(data) self._fp.write(compressed) - self._pos += len(data) - return len(data) + self._pos += length + return length def writelines(self, seq): """Write a sequence of byte strings to the file. diff --git a/Lib/cProfile.py b/Lib/cProfile.py index 4f202038d61260..22a7d0aade855f 100755 --- a/Lib/cProfile.py +++ b/Lib/cProfile.py @@ -152,6 +152,11 @@ def main(): (options, args) = parser.parse_args() sys.argv[:] = args + # The script that we're profiling may chdir, so capture the absolute path + # to the output file at startup. + if options.outfile is not None: + options.outfile = os.path.abspath(options.outfile) + if len(args) > 0: if options.module: code = "run_module(modname, run_name='__main__')" @@ -170,7 +175,12 @@ def main(): '__package__': None, '__cached__': None, } - runctx(code, globs, None, options.outfile, options.sort) + try: + runctx(code, globs, None, options.outfile, options.sort) + except BrokenPipeError as exc: + # Prevent "Exception ignored" during interpreter shutdown. + sys.stdout = None + sys.exit(exc.errno) else: parser.print_usage() return parser diff --git a/Lib/cgi.py b/Lib/cgi.py index c22c71b3878516..1e880e51848af2 100755 --- a/Lib/cgi.py +++ b/Lib/cgi.py @@ -115,7 +115,8 @@ def closelog(): # 0 ==> unlimited input maxlen = 0 -def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): +def parse(fp=None, environ=os.environ, keep_blank_values=0, + strict_parsing=0, separator='&'): """Parse a query in the environment or from a file (default stdin) Arguments, all optional: @@ -134,6 +135,9 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): strict_parsing: flag indicating what to do with parsing errors. If false (the default), errors are silently ignored. If true, errors raise a ValueError exception. + + separator: str. The symbol to use for separating the query arguments. + Defaults to &. """ if fp is None: fp = sys.stdin @@ -154,7 +158,7 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): if environ['REQUEST_METHOD'] == 'POST': ctype, pdict = parse_header(environ['CONTENT_TYPE']) if ctype == 'multipart/form-data': - return parse_multipart(fp, pdict) + return parse_multipart(fp, pdict, separator=separator) elif ctype == 'application/x-www-form-urlencoded': clength = int(environ['CONTENT_LENGTH']) if maxlen and clength > maxlen: @@ -178,10 +182,10 @@ def parse(fp=None, environ=os.environ, keep_blank_values=0, strict_parsing=0): qs = "" environ['QUERY_STRING'] = qs # XXX Shouldn't, really return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing, - encoding=encoding) + encoding=encoding, separator=separator) -def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): +def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'): """Parse multipart input. Arguments: @@ -200,9 +204,12 @@ def parse_multipart(fp, pdict, encoding="utf-8", errors="replace"): ctype = "multipart/form-data; boundary={}".format(boundary) headers = Message() headers.set_type(ctype) - headers['Content-Length'] = pdict['CONTENT-LENGTH'] + try: + headers['Content-Length'] = pdict['CONTENT-LENGTH'] + except KeyError: + pass fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors, - environ={'REQUEST_METHOD': 'POST'}) + environ={'REQUEST_METHOD': 'POST'}, separator=separator) return {k: fs.getlist(k) for k in fs} def _parseparam(s): @@ -312,7 +319,7 @@ class FieldStorage: def __init__(self, fp=None, headers=None, outerboundary=b'', environ=os.environ, keep_blank_values=0, strict_parsing=0, limit=None, encoding='utf-8', errors='replace', - max_num_fields=None): + max_num_fields=None, separator='&'): """Constructor. Read multipart/* until last part. Arguments, all optional: @@ -360,6 +367,7 @@ def __init__(self, fp=None, headers=None, outerboundary=b'', self.keep_blank_values = keep_blank_values self.strict_parsing = strict_parsing self.max_num_fields = max_num_fields + self.separator = separator if 'REQUEST_METHOD' in environ: method = environ['REQUEST_METHOD'].upper() self.qs_on_post = None @@ -586,7 +594,7 @@ def read_urlencoded(self): query = urllib.parse.parse_qsl( qs, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields) + max_num_fields=self.max_num_fields, separator=self.separator) self.list = [MiniFieldStorage(key, value) for key, value in query] self.skip_lines() @@ -602,7 +610,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): query = urllib.parse.parse_qsl( self.qs_on_post, self.keep_blank_values, self.strict_parsing, encoding=self.encoding, errors=self.errors, - max_num_fields=self.max_num_fields) + max_num_fields=self.max_num_fields, separator=self.separator) self.list.extend(MiniFieldStorage(key, value) for key, value in query) klass = self.FieldStorageClass or self.__class__ @@ -646,7 +654,7 @@ def read_multi(self, environ, keep_blank_values, strict_parsing): else self.limit - self.bytes_read part = klass(self.fp, headers, ib, environ, keep_blank_values, strict_parsing, limit, - self.encoding, self.errors, max_num_fields) + self.encoding, self.errors, max_num_fields, self.separator) if max_num_fields is not None: max_num_fields -= 1 @@ -736,7 +744,8 @@ def read_lines_to_outerboundary(self): last_line_lfend = True _read = 0 while 1: - if self.limit is not None and _read >= self.limit: + + if self.limit is not None and 0 <= self.limit <= _read: break line = self.fp.readline(1<<16) # bytes self.bytes_read += len(line) diff --git a/Lib/codecs.py b/Lib/codecs.py index 7f23e9775df804..d2edd148a290aa 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -386,7 +386,7 @@ def writelines(self, list): def reset(self): - """ Flushes and resets the codec buffers used for keeping state. + """ Resets the codec buffers used for keeping internal state. Calling this method should ensure that the data on the output is put into a clean state, that allows appending @@ -620,7 +620,7 @@ def readlines(self, sizehint=None, keepends=True): def reset(self): - """ Resets the codec buffers used for keeping state. + """ Resets the codec buffers used for keeping internal state. Note that no stream repositioning should take place. This method is primarily intended to be able to recover diff --git a/Lib/codeop.py b/Lib/codeop.py index 835e68c09ba272..4c10470aee7b7c 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -57,6 +57,7 @@ """ import __future__ +import warnings _features = [getattr(__future__, fname) for fname in __future__.all_feature_names] @@ -83,15 +84,20 @@ def _maybe_compile(compiler, source, filename, symbol): except SyntaxError: pass - try: - code1 = compiler(source + "\n", filename, symbol) - except SyntaxError as e: - err1 = e + # Catch syntax warnings after the first compile + # to emit warnings (SyntaxWarning, DeprecationWarning) at most once. + with warnings.catch_warnings(): + warnings.simplefilter("error") - try: - code2 = compiler(source + "\n\n", filename, symbol) - except SyntaxError as e: - err2 = e + try: + code1 = compiler(source + "\n", filename, symbol) + except SyntaxError as e: + err1 = e + + try: + code2 = compiler(source + "\n\n", filename, symbol) + except SyntaxError as e: + err2 = e try: if code: diff --git a/Lib/collections/__init__.py b/Lib/collections/__init__.py index c4bff592dc0e71..5bdd3b3516d3b2 100644 --- a/Lib/collections/__init__.py +++ b/Lib/collections/__init__.py @@ -14,17 +14,30 @@ ''' -__all__ = ['deque', 'defaultdict', 'namedtuple', 'UserDict', 'UserList', - 'UserString', 'Counter', 'OrderedDict', 'ChainMap'] +__all__ = [ + 'ChainMap', + 'Counter', + 'OrderedDict', + 'UserDict', + 'UserList', + 'UserString', + 'defaultdict', + 'deque', + 'namedtuple', +] import _collections_abc -from operator import itemgetter as _itemgetter, eq as _eq -from keyword import iskeyword as _iskeyword -import sys as _sys import heapq as _heapq -from _weakref import proxy as _proxy -from itertools import repeat as _repeat, chain as _chain, starmap as _starmap +import sys as _sys + +from itertools import chain as _chain +from itertools import repeat as _repeat +from itertools import starmap as _starmap +from keyword import iskeyword as _iskeyword +from operator import eq as _eq +from operator import itemgetter as _itemgetter from reprlib import recursive_repr as _recursive_repr +from _weakref import proxy as _proxy try: from _collections import deque @@ -54,6 +67,7 @@ def __getattr__(name): return obj raise AttributeError(f'module {__name__!r} has no attribute {name!r}') + ################################################################################ ### OrderedDict ################################################################################ @@ -399,18 +413,23 @@ def namedtuple(typename, field_names, *, rename=False, defaults=None, module=Non # Variables used in the methods and docstrings field_names = tuple(map(_sys.intern, field_names)) num_fields = len(field_names) - arg_list = repr(field_names).replace("'", "")[1:-1] + arg_list = ', '.join(field_names) + if num_fields == 1: + arg_list += ',' repr_fmt = '(' + ', '.join(f'{name}=%r' for name in field_names) + ')' tuple_new = tuple.__new__ _dict, _tuple, _len, _map, _zip = dict, tuple, len, map, zip # Create all the named tuple methods to be added to the class namespace - s = f'def __new__(_cls, {arg_list}): return _tuple_new(_cls, ({arg_list}))' - namespace = {'_tuple_new': tuple_new, '__name__': f'namedtuple_{typename}'} - # Note: exec() has the side-effect of interning the field names - exec(s, namespace) - __new__ = namespace['__new__'] + namespace = { + '_tuple_new': tuple_new, + '__builtins__': {}, + '__name__': f'namedtuple_{typename}', + } + code = f'lambda _cls, {arg_list}: _tuple_new(_cls, ({arg_list}))' + __new__ = eval(code, namespace) + __new__.__name__ = '__new__' __new__.__doc__ = f'Create new instance of {typename}({arg_list})' if defaults is not None: __new__.__defaults__ = defaults @@ -447,8 +466,14 @@ def __getnewargs__(self): return _tuple(self) # Modify function metadata to help with introspection and debugging - for method in (__new__, _make.__func__, _replace, - __repr__, _asdict, __getnewargs__): + for method in ( + __new__, + _make.__func__, + _replace, + __repr__, + _asdict, + __getnewargs__, + ): method.__qualname__ = f'{typename}.{method.__name__}' # Build-up the class namespace dictionary @@ -564,7 +589,7 @@ def __init__(self, iterable=None, /, **kwds): >>> c = Counter(a=4, b=2) # a new counter from keyword args ''' - super(Counter, self).__init__() + super().__init__() self.update(iterable, **kwds) def __missing__(self, key): @@ -648,7 +673,8 @@ def update(self, iterable=None, /, **kwds): for elem, count in iterable.items(): self[elem] = count + self_get(elem, 0) else: - super(Counter, self).update(iterable) # fast path when counter is empty + # fast path when counter is empty + super().update(iterable) else: _count_elements(self, iterable) if kwds: @@ -695,13 +721,14 @@ def __delitem__(self, elem): def __repr__(self): if not self: - return '%s()' % self.__class__.__name__ + return f'{self.__class__.__name__}()' try: - items = ', '.join(map('%r: %r'.__mod__, self.most_common())) - return '%s({%s})' % (self.__class__.__name__, items) + # dict() preserves the ordering returned by most_common() + d = dict(self.most_common()) except TypeError: # handle case where values are not orderable - return '{0}({1!r})'.format(self.__class__.__name__, dict(self)) + d = dict(self) + return f'{self.__class__.__name__}({d!r})' # Multiset-style mathematical operations discussed in: # Knuth TAOCP Volume II section 4.6.3 exercise 19 @@ -922,7 +949,7 @@ def __len__(self): def __iter__(self): d = {} for mapping in reversed(self.maps): - d.update(mapping) # reuses stored hash values if possible + d.update(dict.fromkeys(mapping)) # reuses stored hash values if possible return iter(d) def __contains__(self, key): @@ -966,7 +993,7 @@ def __delitem__(self, key): try: del self.maps[0][key] except KeyError: - raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + raise KeyError(f'Key not found in the first mapping: {key!r}') def popitem(self): 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' @@ -980,30 +1007,30 @@ def pop(self, key, *args): try: return self.maps[0].pop(key, *args) except KeyError: - raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + raise KeyError(f'Key not found in the first mapping: {key!r}') def clear(self): 'Clear maps[0], leaving maps[1:] intact.' self.maps[0].clear() def __ior__(self, other): - self.maps[0] |= other + self.maps[0].update(other) return self def __or__(self, other): - if isinstance(other, _collections_abc.Mapping): - m = self.maps[0].copy() - m.update(other) - return self.__class__(m, *self.maps[1:]) - return NotImplemented + if not isinstance(other, _collections_abc.Mapping): + return NotImplemented + m = self.copy() + m.maps[0].update(other) + return m def __ror__(self, other): - if isinstance(other, _collections_abc.Mapping): - m = dict(other) - for child in reversed(self.maps): - m.update(child) - return self.__class__(m) - return NotImplemented + if not isinstance(other, _collections_abc.Mapping): + return NotImplemented + m = dict(other) + for child in reversed(self.maps): + m.update(child) + return self.__class__(m) ################################################################################ @@ -1020,15 +1047,22 @@ def __init__(self, dict=None, /, **kwargs): if kwargs: self.update(kwargs) - def __len__(self): return len(self.data) + def __len__(self): + return len(self.data) + def __getitem__(self, key): if key in self.data: return self.data[key] if hasattr(self.__class__, "__missing__"): return self.__class__.__missing__(self, key) raise KeyError(key) - def __setitem__(self, key, item): self.data[key] = item - def __delitem__(self, key): del self.data[key] + + def __setitem__(self, key, item): + self.data[key] = item + + def __delitem__(self, key): + del self.data[key] + def __iter__(self): return iter(self.data) @@ -1037,7 +1071,8 @@ def __contains__(self, key): return key in self.data # Now, add the methods in dicts but not in MutableMapping - def __repr__(self): return repr(self.data) + def __repr__(self): + return repr(self.data) def __or__(self, other): if isinstance(other, UserDict): @@ -1045,12 +1080,14 @@ def __or__(self, other): if isinstance(other, dict): return self.__class__(self.data | other) return NotImplemented + def __ror__(self, other): if isinstance(other, UserDict): return self.__class__(other.data | self.data) if isinstance(other, dict): return self.__class__(other | self.data) return NotImplemented + def __ior__(self, other): if isinstance(other, UserDict): self.data |= other.data @@ -1086,13 +1123,13 @@ def fromkeys(cls, iterable, value=None): return d - ################################################################################ ### UserList ################################################################################ class UserList(_collections_abc.MutableSequence): """A more or less complete user-defined wrapper around list objects.""" + def __init__(self, initlist=None): self.data = [] if initlist is not None: @@ -1103,35 +1140,60 @@ def __init__(self, initlist=None): self.data[:] = initlist.data[:] else: self.data = list(initlist) - def __repr__(self): return repr(self.data) - def __lt__(self, other): return self.data < self.__cast(other) - def __le__(self, other): return self.data <= self.__cast(other) - def __eq__(self, other): return self.data == self.__cast(other) - def __gt__(self, other): return self.data > self.__cast(other) - def __ge__(self, other): return self.data >= self.__cast(other) + + def __repr__(self): + return repr(self.data) + + def __lt__(self, other): + return self.data < self.__cast(other) + + def __le__(self, other): + return self.data <= self.__cast(other) + + def __eq__(self, other): + return self.data == self.__cast(other) + + def __gt__(self, other): + return self.data > self.__cast(other) + + def __ge__(self, other): + return self.data >= self.__cast(other) + def __cast(self, other): return other.data if isinstance(other, UserList) else other - def __contains__(self, item): return item in self.data - def __len__(self): return len(self.data) + + def __contains__(self, item): + return item in self.data + + def __len__(self): + return len(self.data) + def __getitem__(self, i): if isinstance(i, slice): return self.__class__(self.data[i]) else: return self.data[i] - def __setitem__(self, i, item): self.data[i] = item - def __delitem__(self, i): del self.data[i] + + def __setitem__(self, i, item): + self.data[i] = item + + def __delitem__(self, i): + del self.data[i] + def __add__(self, other): if isinstance(other, UserList): return self.__class__(self.data + other.data) elif isinstance(other, type(self.data)): return self.__class__(self.data + other) return self.__class__(self.data + list(other)) + def __radd__(self, other): if isinstance(other, UserList): return self.__class__(other.data + self.data) elif isinstance(other, type(self.data)): return self.__class__(other + self.data) return self.__class__(list(other) + self.data) + def __iadd__(self, other): if isinstance(other, UserList): self.data += other.data @@ -1140,28 +1202,53 @@ def __iadd__(self, other): else: self.data += list(other) return self + def __mul__(self, n): - return self.__class__(self.data*n) + return self.__class__(self.data * n) + __rmul__ = __mul__ + def __imul__(self, n): self.data *= n return self + def __copy__(self): inst = self.__class__.__new__(self.__class__) inst.__dict__.update(self.__dict__) # Create a copy and avoid triggering descriptors inst.__dict__["data"] = self.__dict__["data"][:] return inst - def append(self, item): self.data.append(item) - def insert(self, i, item): self.data.insert(i, item) - def pop(self, i=-1): return self.data.pop(i) - def remove(self, item): self.data.remove(item) - def clear(self): self.data.clear() - def copy(self): return self.__class__(self) - def count(self, item): return self.data.count(item) - def index(self, item, *args): return self.data.index(item, *args) - def reverse(self): self.data.reverse() - def sort(self, /, *args, **kwds): self.data.sort(*args, **kwds) + + def append(self, item): + self.data.append(item) + + def insert(self, i, item): + self.data.insert(i, item) + + def pop(self, i=-1): + return self.data.pop(i) + + def remove(self, item): + self.data.remove(item) + + def clear(self): + self.data.clear() + + def copy(self): + return self.__class__(self) + + def count(self, item): + return self.data.count(item) + + def index(self, item, *args): + return self.data.index(item, *args) + + def reverse(self): + self.data.reverse() + + def sort(self, /, *args, **kwds): + self.data.sort(*args, **kwds) + def extend(self, other): if isinstance(other, UserList): self.data.extend(other.data) @@ -1169,12 +1256,12 @@ def extend(self, other): self.data.extend(other) - ################################################################################ ### UserString ################################################################################ class UserString(_collections_abc.Sequence): + def __init__(self, seq): if isinstance(seq, str): self.data = seq @@ -1182,12 +1269,25 @@ def __init__(self, seq): self.data = seq.data[:] else: self.data = str(seq) - def __str__(self): return str(self.data) - def __repr__(self): return repr(self.data) - def __int__(self): return int(self.data) - def __float__(self): return float(self.data) - def __complex__(self): return complex(self.data) - def __hash__(self): return hash(self.data) + + def __str__(self): + return str(self.data) + + def __repr__(self): + return repr(self.data) + + def __int__(self): + return int(self.data) + + def __float__(self): + return float(self.data) + + def __complex__(self): + return complex(self.data) + + def __hash__(self): + return hash(self.data) + def __getnewargs__(self): return (self.data[:],) @@ -1195,18 +1295,22 @@ def __eq__(self, string): if isinstance(string, UserString): return self.data == string.data return self.data == string + def __lt__(self, string): if isinstance(string, UserString): return self.data < string.data return self.data < string + def __le__(self, string): if isinstance(string, UserString): return self.data <= string.data return self.data <= string + def __gt__(self, string): if isinstance(string, UserString): return self.data > string.data return self.data > string + def __ge__(self, string): if isinstance(string, UserString): return self.data >= string.data @@ -1217,110 +1321,188 @@ def __contains__(self, char): char = char.data return char in self.data - def __len__(self): return len(self.data) - def __getitem__(self, index): return self.__class__(self.data[index]) + def __len__(self): + return len(self.data) + + def __getitem__(self, index): + return self.__class__(self.data[index]) + def __add__(self, other): if isinstance(other, UserString): return self.__class__(self.data + other.data) elif isinstance(other, str): return self.__class__(self.data + other) return self.__class__(self.data + str(other)) + def __radd__(self, other): if isinstance(other, str): return self.__class__(other + self.data) return self.__class__(str(other) + self.data) + def __mul__(self, n): - return self.__class__(self.data*n) + return self.__class__(self.data * n) + __rmul__ = __mul__ + def __mod__(self, args): return self.__class__(self.data % args) + def __rmod__(self, template): return self.__class__(str(template) % self) + # the following methods are defined in alphabetical order: - def capitalize(self): return self.__class__(self.data.capitalize()) + def capitalize(self): + return self.__class__(self.data.capitalize()) + def casefold(self): return self.__class__(self.data.casefold()) + def center(self, width, *args): return self.__class__(self.data.center(width, *args)) + def count(self, sub, start=0, end=_sys.maxsize): if isinstance(sub, UserString): sub = sub.data return self.data.count(sub, start, end) + def removeprefix(self, prefix, /): if isinstance(prefix, UserString): prefix = prefix.data return self.__class__(self.data.removeprefix(prefix)) + def removesuffix(self, suffix, /): if isinstance(suffix, UserString): suffix = suffix.data return self.__class__(self.data.removesuffix(suffix)) + def encode(self, encoding='utf-8', errors='strict'): encoding = 'utf-8' if encoding is None else encoding errors = 'strict' if errors is None else errors return self.data.encode(encoding, errors) + def endswith(self, suffix, start=0, end=_sys.maxsize): return self.data.endswith(suffix, start, end) + def expandtabs(self, tabsize=8): return self.__class__(self.data.expandtabs(tabsize)) + def find(self, sub, start=0, end=_sys.maxsize): if isinstance(sub, UserString): sub = sub.data return self.data.find(sub, start, end) + def format(self, /, *args, **kwds): return self.data.format(*args, **kwds) + def format_map(self, mapping): return self.data.format_map(mapping) + def index(self, sub, start=0, end=_sys.maxsize): return self.data.index(sub, start, end) - def isalpha(self): return self.data.isalpha() - def isalnum(self): return self.data.isalnum() - def isascii(self): return self.data.isascii() - def isdecimal(self): return self.data.isdecimal() - def isdigit(self): return self.data.isdigit() - def isidentifier(self): return self.data.isidentifier() - def islower(self): return self.data.islower() - def isnumeric(self): return self.data.isnumeric() - def isprintable(self): return self.data.isprintable() - def isspace(self): return self.data.isspace() - def istitle(self): return self.data.istitle() - def isupper(self): return self.data.isupper() - def join(self, seq): return self.data.join(seq) + + def isalpha(self): + return self.data.isalpha() + + def isalnum(self): + return self.data.isalnum() + + def isascii(self): + return self.data.isascii() + + def isdecimal(self): + return self.data.isdecimal() + + def isdigit(self): + return self.data.isdigit() + + def isidentifier(self): + return self.data.isidentifier() + + def islower(self): + return self.data.islower() + + def isnumeric(self): + return self.data.isnumeric() + + def isprintable(self): + return self.data.isprintable() + + def isspace(self): + return self.data.isspace() + + def istitle(self): + return self.data.istitle() + + def isupper(self): + return self.data.isupper() + + def join(self, seq): + return self.data.join(seq) + def ljust(self, width, *args): return self.__class__(self.data.ljust(width, *args)) - def lower(self): return self.__class__(self.data.lower()) - def lstrip(self, chars=None): return self.__class__(self.data.lstrip(chars)) + + def lower(self): + return self.__class__(self.data.lower()) + + def lstrip(self, chars=None): + return self.__class__(self.data.lstrip(chars)) + maketrans = str.maketrans + def partition(self, sep): return self.data.partition(sep) + def replace(self, old, new, maxsplit=-1): if isinstance(old, UserString): old = old.data if isinstance(new, UserString): new = new.data return self.__class__(self.data.replace(old, new, maxsplit)) + def rfind(self, sub, start=0, end=_sys.maxsize): if isinstance(sub, UserString): sub = sub.data return self.data.rfind(sub, start, end) + def rindex(self, sub, start=0, end=_sys.maxsize): return self.data.rindex(sub, start, end) + def rjust(self, width, *args): return self.__class__(self.data.rjust(width, *args)) + def rpartition(self, sep): return self.data.rpartition(sep) + def rstrip(self, chars=None): return self.__class__(self.data.rstrip(chars)) + def split(self, sep=None, maxsplit=-1): return self.data.split(sep, maxsplit) + def rsplit(self, sep=None, maxsplit=-1): return self.data.rsplit(sep, maxsplit) - def splitlines(self, keepends=False): return self.data.splitlines(keepends) + + def splitlines(self, keepends=False): + return self.data.splitlines(keepends) + def startswith(self, prefix, start=0, end=_sys.maxsize): return self.data.startswith(prefix, start, end) - def strip(self, chars=None): return self.__class__(self.data.strip(chars)) - def swapcase(self): return self.__class__(self.data.swapcase()) - def title(self): return self.__class__(self.data.title()) + + def strip(self, chars=None): + return self.__class__(self.data.strip(chars)) + + def swapcase(self): + return self.__class__(self.data.swapcase()) + + def title(self): + return self.__class__(self.data.title()) + def translate(self, *args): return self.__class__(self.data.translate(*args)) - def upper(self): return self.__class__(self.data.upper()) - def zfill(self, width): return self.__class__(self.data.zfill(width)) + + def upper(self): + return self.__class__(self.data.upper()) + + def zfill(self, width): + return self.__class__(self.data.zfill(width)) diff --git a/Lib/collections/abc.py b/Lib/collections/abc.py index 891600d16bee9e..86ca8b8a8414b3 100644 --- a/Lib/collections/abc.py +++ b/Lib/collections/abc.py @@ -1,2 +1,3 @@ from _collections_abc import * from _collections_abc import __all__ +from _collections_abc import _CallableGenericAlias diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py index bf546f8ae1d1cc..6095026cb278b9 100644 --- a/Lib/concurrent/futures/_base.py +++ b/Lib/concurrent/futures/_base.py @@ -386,7 +386,11 @@ def done(self): def __get_result(self): if self._exception: - raise self._exception + try: + raise self._exception + finally: + # Break a reference cycle with the exception in self._exception + self = None else: return self._result @@ -426,20 +430,24 @@ def result(self, timeout=None): timeout. Exception: If the call raised then that exception will be raised. """ - with self._condition: - if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: - raise CancelledError() - elif self._state == FINISHED: - return self.__get_result() - - self._condition.wait(timeout) - - if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: - raise CancelledError() - elif self._state == FINISHED: - return self.__get_result() - else: - raise TimeoutError() + try: + with self._condition: + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + + self._condition.wait(timeout) + + if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: + raise CancelledError() + elif self._state == FINISHED: + return self.__get_result() + else: + raise TimeoutError() + finally: + # Break a reference cycle with the exception in self._exception + self = None def exception(self, timeout=None): """Return the exception raised by the call that the future represents. @@ -605,7 +613,7 @@ def result_iterator(): future.cancel() return result_iterator() - def shutdown(self, wait=True): + def shutdown(self, wait=True, *, cancel_futures=False): """Clean-up the resources associated with the Executor. It is safe to call this method several times. Otherwise, no other @@ -615,6 +623,9 @@ def shutdown(self, wait=True): wait: If True then shutdown will not return until all running futures have finished executing and the resources used by the executor have been reclaimed. + cancel_futures: If True then shutdown will cancel all pending + futures. Futures that are completed or running will not be + cancelled. """ pass diff --git a/Lib/configparser.py b/Lib/configparser.py index 924cc56a3f150d..8dd5c13bcc0092 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -907,6 +907,9 @@ def write(self, fp, space_around_delimiters=True): If `space_around_delimiters' is True (the default), delimiters between keys and values are surrounded by spaces. + + Please note that comments in the original configuration file are not + preserved when writing the configuration back. """ if space_around_delimiters: d = " {} ".format(self._delimiters[0]) @@ -1005,7 +1008,7 @@ def _read(self, fp, fpname): Configuration files may include comments, prefixed by specific characters (`#' and `;' by default). Comments may appear on their own in an otherwise empty line or may be entered in lines holding values or - section names. + section names. Please note that comments get stripped off when reading configuration files. """ elements_added = set() cursect = None # None, or a dictionary diff --git a/Lib/copyreg.py b/Lib/copyreg.py index dfc463c49a389d..7ab8c128eb0445 100644 --- a/Lib/copyreg.py +++ b/Lib/copyreg.py @@ -48,6 +48,7 @@ def _reconstructor(cls, base, state): return obj _HEAPTYPE = 1<<9 +_new_type = type(int.__new__) # Python code for object.__reduce_ex__ for protocols 0 and 1 @@ -57,6 +58,9 @@ def _reduce_ex(self, proto): for base in cls.__mro__: if hasattr(base, '__flags__') and not base.__flags__ & _HEAPTYPE: break + new = base.__new__ + if isinstance(new, _new_type) and new.__self__ is base: + break else: base = object # not really reachable if base is object: diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765a3e..1c3f8fd38b0665 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/ctypes/test/test_callbacks.py index f622093df61da5..d8e9c5a760e2c2 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/ctypes/test/test_callbacks.py @@ -1,5 +1,7 @@ import functools import unittest +from test import support + from ctypes import * from ctypes.test import need_symbol import _ctypes_test @@ -287,7 +289,36 @@ def callback(check, s): self.assertEqual(s.second, check.second) self.assertEqual(s.third, check.third) -################################################################ + def test_callback_too_many_args(self): + def func(*args): + return len(args) + + CTYPES_MAX_ARGCOUNT = 1024 + proto = CFUNCTYPE(c_int, *(c_int,) * CTYPES_MAX_ARGCOUNT) + cb = proto(func) + args1 = (1,) * CTYPES_MAX_ARGCOUNT + self.assertEqual(cb(*args1), CTYPES_MAX_ARGCOUNT) + + args2 = (1,) * (CTYPES_MAX_ARGCOUNT + 1) + with self.assertRaises(ArgumentError): + cb(*args2) + + def test_convert_result_error(self): + def func(): + return ("tuple",) + + proto = CFUNCTYPE(c_int) + ctypes_func = proto(func) + with support.catch_unraisable_exception() as cm: + # don't test the result since it is an uninitialized value + result = ctypes_func() + + self.assertIsInstance(cm.unraisable.exc_value, TypeError) + self.assertEqual(cm.unraisable.err_msg, + "Exception ignored on converting result " + "of ctypes callback function") + self.assertIs(cm.unraisable.object, func) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_find.py b/Lib/ctypes/test/test_find.py index b99fdcba7b28fc..92ac1840ad7d43 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/ctypes/test/test_find.py @@ -1,4 +1,5 @@ import unittest +import unittest.mock import os.path import sys import test.support @@ -72,7 +73,7 @@ def test_shell_injection(self): @unittest.skipUnless(sys.platform.startswith('linux'), 'Test only valid for Linux') -class LibPathFindTest(unittest.TestCase): +class FindLibraryLinux(unittest.TestCase): def test_find_on_libpath(self): import subprocess import tempfile @@ -111,6 +112,15 @@ def test_find_on_libpath(self): # LD_LIBRARY_PATH) self.assertEqual(find_library(libname), 'lib%s.so' % libname) + def test_find_library_with_gcc(self): + with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None): + self.assertNotEqual(find_library('c'), None) + + def test_find_library_with_ld(self): + with unittest.mock.patch("ctypes.util._findSoname_ldconfig", lambda *args: None), \ + unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): + self.assertNotEqual(find_library('c'), None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951acfab..a1bac26a7df058 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/ctypes/test/test_parameters.py index e4c25fd880cefb..38af7ac13d756c 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/ctypes/test/test_parameters.py @@ -201,6 +201,49 @@ def __dict__(self): with self.assertRaises(ZeroDivisionError): WorseStruct().__setstate__({}, b'foo') + def test_parameter_repr(self): + from ctypes import ( + c_bool, + c_char, + c_wchar, + c_byte, + c_ubyte, + c_short, + c_ushort, + c_int, + c_uint, + c_long, + c_ulong, + c_longlong, + c_ulonglong, + c_float, + c_double, + c_longdouble, + c_char_p, + c_wchar_p, + c_void_p, + ) + self.assertRegex(repr(c_bool.from_param(True)), r"^$") + self.assertEqual(repr(c_char.from_param(97)), "") + self.assertRegex(repr(c_wchar.from_param('a')), r"^$") + self.assertEqual(repr(c_byte.from_param(98)), "") + self.assertEqual(repr(c_ubyte.from_param(98)), "") + self.assertEqual(repr(c_short.from_param(511)), "") + self.assertEqual(repr(c_ushort.from_param(511)), "") + self.assertRegex(repr(c_int.from_param(20000)), r"^$") + self.assertRegex(repr(c_uint.from_param(20000)), r"^$") + self.assertRegex(repr(c_long.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulong.from_param(20000)), r"^$") + self.assertRegex(repr(c_longlong.from_param(20000)), r"^$") + self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") + self.assertEqual(repr(c_float.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1.5)), "") + self.assertEqual(repr(c_double.from_param(1e300)), "") + self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") + self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") + self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") + self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") + ################################################################ if __name__ == '__main__': diff --git a/Lib/ctypes/test/test_random_things.py b/Lib/ctypes/test/test_random_things.py index ee5b2128ea0fab..2988e275cf4bbf 100644 --- a/Lib/ctypes/test/test_random_things.py +++ b/Lib/ctypes/test/test_random_things.py @@ -1,5 +1,9 @@ from ctypes import * -import unittest, sys +import contextlib +from test import support +import unittest +import sys + def callback_func(arg): 42 / arg @@ -34,41 +38,40 @@ class CallbackTracbackTestCase(unittest.TestCase): # created, then a full traceback printed. When SystemExit is # raised in a callback function, the interpreter exits. - def capture_stderr(self, func, *args, **kw): - # helper - call function 'func', and return the captured stderr - import io - old_stderr = sys.stderr - logger = sys.stderr = io.StringIO() - try: - func(*args, **kw) - finally: - sys.stderr = old_stderr - return logger.getvalue() + @contextlib.contextmanager + def expect_unraisable(self, exc_type, exc_msg=None): + with support.catch_unraisable_exception() as cm: + yield + + self.assertIsInstance(cm.unraisable.exc_value, exc_type) + if exc_msg is not None: + self.assertEqual(str(cm.unraisable.exc_value), exc_msg) + self.assertEqual(cm.unraisable.err_msg, + "Exception ignored on calling ctypes " + "callback function") + self.assertIs(cm.unraisable.object, callback_func) def test_ValueError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 42) - self.assertEqual(out.splitlines()[-1], - "ValueError: 42") + with self.expect_unraisable(ValueError, '42'): + cb(42) def test_IntegerDivisionError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) - out = self.capture_stderr(cb, 0) - self.assertEqual(out.splitlines()[-1][:19], - "ZeroDivisionError: ") + with self.expect_unraisable(ZeroDivisionError): + cb(0) def test_FloatDivisionError(self): cb = CFUNCTYPE(c_int, c_double)(callback_func) - out = self.capture_stderr(cb, 0.0) - self.assertEqual(out.splitlines()[-1][:19], - "ZeroDivisionError: ") + with self.expect_unraisable(ZeroDivisionError): + cb(0.0) def test_TypeErrorDivisionError(self): cb = CFUNCTYPE(c_int, c_char_p)(callback_func) - out = self.capture_stderr(cb, b"spam") - self.assertEqual(out.splitlines()[-1], - "TypeError: " - "unsupported operand type(s) for /: 'int' and 'bytes'") + err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'" + with self.expect_unraisable(TypeError, err_msg): + cb(b"spam") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_unaligned_structures.py b/Lib/ctypes/test/test_unaligned_structures.py index bcacfc8184b439..ee7fb45809bf7b 100644 --- a/Lib/ctypes/test/test_unaligned_structures.py +++ b/Lib/ctypes/test/test_unaligned_structures.py @@ -27,7 +27,6 @@ class Y(SwappedStructure): class TestStructures(unittest.TestCase): def test_native(self): for typ in structures: -## print typ.value self.assertEqual(typ.value.offset, 1) o = typ() o.value = 4 @@ -35,7 +34,6 @@ def test_native(self): def test_swapped(self): for typ in byteswapped_structures: -## print >> sys.stderr, typ.value self.assertEqual(typ.value.offset, 1) o = typ() o.value = 4 diff --git a/Lib/ctypes/test/test_unicode.py b/Lib/ctypes/test/test_unicode.py index c200af7b650661..60c75424b767fa 100644 --- a/Lib/ctypes/test/test_unicode.py +++ b/Lib/ctypes/test/test_unicode.py @@ -26,6 +26,14 @@ def test_buffers(self): self.assertEqual(buf[::2], 'a\xe4\xfc') self.assertEqual(buf[6:5:-1], "") + def test_embedded_null(self): + class TestStruct(ctypes.Structure): + _fields_ = [("unicode", ctypes.c_wchar_p)] + t = TestStruct() + # This would raise a ValueError: + t.unicode = "foo\0bar\0\0" + + func = ctypes.CDLL(_ctypes_test.__file__)._testfunc_p_p class StringTestCase(UnicodeTestCase): diff --git a/Lib/ctypes/test/test_wintypes.py b/Lib/ctypes/test/test_wintypes.py index 71442df8301aff..243d5962ffa7f1 100644 --- a/Lib/ctypes/test/test_wintypes.py +++ b/Lib/ctypes/test/test_wintypes.py @@ -1,12 +1,13 @@ -import sys import unittest +# also work on POSIX + from ctypes import * +from ctypes import wintypes + -@unittest.skipUnless(sys.platform.startswith('win'), 'Windows-only test') class WinTypesTest(unittest.TestCase): def test_variant_bool(self): - from ctypes import wintypes # reads 16-bits from memory, anything non-zero is True for true_value in (1, 32767, 32768, 65535, 65537): true = POINTER(c_int16)(c_int16(true_value)) @@ -37,5 +38,6 @@ def test_variant_bool(self): vb.value = [] self.assertIs(vb.value, False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 01176bf9696577..0c2510e1619c8e 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -93,6 +93,12 @@ def find_library(name): # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile + def _is_elf(filename): + "Return True if the given file is an ELF file" + elf_header = b'\x7fELF' + with open(filename, 'br') as thefile: + return thefile.read(4) == elf_header + def _findLib_gcc(name): # Run GCC's linker with the -t (aka --trace) option and examine the # library name it prints out. The GCC command will fail because we @@ -130,10 +136,17 @@ def _findLib_gcc(name): # Raised if the file was already removed, which is the normal # behaviour of GCC if linking fails pass - res = re.search(expr, trace) + res = re.findall(expr, trace) if not res: return None - return os.fsdecode(res.group(0)) + + for file in res: + # Check if the given file is an elf file: gcc can report + # some files that are linker scripts and not actual + # shared objects. See bpo-41976 for more details + if not _is_elf(file): + continue + return os.fsdecode(file) if sys.platform == "sunos5": @@ -299,9 +312,14 @@ def _findLib_ld(name): stderr=subprocess.PIPE, universal_newlines=True) out, _ = p.communicate() - res = re.search(expr, os.fsdecode(out)) - if res: - result = res.group(0) + res = re.findall(expr, os.fsdecode(out)) + for file in res: + # Check if the given file is an elf file: gcc can report + # some files that are linker scripts and not actual + # shared objects. See bpo-41976 for more details + if not _is_elf(file): + continue + return os.fsdecode(file) except Exception: pass # result will be None return result @@ -309,7 +327,7 @@ def _findLib_ld(name): def find_library(name): # See issue #9998 return _findSoname_ldconfig(name) or \ - _get_soname(_findLib_gcc(name) or _findLib_ld(name)) + _get_soname(_findLib_gcc(name)) or _get_soname(_findLib_ld(name)) ################################################################ # test code diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index fc69508354bbe8..c98e74d4ff9cc2 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -696,7 +696,7 @@ def _get_field(cls, a_name, a_type): # In addition to checking for actual types here, also check for # string annotations. get_type_hints() won't always work for us # (see https://github.com/python/typing/issues/508 for example), - # plus it's expensive and would require an eval for every stirng + # plus it's expensive and would require an eval for every string # annotation. So, make a best effort to see if this is a ClassVar # or InitVar using regex's and checking that the thing referenced # is actually of the correct type. @@ -836,7 +836,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): # Only process classes that have been processed by our # decorator. That is, they have a _FIELDS attribute. base_fields = getattr(b, _FIELDS, None) - if base_fields: + if base_fields is not None: has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f @@ -944,7 +944,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen): _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) if eq: - # Create _eq__ method. There's no need for a __ne__ method, + # Create __eq__ method. There's no need for a __ne__ method, # since python will call __eq__ and negate it. flds = [f for f in field_list if f.compare] self_tuple = _tuple_str('self', flds) @@ -1094,7 +1094,7 @@ def _asdict_inner(obj, dict_factory): # method, because: # - it does not recurse in to the namedtuple fields and # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The the main + # - I don't actually want to return a dict here. The main # use case here is json.dumps, and it handles converting # namedtuples to lists. Admittedly we're losing some # information here when we produce a json list instead of a @@ -1271,7 +1271,7 @@ class C: continue if f.name not in changes: - if f._field_type is _FIELD_INITVAR: + if f._field_type is _FIELD_INITVAR and f.default is MISSING: raise ValueError(f"InitVar {f.name!r} " 'must be specified with replace()') changes[f.name] = getattr(obj, f.name) diff --git a/Lib/datetime.py b/Lib/datetime.py index 952aebfdec0a7c..23d2bf0918145f 100644 --- a/Lib/datetime.py +++ b/Lib/datetime.py @@ -1452,7 +1452,8 @@ def isoformat(self, timespec='auto'): part is omitted if self.microsecond == 0. The optional argument timespec specifies the number of additional - terms of the time to include. + terms of the time to include. Valid options are 'auto', 'hours', + 'minutes', 'seconds', 'milliseconds' and 'microseconds'. """ s = _format_time(self._hour, self._minute, self._second, self._microsecond, timespec) @@ -1578,7 +1579,7 @@ def __setstate(self, string, tzinfo): self._tzinfo = tzinfo def __reduce_ex__(self, protocol): - return (time, self._getstate(protocol)) + return (self.__class__, self._getstate(protocol)) def __reduce__(self): return self.__reduce_ex__(2) @@ -1937,7 +1938,8 @@ def isoformat(self, sep='T', timespec='auto'): time, default 'T'. The optional argument timespec specifies the number of additional - terms of the time to include. + terms of the time to include. Valid options are 'auto', 'hours', + 'minutes', 'seconds', 'milliseconds' and 'microseconds'. """ s = ("%04d-%02d-%02d%c" % (self._year, self._month, self._day, sep) + _format_time(self._hour, self._minute, self._second, @@ -2356,7 +2358,7 @@ def _name_from_offset(delta): # This is again a requirement for a sane tzinfo class. # # 4. (x+k).s = x.s -# This follows from #2, and that datimetimetz+timedelta preserves tzinfo. +# This follows from #2, and that datetime.timetz+timedelta preserves tzinfo. # # 5. (x+k).n = x.n + k # Again follows from how arithmetic is defined. diff --git a/Lib/dis.py b/Lib/dis.py index 10e5f7fb08ab21..e289e176c78ffd 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -542,7 +542,7 @@ def _test(): import argparse parser = argparse.ArgumentParser() - parser.add_argument('infile', type=argparse.FileType(), nargs='?', default='-') + parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') args = parser.parse_args() with args.infile as infile: source = infile.read() diff --git a/Lib/distutils/command/build_py.py b/Lib/distutils/command/build_py.py index cf0ca57c320472..edc2171cd122dd 100644 --- a/Lib/distutils/command/build_py.py +++ b/Lib/distutils/command/build_py.py @@ -5,7 +5,7 @@ import os import importlib.util import sys -from glob import glob +import glob from distutils.core import Command from distutils.errors import * @@ -125,7 +125,7 @@ def find_data_files(self, package, src_dir): files = [] for pattern in globs: # Each pattern has to be converted to a platform-specific path - filelist = glob(os.path.join(src_dir, convert_path(pattern))) + filelist = glob.glob(os.path.join(glob.escape(src_dir), convert_path(pattern))) # Files that match more than one pattern are only added once files.extend([fn for fn in filelist if fn not in files and os.path.isfile(fn)]) @@ -216,7 +216,7 @@ def check_module(self, module, module_file): def find_package_modules(self, package, package_dir): self.check_package(package, package_dir) - module_files = glob(os.path.join(package_dir, "*.py")) + module_files = glob.glob(os.path.join(glob.escape(package_dir), "*.py")) modules = [] setup_script = os.path.abspath(self.distribution.script_name) diff --git a/Lib/distutils/command/upload.py b/Lib/distutils/command/upload.py index d822ba01338af9..e0ecb655b93faf 100644 --- a/Lib/distutils/command/upload.py +++ b/Lib/distutils/command/upload.py @@ -9,13 +9,24 @@ import io import hashlib from base64 import standard_b64encode -from urllib.request import urlopen, Request, HTTPError +from urllib.error import HTTPError +from urllib.request import urlopen, Request from urllib.parse import urlparse from distutils.errors import DistutilsError, DistutilsOptionError from distutils.core import PyPIRCCommand from distutils.spawn import spawn from distutils import log + +# PyPI Warehouse supports MD5, SHA256, and Blake2 (blake2-256) +# https://bugs.python.org/issue40698 +_FILE_CONTENT_DIGESTS = { + "md5_digest": getattr(hashlib, "md5", None), + "sha256_digest": getattr(hashlib, "sha256", None), + "blake2_256_digest": getattr(hashlib, "blake2b", None), +} + + class upload(PyPIRCCommand): description = "upload binary package to PyPI" @@ -87,6 +98,7 @@ def upload_file(self, command, pyversion, filename): content = f.read() finally: f.close() + meta = self.distribution.metadata data = { # action @@ -101,7 +113,6 @@ def upload_file(self, command, pyversion, filename): 'content': (os.path.basename(filename),content), 'filetype': command, 'pyversion': pyversion, - 'md5_digest': hashlib.md5(content).hexdigest(), # additional meta-data 'metadata_version': '1.0', @@ -123,6 +134,16 @@ def upload_file(self, command, pyversion, filename): data['comment'] = '' + # file content digests + for digest_name, digest_cons in _FILE_CONTENT_DIGESTS.items(): + if digest_cons is None: + continue + try: + data[digest_name] = digest_cons(content).hexdigest() + except ValueError: + # hash digest not available or blocked by security policy + pass + if self.sign: with open(filename + ".asc", "rb") as f: data['gpg_signature'] = (os.path.basename(filename) + ".asc", diff --git a/Lib/distutils/spawn.py b/Lib/distutils/spawn.py index aad277b0ca7767..31df3f7faca552 100644 --- a/Lib/distutils/spawn.py +++ b/Lib/distutils/spawn.py @@ -59,21 +59,31 @@ def spawn(cmd, search_path=1, verbose=0, dry_run=0): if _cfg_target: _cfg_target_split = [int(x) for x in _cfg_target.split('.')] if _cfg_target: - # ensure that the deployment target of build process is not less - # than that used when the interpreter was built. This ensures - # extension modules are built with correct compatibility values + # Ensure that the deployment target of the build process is not + # less than 10.3 if the interpreter was built for 10.3 or later. + # This ensures extension modules are built with correct + # compatibility values, specifically LDSHARED which can use + # '-undefined dynamic_lookup' which only works on >= 10.3. cur_target = os.environ.get('MACOSX_DEPLOYMENT_TARGET', _cfg_target) - if _cfg_target_split > [int(x) for x in cur_target.split('.')]: + cur_target_split = [int(x) for x in cur_target.split('.')] + if _cfg_target_split[:2] >= [10, 3] and cur_target_split[:2] < [10, 3]: my_msg = ('$MACOSX_DEPLOYMENT_TARGET mismatch: ' - 'now "%s" but "%s" during configure' + 'now "%s" but "%s" during configure;' + 'must use 10.3 or later' % (cur_target, _cfg_target)) raise DistutilsPlatformError(my_msg) env = dict(os.environ, MACOSX_DEPLOYMENT_TARGET=cur_target) - proc = subprocess.Popen(cmd, env=env) - proc.wait() - exitcode = proc.returncode + try: + proc = subprocess.Popen(cmd, env=env) + proc.wait() + exitcode = proc.returncode + except OSError as exc: + if not DEBUG: + cmd = cmd[0] + raise DistutilsExecError( + "command %r failed: %s" % (cmd, exc.args[-1])) from exc if exitcode: if not DEBUG: diff --git a/Lib/distutils/tests/__init__.py b/Lib/distutils/tests/__init__.py index 5d2e69e3e6a8f6..68037216c7d06e 100644 --- a/Lib/distutils/tests/__init__.py +++ b/Lib/distutils/tests/__init__.py @@ -15,26 +15,24 @@ import os import sys import unittest -import warnings -from test.support import run_unittest +from test.support import run_unittest, save_restore_warnings_filters here = os.path.dirname(__file__) or os.curdir def test_suite(): - old_filters = warnings.filters[:] suite = unittest.TestSuite() for fn in os.listdir(here): if fn.startswith("test") and fn.endswith(".py"): modname = "distutils.tests." + fn[:-3] - __import__(modname) + # bpo-40055: Save/restore warnings filters to leave them unchanged. + # Importing tests imports docutils which imports pkg_resources + # which adds a warnings filter. + with save_restore_warnings_filters(): + __import__(modname) module = sys.modules[modname] suite.addTest(module.test_suite()) - # bpo-40055: Save/restore warnings filters to leave them unchanged. - # Importing tests imports docutils which imports pkg_resources which adds a - # warnings filter. - warnings.filters[:] = old_filters return suite diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index 5e47e0773a9649..808c0dc2874cd5 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -492,12 +492,16 @@ def _try_compile_deployment_target(self, operator, target): # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: # for 10.10 and beyond -> "10nn00" - target = '%02d%02d00' % target + if len(target) >= 2: + target = '%02d%02d00' % target + else: + # 11 and later can have no minor version (11 instead of 11.0) + target = '%02d0000' % target deptarget_ext = Extension( 'deptarget', [deptarget_c], diff --git a/Lib/distutils/tests/test_spawn.py b/Lib/distutils/tests/test_spawn.py index cf1faad5f4dd5d..ad5038142faadf 100644 --- a/Lib/distutils/tests/test_spawn.py +++ b/Lib/distutils/tests/test_spawn.py @@ -124,6 +124,11 @@ def test_find_executable(self): rv = find_executable(program) self.assertEqual(rv, filename) + def test_spawn_missing_exe(self): + with self.assertRaises(DistutilsExecError) as ctx: + spawn(['does-not-exist']) + self.assertIn("command 'does-not-exist' failed", str(ctx.exception)) + def test_suite(): return unittest.makeSuite(SpawnTestCase) diff --git a/Lib/distutils/tests/test_upload.py b/Lib/distutils/tests/test_upload.py index c17d8e7d54e98c..74f0bc0a67131b 100644 --- a/Lib/distutils/tests/test_upload.py +++ b/Lib/distutils/tests/test_upload.py @@ -2,7 +2,7 @@ import os import unittest import unittest.mock as mock -from urllib.request import HTTPError +from urllib.error import HTTPError from test.support import run_unittest @@ -130,14 +130,30 @@ def test_upload(self): # what did we send ? headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2162') + self.assertGreaterEqual(int(headers['Content-length']), 2162) content_type = headers['Content-type'] self.assertTrue(content_type.startswith('multipart/form-data')) self.assertEqual(self.last_open.req.get_method(), 'POST') expected_url = 'https://upload.pypi.org/legacy/' self.assertEqual(self.last_open.req.get_full_url(), expected_url) - self.assertTrue(b'xxx' in self.last_open.req.data) - self.assertIn(b'protocol_version', self.last_open.req.data) + data = self.last_open.req.data + self.assertIn(b'xxx',data) + self.assertIn(b'protocol_version', data) + self.assertIn(b'sha256_digest', data) + self.assertIn( + b'cd2eb0837c9b4c962c22d2ff8b5441b7b45805887f051d39bf133b583baf' + b'6860', + data + ) + if b'md5_digest' in data: + self.assertIn(b'f561aaf6ef0bf14d4208bb46a4ccb3ad', data) + if b'blake2_256_digest' in data: + self.assertIn( + b'b6f289a27d4fe90da63c503bfe0a9b761a8f76bb86148565065f040be' + b'6d1c3044cf7ded78ef800509bccb4b648e507d88dc6383d67642aadcc' + b'ce443f1534330a', + data + ) # The PyPI response body was echoed results = self.get_logs(INFO) @@ -166,7 +182,7 @@ def test_upload_correct_cr(self): cmd.run() headers = dict(self.last_open.req.headers) - self.assertEqual(headers['Content-length'], '2172') + self.assertGreaterEqual(int(headers['Content-length']), 2172) self.assertIn(b'long description\r', self.last_open.req.data) def test_upload_fails(self): diff --git a/Lib/distutils/unixccompiler.py b/Lib/distutils/unixccompiler.py index 4d7a6de740ab3a..f0792de74a1a48 100644 --- a/Lib/distutils/unixccompiler.py +++ b/Lib/distutils/unixccompiler.py @@ -290,7 +290,7 @@ def find_library_file(self, dirs, lib, debug=0): cflags = sysconfig.get_config_var('CFLAGS') m = re.search(r'-isysroot\s*(\S+)', cflags) if m is None: - sysroot = '/' + sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) else: sysroot = m.group(1) diff --git a/Lib/email/_header_value_parser.py b/Lib/email/_header_value_parser.py index 9c55ef7fb453be..51d355fbb0abc5 100644 --- a/Lib/email/_header_value_parser.py +++ b/Lib/email/_header_value_parser.py @@ -1218,12 +1218,21 @@ def get_bare_quoted_string(value): if value[0] in WSP: token, value = get_fws(value) elif value[:2] == '=?': + valid_ew = False try: token, value = get_encoded_word(value) bare_quoted_string.defects.append(errors.InvalidHeaderDefect( "encoded word inside quoted string")) + valid_ew = True except errors.HeaderParseError: token, value = get_qcontent(value) + # Collapse the whitespace between two encoded words that occur in a + # bare-quoted-string. + if valid_ew and len(bare_quoted_string) > 1: + if (bare_quoted_string[-1].token_type == 'fws' and + bare_quoted_string[-2].token_type == 'encoded-word'): + bare_quoted_string[-1] = EWWhiteSpaceTerminal( + bare_quoted_string[-1], 'fws') else: token, value = get_qcontent(value) bare_quoted_string.append(token) diff --git a/Lib/email/contentmanager.py b/Lib/email/contentmanager.py index 2b4b8757f46f62..b91fb0e5bca7a8 100644 --- a/Lib/email/contentmanager.py +++ b/Lib/email/contentmanager.py @@ -146,7 +146,7 @@ def embedded_body(lines): return linesep.join(lines) + linesep def normal_body(lines): return b'\n'.join(lines) + b'\n' if cte==None: # Use heuristics to decide on the "best" encoding. - if max(len(x) for x in lines) <= policy.max_line_length: + if max((len(x) for x in lines), default=0) <= policy.max_line_length: try: return '7bit', normal_body(lines).decode('ascii') except UnicodeDecodeError: diff --git a/Lib/email/generator.py b/Lib/email/generator.py index ae670c2353c858..c9b121624e08d5 100644 --- a/Lib/email/generator.py +++ b/Lib/email/generator.py @@ -186,7 +186,11 @@ def _write(self, msg): # If we munged the cte, copy the message again and re-fix the CTE. if munge_cte: msg = deepcopy(msg) - msg.replace_header('content-transfer-encoding', munge_cte[0]) + # Preserve the header order if the CTE header already exists. + if msg.get('content-transfer-encoding') is None: + msg['Content-Transfer-Encoding'] = munge_cte[0] + else: + msg.replace_header('content-transfer-encoding', munge_cte[0]) msg.replace_header('content-type', munge_cte[1]) # Write the headers. First we see if the message object wants to # handle that itself. If not, we'll do it generically. diff --git a/Lib/email/message.py b/Lib/email/message.py index 12626026179608..db30d9a1708992 100644 --- a/Lib/email/message.py +++ b/Lib/email/message.py @@ -141,7 +141,7 @@ def as_string(self, unixfrom=False, maxheaderlen=0, policy=None): header. For backward compatibility reasons, if maxheaderlen is not specified it defaults to 0, so you must override it explicitly if you want a different maxheaderlen. 'policy' is passed to the - Generator instance used to serialize the mesasge; if it is not + Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. If the message object contains binary data that is not encoded @@ -948,7 +948,7 @@ def __init__(self, policy=None): if policy is None: from email.policy import default policy = default - Message.__init__(self, policy) + super().__init__(policy) def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): @@ -958,14 +958,14 @@ def as_string(self, unixfrom=False, maxheaderlen=None, policy=None): header. maxheaderlen is retained for backward compatibility with the base Message class, but defaults to None, meaning that the policy value for max_line_length controls the header maximum length. 'policy' is - passed to the Generator instance used to serialize the mesasge; if it + passed to the Generator instance used to serialize the message; if it is not specified the policy associated with the message instance is used. """ policy = self.policy if policy is None else policy if maxheaderlen is None: maxheaderlen = policy.max_line_length - return super().as_string(maxheaderlen=maxheaderlen, policy=policy) + return super().as_string(unixfrom, maxheaderlen, policy) def __str__(self): return self.as_string(policy=self.policy.clone(utf8=True)) diff --git a/Lib/email/utils.py b/Lib/email/utils.py index b137ce3973a4b4..1a7719dbc4898f 100644 --- a/Lib/email/utils.py +++ b/Lib/email/utils.py @@ -81,7 +81,7 @@ def formataddr(pair, charset='utf-8'): If the first element of pair is false, then the second element is returned unmodified. - Optional charset if given is the character set that is used to encode + The optional charset is the character set that is used to encode realname in case realname is not ASCII safe. Can be an instance of str or a Charset-like object which has a header_encode method. Default is 'utf-8'. diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 545fce656fd6f5..14a39037e0c771 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -3,6 +3,7 @@ import sys import runpy import tempfile +import subprocess from importlib import resources from . import _bundled @@ -12,33 +13,29 @@ __all__ = ["version", "bootstrap"] -_SETUPTOOLS_VERSION = "41.2.0" +_SETUPTOOLS_VERSION = "56.0.0" -_PIP_VERSION = "19.2.3" +_PIP_VERSION = "21.1.3" _PROJECTS = [ - ("setuptools", _SETUPTOOLS_VERSION), - ("pip", _PIP_VERSION), + ("setuptools", _SETUPTOOLS_VERSION, "py3"), + ("pip", _PIP_VERSION, "py3"), ] def _run_pip(args, additional_paths=None): - # Add our bundled software to the sys.path so we can import it - if additional_paths is not None: - sys.path = additional_paths + sys.path - - # Invoke pip as if it's the main module, and catch the exit. - backup_argv = sys.argv[:] - sys.argv[1:] = args - try: - # run_module() alters sys.modules and sys.argv, but restores them at exit - runpy.run_module("pip", run_name="__main__", alter_sys=True) - except SystemExit as exc: - return exc.code - finally: - sys.argv[:] = backup_argv - - raise SystemError("pip did not exit, this should never happen") + # Run the bootstraping in a subprocess to avoid leaking any state that happens + # after pip has executed. Particulary, this avoids the case when pip holds onto + # the files in *additional_paths*, preventing us to remove them at the end of the + # invocation. + code = f""" +import runpy +import sys +sys.path = {additional_paths or []} + sys.path +sys.argv[1:] = {args} +runpy.run_module("pip", run_name="__main__", alter_sys=True) +""" + return subprocess.run([sys.executable, "-c", code], check=True).returncode def version(): @@ -107,8 +104,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path additional_paths = [] - for project, version in _PROJECTS: - wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version) + for project, version, py_tag in _PROJECTS: + wheel_name = "{}-{}-{}-none-any.whl".format(project, version, py_tag) whl = resources.read_binary( _bundled, wheel_name, @@ -119,7 +116,7 @@ def _bootstrap(*, root=None, upgrade=False, user=False, additional_paths.append(os.path.join(tmpdir, wheel_name)) # Construct the arguments to be passed to the pip command - args = ["install", "--no-index", "--find-links", tmpdir] + args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] if root: args += ["--root", root] if upgrade: diff --git a/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl deleted file mode 100644 index 8118df8ac1940f..00000000000000 Binary files a/Lib/ensurepip/_bundled/pip-19.2.3-py2.py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl new file mode 100644 index 00000000000000..d96a40a9291fb2 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-21.1.3-py3-none-any.whl differ diff --git a/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl deleted file mode 100644 index 82df6f63f4ee97..00000000000000 Binary files a/Lib/ensurepip/_bundled/setuptools-41.2.0-py2.py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl b/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl new file mode 100644 index 00000000000000..264ef10e826679 Binary files /dev/null and b/Lib/ensurepip/_bundled/setuptools-56.0.0-py3-none-any.whl differ diff --git a/Lib/enum.py b/Lib/enum.py index 49b552ba0ecf6c..ee4c4c04f98bdd 100644 --- a/Lib/enum.py +++ b/Lib/enum.py @@ -10,31 +10,54 @@ def _is_descriptor(obj): - """Returns True if obj is a descriptor, False otherwise.""" + """ + Returns True if obj is a descriptor, False otherwise. + """ return ( hasattr(obj, '__get__') or hasattr(obj, '__set__') or - hasattr(obj, '__delete__')) - + hasattr(obj, '__delete__') + ) def _is_dunder(name): - """Returns True if a __dunder__ name, False otherwise.""" - return (len(name) > 4 and + """ + Returns True if a __dunder__ name, False otherwise. + """ + return ( + len(name) > 4 and name[:2] == name[-2:] == '__' and name[2] != '_' and - name[-3] != '_') - + name[-3] != '_' + ) def _is_sunder(name): - """Returns True if a _sunder_ name, False otherwise.""" - return (len(name) > 2 and + """ + Returns True if a _sunder_ name, False otherwise. + """ + return ( + len(name) > 2 and name[0] == name[-1] == '_' and name[1:2] != '_' and - name[-2:-1] != '_') - + name[-2:-1] != '_' + ) + +def _is_private(cls_name, name): + # do not use `re` as `re` imports `enum` + pattern = '_%s__' % (cls_name, ) + if ( + len(name) >= 5 + and name.startswith(pattern) + and name[len(pattern)] != '_' + and (name[-1] != '_' or name[-2] != '_') + ): + return True + else: + return False def _make_class_unpicklable(cls): - """Make the given class un-picklable.""" + """ + Make the given class un-picklable. + """ def _break_on_call_reduce(self, proto): raise TypeError('%r cannot be pickled' % self) cls.__reduce_ex__ = _break_on_call_reduce @@ -49,11 +72,11 @@ class auto: class _EnumDict(dict): - """Track enum member order and ensure member names are not reused. + """ + Track enum member order and ensure member names are not reused. EnumMeta will use the names found in self._member_names as the enumeration member names. - """ def __init__(self): super().__init__() @@ -63,14 +86,22 @@ def __init__(self): self._auto_called = False def __setitem__(self, key, value): - """Changes anything not dundered or not a descriptor. + """ + Changes anything not dundered or not a descriptor. If an enum member name is used twice, an error is raised; duplicate values are not checked for. Single underscore (sunder) names are reserved. - """ + if _is_private(self._cls_name, key): + import warnings + warnings.warn( + "private variables, such as %r, will be normal attributes in 3.10" + % (key, ), + DeprecationWarning, + stacklevel=2, + ) if _is_sunder(key): if key not in ( '_order_', '_create_pseudo_member_', @@ -90,7 +121,10 @@ def __setitem__(self, key, value): self._ignore = value already = set(value) & set(self._member_names) if already: - raise ValueError('_ignore_ cannot specify already set names: %r' % (already, )) + raise ValueError( + '_ignore_ cannot specify already set names: %r' + % (already, ) + ) elif _is_dunder(key): if key == '__order__': key = '_order_' @@ -104,9 +138,14 @@ def __setitem__(self, key, value): # enum overwriting a descriptor? raise TypeError('%r already defined as: %r' % (key, self[key])) if isinstance(value, auto): - self._auto_called = True if value.value == _auto_null: - value.value = self._generate_next_value(key, 1, len(self._member_names), self._last_values[:]) + value.value = self._generate_next_value( + key, + 1, + len(self._member_names), + self._last_values[:], + ) + self._auto_called = True value = value.value self._member_names.append(key) self._last_values.append(value) @@ -118,20 +157,26 @@ def __setitem__(self, key, value): # This is also why there are checks in EnumMeta like `if Enum is not None` Enum = None - class EnumMeta(type): - """Metaclass for Enum""" + """ + Metaclass for Enum + """ @classmethod - def __prepare__(metacls, cls, bases): + def __prepare__(metacls, cls, bases, **kwds): + # check that previous enum members do not exist + metacls._check_for_existing_members(cls, bases) # create the namespace dict enum_dict = _EnumDict() + enum_dict._cls_name = cls # inherit previous flags and _generate_next_value_ function - member_type, first_enum = metacls._get_mixins_(bases) + member_type, first_enum = metacls._get_mixins_(cls, bases) if first_enum is not None: - enum_dict['_generate_next_value_'] = getattr(first_enum, '_generate_next_value_', None) + enum_dict['_generate_next_value_'] = getattr( + first_enum, '_generate_next_value_', None, + ) return enum_dict - def __new__(metacls, cls, bases, classdict): + def __new__(metacls, cls, bases, classdict, **kwds): # an Enum class is final once enumeration items have been defined; it # cannot be mixed with other types (int, float, etc.) if it has an # inherited __new__ unless a new __new__ is defined (or the resulting @@ -142,9 +187,10 @@ def __new__(metacls, cls, bases, classdict): ignore = classdict['_ignore_'] for key in ignore: classdict.pop(key, None) - member_type, first_enum = metacls._get_mixins_(bases) - __new__, save_new, use_args = metacls._find_new_(classdict, member_type, - first_enum) + member_type, first_enum = metacls._get_mixins_(cls, bases) + __new__, save_new, use_args = metacls._find_new_( + classdict, member_type, first_enum, + ) # save enum items into separate mapping so they don't get baked into # the new class @@ -165,17 +211,18 @@ def __new__(metacls, cls, bases, classdict): if '__doc__' not in classdict: classdict['__doc__'] = 'An enumeration.' - # create our new Enum type - enum_class = super().__new__(metacls, cls, bases, classdict) + enum_class = super().__new__(metacls, cls, bases, classdict, **kwds) enum_class._member_names_ = [] # names in definition order enum_class._member_map_ = {} # name->value map enum_class._member_type_ = member_type # save DynamicClassAttribute attributes from super classes so we know # if we can take the shortcut of storing members in the class dict - dynamic_attributes = {k for c in enum_class.mro() - for k, v in c.__dict__.items() - if isinstance(v, DynamicClassAttribute)} + dynamic_attributes = { + k for c in enum_class.mro() + for k, v in c.__dict__.items() + if isinstance(v, DynamicClassAttribute) + } # Reverse value->name map for hashable values. enum_class._value2member_map_ = {} @@ -195,8 +242,32 @@ def __new__(metacls, cls, bases, classdict): methods = ('__getnewargs_ex__', '__getnewargs__', '__reduce_ex__', '__reduce__') if not any(m in member_type.__dict__ for m in methods): - _make_class_unpicklable(enum_class) - + if '__new__' in classdict: + # too late, sabotage + _make_class_unpicklable(enum_class) + else: + # final attempt to verify that pickling would work: + # travel mro until __new__ is found, checking for + # __reduce__ and friends along the way -- if any of them + # are found before/when __new__ is found, pickling should + # work + sabotage = None + for chain in bases: + for base in chain.__mro__: + if base is object: + continue + elif any(m in base.__dict__ for m in methods): + # found one, we're good + sabotage = False + break + elif '__new__' in base.__dict__: + # not good + sabotage = True + break + if sabotage is not None: + break + if sabotage: + _make_class_unpicklable(enum_class) # instantiate them, checking for duplicates as we go # we instantiate first instead of checking for duplicates first in case # a custom __new__ is doing something funky with the values -- such as @@ -249,7 +320,11 @@ def __new__(metacls, cls, bases, classdict): # double check that repr and friends are not the mixin's or various # things break (such as pickle) + # however, if the method is defined in the Enum itself, don't replace + # it for name in ('__repr__', '__str__', '__format__', '__reduce_ex__'): + if name in classdict: + continue class_method = getattr(enum_class, name) obj_method = getattr(member_type, name, None) enum_method = getattr(first_enum, name, None) @@ -281,7 +356,8 @@ def __bool__(self): return True def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1): - """Either returns an existing member, or creates a new enum class. + """ + Either returns an existing member, or creates a new enum class. This method is used both when an enum class is given a value to match to an enumeration member (i.e. Color(3)) and for the functional API @@ -303,12 +379,18 @@ def __call__(cls, value, names=None, *, module=None, qualname=None, type=None, s not correct, unpickling will fail in some circumstances. `type`, if set, will be mixed in as the first base class. - """ if names is None: # simple value lookup return cls.__new__(cls, value) # otherwise, functional API: we're creating a new Enum type - return cls._create_(value, names, module=module, qualname=qualname, type=type, start=start) + return cls._create_( + value, + names, + module=module, + qualname=qualname, + type=type, + start=start, + ) def __contains__(cls, member): if not isinstance(member, Enum): @@ -321,22 +403,23 @@ def __delattr__(cls, attr): # nicer error message when someone tries to delete an attribute # (see issue19025). if attr in cls._member_map_: - raise AttributeError( - "%s: cannot delete Enum member." % cls.__name__) + raise AttributeError("%s: cannot delete Enum member." % cls.__name__) super().__delattr__(attr) def __dir__(self): - return (['__class__', '__doc__', '__members__', '__module__'] + - self._member_names_) + return ( + ['__class__', '__doc__', '__members__', '__module__'] + + self._member_names_ + ) def __getattr__(cls, name): - """Return the enum member matching `name` + """ + Return the enum member matching `name` We use __getattr__ instead of descriptors or inserting into the enum class' __dict__ in order to support `name` and `value` being both properties for enum members (which live in the class' __dict__) and enum members themselves. - """ if _is_dunder(name): raise AttributeError(name) @@ -349,6 +432,9 @@ def __getitem__(cls, name): return cls._member_map_[name] def __iter__(cls): + """ + Returns members in definition order. + """ return (cls._member_map_[name] for name in cls._member_names_) def __len__(cls): @@ -356,11 +442,11 @@ def __len__(cls): @property def __members__(cls): - """Returns a mapping of member name->value. + """ + Returns a mapping of member name->value. This mapping lists all enum members, including aliases. Note that this is a read-only view of the internal mapping. - """ return MappingProxyType(cls._member_map_) @@ -368,15 +454,18 @@ def __repr__(cls): return "" % cls.__name__ def __reversed__(cls): + """ + Returns members in reverse definition order. + """ return (cls._member_map_[name] for name in reversed(cls._member_names_)) def __setattr__(cls, name, value): - """Block attempts to reassign Enum members. + """ + Block attempts to reassign Enum members. A simple assignment to the class namespace only changes one of the several possible ways to get an Enum member from the Enum class, resulting in an inconsistent Enumeration. - """ member_map = cls.__dict__.get('_member_map_', {}) if name in member_map: @@ -384,7 +473,8 @@ def __setattr__(cls, name, value): super().__setattr__(name, value) def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, start=1): - """Convenience method to create a new Enum class. + """ + Convenience method to create a new Enum class. `names` can be: @@ -393,11 +483,10 @@ def _create_(cls, class_name, names, *, module=None, qualname=None, type=None, s * An iterable of member names. Values are incremented by 1 from `start`. * An iterable of (member name, value) pairs. * A mapping of member name -> value pairs. - """ metacls = cls.__class__ bases = (cls, ) if type is None else (type, cls) - _, first_enum = cls._get_mixins_(bases) + _, first_enum = cls._get_mixins_(cls, bases) classdict = metacls.__prepare__(class_name, bases) # special processing needed for names? @@ -470,25 +559,50 @@ def _convert_(cls, name, module, filter, source=None): return cls @staticmethod - def _get_mixins_(bases): - """Returns the type for creating enum members, and the first inherited + def _check_for_existing_members(class_name, bases): + for chain in bases: + for base in chain.__mro__: + if issubclass(base, Enum) and base._member_names_: + raise TypeError( + "%s: cannot extend enumeration %r" + % (class_name, base.__name__) + ) + + @staticmethod + def _get_mixins_(class_name, bases): + """ + Returns the type for creating enum members, and the first inherited enum class. bases: the tuple of bases that was given to __new__ - """ if not bases: return object, Enum def _find_data_type(bases): + data_types = set() for chain in bases: + candidate = None for base in chain.__mro__: if base is object: continue + elif issubclass(base, Enum): + if base._member_type_ is not object: + data_types.add(base._member_type_) + break elif '__new__' in base.__dict__: if issubclass(base, Enum): continue - return base + data_types.add(candidate or base) + break + else: + candidate = candidate or base + if len(data_types) > 1: + raise TypeError('%r: too many data types: %r' % (class_name, data_types)) + elif data_types: + return data_types.pop() + else: + return None # ensure final parent class is an Enum derivative, find any concrete # data type, and check that Enum has no members @@ -503,12 +617,12 @@ def _find_data_type(bases): @staticmethod def _find_new_(classdict, member_type, first_enum): - """Returns the __new__ to be used for creating the enum members. + """ + Returns the __new__ to be used for creating the enum members. classdict: the class dictionary given to __new__ member_type: the data type whose __new__ will be used by default first_enum: enumeration to check for an overriding __new__ - """ # now find the correct __new__, checking to see of one was defined # by the user; also check earlier enum classes in case a __new__ was @@ -548,10 +662,10 @@ def _find_new_(classdict, member_type, first_enum): class Enum(metaclass=EnumMeta): - """Generic enumeration. + """ + Generic enumeration. Derive from this class to define new enumerations. - """ def __new__(cls, value): # all enum instances are actually created during class construction @@ -579,21 +693,34 @@ def __new__(cls, value): except Exception as e: exc = e result = None - if isinstance(result, cls): - return result - else: - ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) - if result is None and exc is None: - raise ve_exc - elif exc is None: - exc = TypeError( - 'error in %s._missing_: returned %r instead of None or a valid member' - % (cls.__name__, result) - ) - exc.__context__ = ve_exc - raise exc + try: + if isinstance(result, cls): + return result + else: + ve_exc = ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + if result is None and exc is None: + raise ve_exc + elif exc is None: + exc = TypeError( + 'error in %s._missing_: returned %r instead of None or a valid member' + % (cls.__name__, result) + ) + exc.__context__ = ve_exc + raise exc + finally: + # ensure all variables that could hold an exception are destroyed + exc = None + ve_exc = None def _generate_next_value_(name, start, count, last_values): + """ + Generate the next value when not given. + + name: the name of the member + start: the initial start value or None + count: the number of existing members + last_value: the last value assigned or None + """ for last_value in reversed(last_values): try: return last_value + 1 @@ -604,7 +731,7 @@ def _generate_next_value_(name, start, count, last_values): @classmethod def _missing_(cls, value): - raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) + return None def __repr__(self): return "<%s.%s: %r>" % ( @@ -614,21 +741,27 @@ def __str__(self): return "%s.%s" % (self.__class__.__name__, self._name_) def __dir__(self): + """ + Returns all members and all public methods + """ added_behavior = [ m for cls in self.__class__.mro() for m in cls.__dict__ if m[0] != '_' and m not in self._member_map_ - ] + ] + [m for m in self.__dict__ if m[0] != '_'] return (['__class__', '__doc__', '__module__'] + added_behavior) def __format__(self, format_spec): + """ + Returns format using actual value type unless __str__ has been overridden. + """ # mixed-in Enums should use the mixed-in type's __format__, otherwise # we can get strange results with the Enum name showing up instead of # the value # pure Enum branch, or branch with __str__ explicitly overridden - str_overridden = type(self).__str__ != Enum.__str__ + str_overridden = type(self).__str__ not in (Enum.__str__, Flag.__str__) if self._member_type_ is object or str_overridden: cls = str val = str(self) @@ -670,7 +803,9 @@ def _reduce_ex_by_name(self, proto): return self.name class Flag(Enum): - """Support for flags""" + """ + Support for flags + """ def _generate_next_value_(name, start, count, last_values): """ @@ -693,6 +828,9 @@ def _generate_next_value_(name, start, count, last_values): @classmethod def _missing_(cls, value): + """ + Returns member (possibly creating it) if one can be found for value. + """ original_value = value if value < 0: value = ~value @@ -722,6 +860,9 @@ def _create_pseudo_member_(cls, value): return pseudo_member def __contains__(self, other): + """ + Returns True if self has at least the same flags set as other. + """ if not isinstance(other, self.__class__): raise TypeError( "unsupported operand type(s) for 'in': '%s' and '%s'" % ( @@ -780,10 +921,15 @@ def __invert__(self): class IntFlag(int, Flag): - """Support for integer-based Flags""" + """ + Support for integer-based Flags + """ @classmethod def _missing_(cls, value): + """ + Returns member (possibly creating it) if one can be found for value. + """ if not isinstance(value, int): raise ValueError("%r is not a valid %s" % (value, cls.__qualname__)) new_member = cls._create_pseudo_member_(value) @@ -791,6 +937,9 @@ def _missing_(cls, value): @classmethod def _create_pseudo_member_(cls, value): + """ + Create a composite member iff value contains only members. + """ pseudo_member = cls._value2member_map_.get(value, None) if pseudo_member is None: need_to_create = [value] @@ -845,11 +994,15 @@ def __invert__(self): def _high_bit(value): - """returns index of highest bit, or -1 if value is zero or negative""" + """ + returns index of highest bit, or -1 if value is zero or negative + """ return value.bit_length() - 1 def unique(enumeration): - """Class decorator for enumerations ensuring unique member values.""" + """ + Class decorator for enumerations ensuring unique member values. + """ duplicates = [] for name, member in enumeration.__members__.items(): if name != member.name: @@ -862,7 +1015,9 @@ def unique(enumeration): return enumeration def _decompose(flag, value): - """Extract all members from the value.""" + """ + Extract all members from the value. + """ # _decompose is only called if the value is not named not_covered = value negative = value < 0 diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index 0eb1802bdb53c5..7c52c23067d40f 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -52,7 +52,7 @@ def _compile_pattern(pat): return re.compile(res).match def filter(names, pat): - """Return the subset of the list NAMES that match PAT.""" + """Construct a list from those elements of the iterable NAMES that match PAT.""" result = [] pat = os.path.normcase(pat) match = _compile_pattern(pat) diff --git a/Lib/ftplib.py b/Lib/ftplib.py index 1f760ed1ce0bf0..7c5a50715f6dc6 100644 --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -102,7 +102,9 @@ class FTP: sock = None file = None welcome = None - passiveserver = 1 + passiveserver = True + # Disables https://bugs.python.org/issue43285 security if set to True. + trust_server_pasv_ipv4_address = False def __init__(self, host='', user='', passwd='', acct='', timeout=_GLOBAL_DEFAULT_TIMEOUT, source_address=None, *, @@ -320,8 +322,13 @@ def makeport(self): return sock def makepasv(self): + """Internal: Does the PASV or EPSV handshake -> (address, port)""" if self.af == socket.AF_INET: - host, port = parse227(self.sendcmd('PASV')) + untrusted_host, port = parse227(self.sendcmd('PASV')) + if self.trust_server_pasv_ipv4_address: + host = untrusted_host + else: + host = self.sock.getpeername()[0] else: host, port = parse229(self.sendcmd('EPSV'), self.sock.getpeername()) return host, port diff --git a/Lib/functools.py b/Lib/functools.py index 87c7d87438998b..5cab497d264037 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -11,7 +11,6 @@ __all__ = ['update_wrapper', 'wraps', 'WRAPPER_ASSIGNMENTS', 'WRAPPER_UPDATES', 'total_ordering', 'cache', 'cmp_to_key', 'lru_cache', 'reduce', - 'TopologicalSorter', 'CycleError', 'partial', 'partialmethod', 'singledispatch', 'singledispatchmethod', 'cached_property'] @@ -199,250 +198,6 @@ def total_ordering(cls): setattr(cls, opname, opfunc) return cls -################################################################################ -### topological sort -################################################################################ - -_NODE_OUT = -1 -_NODE_DONE = -2 - - -class _NodeInfo: - __slots__ = 'node', 'npredecessors', 'successors' - - def __init__(self, node): - # The node this class is augmenting. - self.node = node - - # Number of predecessors, generally >= 0. When this value falls to 0, - # and is returned by get_ready(), this is set to _NODE_OUT and when the - # node is marked done by a call to done(), set to _NODE_DONE. - self.npredecessors = 0 - - # List of successor nodes. The list can contain duplicated elements as - # long as they're all reflected in the successor's npredecessors attribute). - self.successors = [] - - -class CycleError(ValueError): - """Subclass of ValueError raised by TopologicalSorterif cycles exist in the graph - - If multiple cycles exist, only one undefined choice among them will be reported - and included in the exception. The detected cycle can be accessed via the second - element in the *args* attribute of the exception instance and consists in a list - of nodes, such that each node is, in the graph, an immediate predecessor of the - next node in the list. In the reported list, the first and the last node will be - the same, to make it clear that it is cyclic. - """ - pass - - -class TopologicalSorter: - """Provides functionality to topologically sort a graph of hashable nodes""" - - def __init__(self, graph=None): - self._node2info = {} - self._ready_nodes = None - self._npassedout = 0 - self._nfinished = 0 - - if graph is not None: - for node, predecessors in graph.items(): - self.add(node, *predecessors) - - def _get_nodeinfo(self, node): - if (result := self._node2info.get(node)) is None: - self._node2info[node] = result = _NodeInfo(node) - return result - - def add(self, node, *predecessors): - """Add a new node and its predecessors to the graph. - - Both the *node* and all elements in *predecessors* must be hashable. - - If called multiple times with the same node argument, the set of dependencies - will be the union of all dependencies passed in. - - It is possible to add a node with no dependencies (*predecessors* is not provided) - as well as provide a dependency twice. If a node that has not been provided before - is included among *predecessors* it will be automatically added to the graph with - no predecessors of its own. - - Raises ValueError if called after "prepare". - """ - if self._ready_nodes is not None: - raise ValueError("Nodes cannot be added after a call to prepare()") - - # Create the node -> predecessor edges - nodeinfo = self._get_nodeinfo(node) - nodeinfo.npredecessors += len(predecessors) - - # Create the predecessor -> node edges - for pred in predecessors: - pred_info = self._get_nodeinfo(pred) - pred_info.successors.append(node) - - def prepare(self): - """Mark the graph as finished and check for cycles in the graph. - - If any cycle is detected, "CycleError" will be raised, but "get_ready" can - still be used to obtain as many nodes as possible until cycles block more - progress. After a call to this function, the graph cannot be modified and - therefore no more nodes can be added using "add". - """ - if self._ready_nodes is not None: - raise ValueError("cannot prepare() more than once") - - self._ready_nodes = [i.node for i in self._node2info.values() - if i.npredecessors == 0] - # ready_nodes is set before we look for cycles on purpose: - # if the user wants to catch the CycleError, that's fine, - # they can continue using the instance to grab as many - # nodes as possible before cycles block more progress - cycle = self._find_cycle() - if cycle: - raise CycleError(f"nodes are in a cycle", cycle) - - def get_ready(self): - """Return a tuple of all the nodes that are ready. - - Initially it returns all nodes with no predecessors; once those are marked - as processed by calling "done", further calls will return all new nodes that - have all their predecessors already processed. Once no more progress can be made, - empty tuples are returned. - - Raises ValueError if called without calling "prepare" previously. - """ - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - - # Get the nodes that are ready and mark them - result = tuple(self._ready_nodes) - n2i = self._node2info - for node in result: - n2i[node].npredecessors = _NODE_OUT - - # Clean the list of nodes that are ready and update - # the counter of nodes that we have returned. - self._ready_nodes.clear() - self._npassedout += len(result) - - return result - - def is_active(self): - """Return True if more progress can be made and ``False`` otherwise. - - Progress can be made if cycles do not block the resolution and either there - are still nodes ready that haven't yet been returned by "get_ready" or the - number of nodes marked "done" is less than the number that have been returned - by "get_ready". - - Raises ValueError if called without calling "prepare" previously. - """ - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - return self._nfinished < self._npassedout or bool(self._ready_nodes) - - def __bool__(self): - return self.is_active() - - def done(self, *nodes): - """Marks a set of nodes returned by "get_ready" as processed. - - This method unblocks any successor of each node in *nodes* for being returned - in the future by a a call to "get_ready" - - Raises :exec:`ValueError` if any node in *nodes* has already been marked as - processed by a previous call to this method, if a node was not added to the - graph by using "add" or if called without calling "prepare" previously or if - node has not yet been returned by "get_ready". - """ - - if self._ready_nodes is None: - raise ValueError("prepare() must be called first") - - n2i = self._node2info - - for node in nodes: - - # Check if we know about this node (it was added previously using add() - if (nodeinfo := n2i.get(node)) is None: - raise ValueError(f"node {node!r} was not added using add()") - - # If the node has not being returned (marked as ready) previously, inform the user. - stat = nodeinfo.npredecessors - if stat != _NODE_OUT: - if stat >= 0: - raise ValueError(f"node {node!r} was not passed out (still not ready)") - elif stat == _NODE_DONE: - raise ValueError(f"node {node!r} was already marked done") - else: - assert False, f"node {node!r}: unknown status {stat}" - - # Mark the node as processed - nodeinfo.npredecessors = _NODE_DONE - - # Go to all the successors and reduce the number of predecessors, collecting all the ones - # that are ready to be returned in the next get_ready() call. - for successor in nodeinfo.successors: - successor_info = n2i[successor] - successor_info.npredecessors -= 1 - if successor_info.npredecessors == 0: - self._ready_nodes.append(successor) - self._nfinished += 1 - - def _find_cycle(self): - n2i = self._node2info - stack = [] - itstack = [] - seen = set() - node2stacki = {} - - for node in n2i: - if node in seen: - continue - - while True: - if node in seen: - # If we have seen already the node and is in the - # current stack we have found a cycle. - if node in node2stacki: - return stack[node2stacki[node]:] + [node] - # else go on to get next successor - else: - seen.add(node) - itstack.append(iter(n2i[node].successors).__next__) - node2stacki[node] = len(stack) - stack.append(node) - - # Backtrack to the topmost stack entry with - # at least another successor. - while stack: - try: - node = itstack[-1]() - break - except StopIteration: - del node2stacki[stack.pop()] - itstack.pop() - else: - break - return None - - def static_order(self): - """Returns an iterable of nodes in a topological order. - - The particular order that is returned may depend on the specific - order in which the items were inserted in the graph. - - Using this method does not require to call "prepare" or "done". If any - cycle is detected, :exc:`CycleError` will be raised. - """ - self.prepare() - while self.is_active(): - node_group = self.get_ready() - yield from node_group - self.done(*node_group) - ################################################################################ ### cmp_to_key() function converter diff --git a/Lib/getpass.py b/Lib/getpass.py index 6911f41d1f2054..6970d8adfbab36 100644 --- a/Lib/getpass.py +++ b/Lib/getpass.py @@ -95,7 +95,7 @@ def unix_getpass(prompt='Password: ', stream=None): def win_getpass(prompt='Password: ', stream=None): - """Prompt for password with echo off, using Windows getch().""" + """Prompt for password with echo off, using Windows getwch().""" if sys.stdin is not sys.__stdin__: return fallback_getpass(prompt, stream) diff --git a/Lib/glob.py b/Lib/glob.py index 0dd2f8be661094..12370611309515 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -1,5 +1,6 @@ """Filename globbing utility.""" +import contextlib import os import re import fnmatch @@ -79,7 +80,7 @@ def _iglob(pathname, recursive, dironly): # takes a literal basename (so it only has to check for its existence). def _glob1(dirname, pattern, dironly): - names = list(_iterdir(dirname, dironly)) + names = _listdir(dirname, dironly) if not _ishidden(pattern): names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) @@ -130,9 +131,13 @@ def _iterdir(dirname, dironly): except OSError: return +def _listdir(dirname, dironly): + with contextlib.closing(_iterdir(dirname, dironly)) as it: + return list(it) + # Recursively yields relative pathnames inside a literal directory. def _rlistdir(dirname, dironly): - names = list(_iterdir(dirname, dironly)) + names = _listdir(dirname, dironly) for x in names: if not _ishidden(x): yield x diff --git a/Lib/graphlib.py b/Lib/graphlib.py new file mode 100644 index 00000000000000..d0e7a4814c565d --- /dev/null +++ b/Lib/graphlib.py @@ -0,0 +1,246 @@ +__all__ = ["TopologicalSorter", "CycleError"] + +_NODE_OUT = -1 +_NODE_DONE = -2 + + +class _NodeInfo: + __slots__ = "node", "npredecessors", "successors" + + def __init__(self, node): + # The node this class is augmenting. + self.node = node + + # Number of predecessors, generally >= 0. When this value falls to 0, + # and is returned by get_ready(), this is set to _NODE_OUT and when the + # node is marked done by a call to done(), set to _NODE_DONE. + self.npredecessors = 0 + + # List of successor nodes. The list can contain duplicated elements as + # long as they're all reflected in the successor's npredecessors attribute). + self.successors = [] + + +class CycleError(ValueError): + """Subclass of ValueError raised by TopologicalSorter.prepare if cycles + exist in the working graph. + + If multiple cycles exist, only one undefined choice among them will be reported + and included in the exception. The detected cycle can be accessed via the second + element in the *args* attribute of the exception instance and consists in a list + of nodes, such that each node is, in the graph, an immediate predecessor of the + next node in the list. In the reported list, the first and the last node will be + the same, to make it clear that it is cyclic. + """ + + pass + + +class TopologicalSorter: + """Provides functionality to topologically sort a graph of hashable nodes""" + + def __init__(self, graph=None): + self._node2info = {} + self._ready_nodes = None + self._npassedout = 0 + self._nfinished = 0 + + if graph is not None: + for node, predecessors in graph.items(): + self.add(node, *predecessors) + + def _get_nodeinfo(self, node): + if (result := self._node2info.get(node)) is None: + self._node2info[node] = result = _NodeInfo(node) + return result + + def add(self, node, *predecessors): + """Add a new node and its predecessors to the graph. + + Both the *node* and all elements in *predecessors* must be hashable. + + If called multiple times with the same node argument, the set of dependencies + will be the union of all dependencies passed in. + + It is possible to add a node with no dependencies (*predecessors* is not provided) + as well as provide a dependency twice. If a node that has not been provided before + is included among *predecessors* it will be automatically added to the graph with + no predecessors of its own. + + Raises ValueError if called after "prepare". + """ + if self._ready_nodes is not None: + raise ValueError("Nodes cannot be added after a call to prepare()") + + # Create the node -> predecessor edges + nodeinfo = self._get_nodeinfo(node) + nodeinfo.npredecessors += len(predecessors) + + # Create the predecessor -> node edges + for pred in predecessors: + pred_info = self._get_nodeinfo(pred) + pred_info.successors.append(node) + + def prepare(self): + """Mark the graph as finished and check for cycles in the graph. + + If any cycle is detected, "CycleError" will be raised, but "get_ready" can + still be used to obtain as many nodes as possible until cycles block more + progress. After a call to this function, the graph cannot be modified and + therefore no more nodes can be added using "add". + """ + if self._ready_nodes is not None: + raise ValueError("cannot prepare() more than once") + + self._ready_nodes = [ + i.node for i in self._node2info.values() if i.npredecessors == 0 + ] + # ready_nodes is set before we look for cycles on purpose: + # if the user wants to catch the CycleError, that's fine, + # they can continue using the instance to grab as many + # nodes as possible before cycles block more progress + cycle = self._find_cycle() + if cycle: + raise CycleError(f"nodes are in a cycle", cycle) + + def get_ready(self): + """Return a tuple of all the nodes that are ready. + + Initially it returns all nodes with no predecessors; once those are marked + as processed by calling "done", further calls will return all new nodes that + have all their predecessors already processed. Once no more progress can be made, + empty tuples are returned. + + Raises ValueError if called without calling "prepare" previously. + """ + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + + # Get the nodes that are ready and mark them + result = tuple(self._ready_nodes) + n2i = self._node2info + for node in result: + n2i[node].npredecessors = _NODE_OUT + + # Clean the list of nodes that are ready and update + # the counter of nodes that we have returned. + self._ready_nodes.clear() + self._npassedout += len(result) + + return result + + def is_active(self): + """Return ``True`` if more progress can be made and ``False`` otherwise. + + Progress can be made if cycles do not block the resolution and either there + are still nodes ready that haven't yet been returned by "get_ready" or the + number of nodes marked "done" is less than the number that have been returned + by "get_ready". + + Raises ValueError if called without calling "prepare" previously. + """ + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + return self._nfinished < self._npassedout or bool(self._ready_nodes) + + def __bool__(self): + return self.is_active() + + def done(self, *nodes): + """Marks a set of nodes returned by "get_ready" as processed. + + This method unblocks any successor of each node in *nodes* for being returned + in the future by a call to "get_ready". + + Raises :exec:`ValueError` if any node in *nodes* has already been marked as + processed by a previous call to this method, if a node was not added to the + graph by using "add" or if called without calling "prepare" previously or if + node has not yet been returned by "get_ready". + """ + + if self._ready_nodes is None: + raise ValueError("prepare() must be called first") + + n2i = self._node2info + + for node in nodes: + + # Check if we know about this node (it was added previously using add() + if (nodeinfo := n2i.get(node)) is None: + raise ValueError(f"node {node!r} was not added using add()") + + # If the node has not being returned (marked as ready) previously, inform the user. + stat = nodeinfo.npredecessors + if stat != _NODE_OUT: + if stat >= 0: + raise ValueError( + f"node {node!r} was not passed out (still not ready)" + ) + elif stat == _NODE_DONE: + raise ValueError(f"node {node!r} was already marked done") + else: + assert False, f"node {node!r}: unknown status {stat}" + + # Mark the node as processed + nodeinfo.npredecessors = _NODE_DONE + + # Go to all the successors and reduce the number of predecessors, collecting all the ones + # that are ready to be returned in the next get_ready() call. + for successor in nodeinfo.successors: + successor_info = n2i[successor] + successor_info.npredecessors -= 1 + if successor_info.npredecessors == 0: + self._ready_nodes.append(successor) + self._nfinished += 1 + + def _find_cycle(self): + n2i = self._node2info + stack = [] + itstack = [] + seen = set() + node2stacki = {} + + for node in n2i: + if node in seen: + continue + + while True: + if node in seen: + # If we have seen already the node and is in the + # current stack we have found a cycle. + if node in node2stacki: + return stack[node2stacki[node] :] + [node] + # else go on to get next successor + else: + seen.add(node) + itstack.append(iter(n2i[node].successors).__next__) + node2stacki[node] = len(stack) + stack.append(node) + + # Backtrack to the topmost stack entry with + # at least another successor. + while stack: + try: + node = itstack[-1]() + break + except StopIteration: + del node2stacki[stack.pop()] + itstack.pop() + else: + break + return None + + def static_order(self): + """Returns an iterable of nodes in a topological order. + + The particular order that is returned may depend on the specific + order in which the items were inserted in the graph. + + Using this method does not require to call "prepare" or "done". If any + cycle is detected, :exc:`CycleError` will be raised. + """ + self.prepare() + while self.is_active(): + node_group = self.get_ready() + yield from node_group + self.done(*node_group) diff --git a/Lib/gzip.py b/Lib/gzip.py index e422773b3edfb7..11a5f41d56679b 100644 --- a/Lib/gzip.py +++ b/Lib/gzip.py @@ -516,7 +516,7 @@ def _add_read_data(self, data): def _read_eof(self): # We've read to the end of the file - # We check the that the computed CRC and size of the + # We check that the computed CRC and size of the # uncompressed data matches the stored values. Note that the size # stored is the true file size mod 2**32. crc32, isize = struct.unpack("\s]* # bare value ) - (?:\s*,)* # possibly followed by a comma + \s* # possibly followed by a space )?(?:\s|/(?!>))* )* )? diff --git a/Lib/http/client.py b/Lib/http/client.py index 019380a7203181..975292505836e1 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -147,6 +147,10 @@ # _is_allowed_url_pchars_re = re.compile(r"^[/!$&'()*+,;=:@%a-zA-Z0-9._~-]+$") # We are more lenient for assumed real world compatibility purposes. +# These characters are not allowed within HTTP method names +# to prevent http header injection. +_contains_disallowed_method_pchar_re = re.compile('[\x00-\x1f]') + # We always set the Content-Length header for these methods because some # servers will otherwise respond with a 411 _METHODS_EXPECTING_BODY = {'PATCH', 'POST', 'PUT'} @@ -197,15 +201,11 @@ def getallmatchingheaders(self, name): lst.append(line) return lst -def parse_headers(fp, _class=HTTPMessage): - """Parses only RFC2822 headers from a file pointer. - - email Parser wants to see strings rather than bytes. - But a TextIOWrapper around self.rfile would buffer too many bytes - from the stream, bytes which we later need to read as bytes. - So we read the correct bytes here, as bytes, for email Parser - to parse. +def _read_headers(fp): + """Reads potential header lines into a list from a file pointer. + Length of line is limited by _MAXLINE, and number of + headers is limited by _MAXHEADERS. """ headers = [] while True: @@ -217,6 +217,19 @@ def parse_headers(fp, _class=HTTPMessage): raise HTTPException("got more than %d headers" % _MAXHEADERS) if line in (b'\r\n', b'\n', b''): break + return headers + +def parse_headers(fp, _class=HTTPMessage): + """Parses only RFC2822 headers from a file pointer. + + email Parser wants to see strings rather than bytes. + But a TextIOWrapper around self.rfile would buffer too many bytes + from the stream, bytes which we later need to read as bytes. + So we read the correct bytes here, as bytes, for email Parser + to parse. + + """ + headers = _read_headers(fp) hstring = b''.join(headers).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) @@ -304,15 +317,10 @@ def begin(self): if status != CONTINUE: break # skip the header from the 100 response - while True: - skip = self.fp.readline(_MAXLINE + 1) - if len(skip) > _MAXLINE: - raise LineTooLong("header line") - skip = skip.strip() - if not skip: - break - if self.debuglevel > 0: - print("header:", skip) + skipped_headers = _read_headers(self.fp) + if self.debuglevel > 0: + print("headers:", skipped_headers) + del skipped_headers self.code = self.status = status self.reason = reason.strip() @@ -345,9 +353,6 @@ def begin(self): # NOTE: RFC 2616, S4.4, #3 says we ignore this if tr_enc is "chunked" self.length = None length = self.headers.get("content-length") - - # are we using the chunked-style of transfer encoding? - tr_enc = self.headers.get("transfer-encoding") if length and not self.chunked: try: self.length = int(length) @@ -842,7 +847,7 @@ def set_tunnel(self, host, port=None, headers=None): the endpoint passed to `set_tunnel`. This done by sending an HTTP CONNECT request to the proxy server when the connection is established. - This method must be called before the HTML connection has been + This method must be called before the HTTP connection has been established. The headers argument should be a mapping of extra HTTP headers to send @@ -882,23 +887,24 @@ def set_debuglevel(self, level): self.debuglevel = level def _tunnel(self): - connect_str = "CONNECT %s:%d HTTP/1.0\r\n" % (self._tunnel_host, - self._tunnel_port) - connect_bytes = connect_str.encode("ascii") - self.send(connect_bytes) + connect = b"CONNECT %s:%d HTTP/1.0\r\n" % ( + self._tunnel_host.encode("ascii"), self._tunnel_port) + headers = [connect] for header, value in self._tunnel_headers.items(): - header_str = "%s: %s\r\n" % (header, value) - header_bytes = header_str.encode("latin-1") - self.send(header_bytes) - self.send(b'\r\n') + headers.append(f"{header}: {value}\r\n".encode("latin-1")) + headers.append(b"\r\n") + # Making a single send() call instead of one per line encourages + # the host OS to use a more optimal packet size instead of + # potentially emitting a series of small packets. + self.send(b"".join(headers)) + del headers response = self.response_class(self.sock, method=self._method) (version, code, message) = response._read_status() if code != http.HTTPStatus.OK: self.close() - raise OSError("Tunnel connection failed: %d %s" % (code, - message.strip())) + raise OSError(f"Tunnel connection failed: {code} {message.strip()}") while True: line = response.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -1087,6 +1093,8 @@ def putrequest(self, method, url, skip_host=False, else: raise CannotSendRequest(self.__state) + self._validate_method(method) + # Save the method for use later in the response phase self._method = method @@ -1177,6 +1185,15 @@ def _encode_request(self, request): # ASCII also helps prevent CVE-2019-9740. return request.encode('ascii') + def _validate_method(self, method): + """Validate a method name for putrequest.""" + # prevent http header injection + match = _contains_disallowed_method_pchar_re.search(method) + if match: + raise ValueError( + f"method can't contain control characters. {method!r} " + f"(found at least {match.group()!r})") + def _validate_path(self, url): """Validate a url for putrequest.""" # Prevent CVE-2019-9740. diff --git a/Lib/http/server.py b/Lib/http/server.py index fa204fbc15e3d7..d7cce20432ead4 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -689,6 +689,7 @@ def send_head(self): parts[3], parts[4]) new_url = urllib.parse.urlunsplit(new_parts) self.send_header("Location", new_url) + self.send_header("Content-Length", "0") self.end_headers() return None for index in "index.html", "index.htm": @@ -1123,12 +1124,7 @@ def run_cgi(self): referer = self.headers.get('referer') if referer: env['HTTP_REFERER'] = referer - accept = [] - for line in self.headers.getallmatchingheaders('accept'): - if line[:1] in "\t\n\r ": - accept.append(line.strip()) - else: - accept = accept + line[7:].split(',') + accept = self.headers.get_all('accept', ()) env['HTTP_ACCEPT'] = ','.join(accept) ua = self.headers.get('user-agent') if ua: diff --git a/Lib/idlelib/Icons/README.txt b/Lib/idlelib/Icons/README.txt index 8b471629ecb3ea..d91c4d5d8d8cfa 100644 --- a/Lib/idlelib/Icons/README.txt +++ b/Lib/idlelib/Icons/README.txt @@ -7,3 +7,7 @@ https://www.doxdesk.com/software/py/pyicons.html Various different formats and sizes are available at this GitHub Pull Request: https://github.com/python/cpython/pull/17473 + +The idle.ico file was created with ImageMagick: + + $ convert idle_16.png idle_32.png idle_48.png idle_256.png idle.ico diff --git a/Lib/idlelib/Icons/idle.ico b/Lib/idlelib/Icons/idle.ico index 3357aef14888c5..2aa9a8300d9e29 100644 Binary files a/Lib/idlelib/Icons/idle.ico and b/Lib/idlelib/Icons/idle.ico differ diff --git a/Lib/idlelib/NEWS.txt b/Lib/idlelib/NEWS.txt index 46b15234a19c63..f68e144d256100 100644 --- a/Lib/idlelib/NEWS.txt +++ b/Lib/idlelib/NEWS.txt @@ -1,7 +1,67 @@ +What's New in IDLE 3.9.z +(since 3.9.0) +========================= + + +bpo-43283: Document why printing to IDLE's Shell is often slower than +printing to a system terminal and that it can be made faster by +pre-formatting a single string before printing. + +bpo-23544: Disable Debug=>Stack Viewer when user code is running or +Debugger is active, to prevent hang or crash. Patch by Zackery Spytz. + +bpo-43008: Make IDLE invoke :func:`sys.excepthook` in normal, +2-process mode. Patch by Ken Hilton. + +bpo-33065: Fix problem debugging user classes with __repr__ method. + +bpo-32631: Finish zzdummy example extension module: make menu entries +work; add docstrings and tests with 100% coverage. + +bpo-42508: Keep IDLE running on macOS. Remove obsolete workaround +that prevented running files with shortcuts when using new universal2 +installers built on macOS 11. + +bpo-42426: Fix reporting offset of the RE error in searchengine. + +bpo-42416: Get docstrings for IDLE calltips more often +by using inspect.getdoc. + +bpo-33987: Mostly finish using ttk widgets, mainly for editor, +settings, and searches. Some patches by Mark Roseman. + +bpo-41775: Make 'IDLE Shell' the shell title. + +bpo-35764: Rewrite the Calltips doc section. + +bpo-40181: In calltips, stop reminding that '/' marks the end of +positional-only arguments. + + What's New in IDLE 3.9.0 (since 3.8.0) Released on 2020-10-05? ====================================== +bpo-41468: Improve IDLE run crash error message (which users should +never see). + +bpo-41373: Save files loaded with no line ending, as when blank, or +different line endings, by setting its line ending to the system +default. Fix regression in 3.8.4 and 3.9.0b4. + +bpo-41300: Save files with non-ascii chars. Fix regression in +3.9.0b4 and 3.8.4. + +bpo-37765: Add keywords to module name completion list. Rewrite +Completions section of IDLE doc. + +bpo-41152: The encoding of ``stdin``, ``stdout`` and ``stderr`` in IDLE +is now always UTF-8. + +bpo-41144: Make Open Module open a special module such as os.path. + +bpo-40723: Make test_idle pass when run after import. +Patch by Florian Dahlitz. bpo-38689: IDLE will no longer freeze when inspect.signature fails when fetching a calltip. @@ -9,8 +69,9 @@ when fetching a calltip. bpo-27115: For 'Go to Line', use a Query entry box subclass with IDLE standard behavior and improved error checking. -bpo-39885: Since clicking to get an IDLE context menu moves the -cursor, any text selection should be and now is cleared. +bpo-39885: When a context menu is invoked by right-clicking outside +of a selection, clear the selection and move the cursor. Cut and +Copy require that the click be within the selection. bpo-39852: Edit "Go to line" now clears any selection, preventing accidental deletion. It also updates Ln and Col on the status bar. @@ -48,9 +109,9 @@ bpo-38636: Fix IDLE Format menu tab toggle and file indent width. These functions (default shortcuts Alt-T and Alt-U) were mistakenly disabled in 3.7.5 and 3.8.0. -bpo-4360: Add an option to toggle IDLE's cursor blink for shell, +bpo-4630: Add an option to toggle IDLE's cursor blink for shell, editor, and output windows. See Settings, General, Window Preferences, -Cursor Blink. Patch by Zachary Spytz. +Cursor Blink. Patch by Zackery Spytz. bpo-26353: Stop adding newline when saving an IDLE shell window. diff --git a/Lib/idlelib/autocomplete.py b/Lib/idlelib/autocomplete.py index c623d45a153423..e1e9e17311eda1 100644 --- a/Lib/idlelib/autocomplete.py +++ b/Lib/idlelib/autocomplete.py @@ -4,6 +4,7 @@ pop up a list of candidates. """ import __main__ +import keyword import os import string import sys @@ -171,10 +172,13 @@ def fetch_completions(self, what, mode): (what, mode), {}) else: if mode == ATTRS: - if what == "": + if what == "": # Main module names. namespace = {**__main__.__builtins__.__dict__, **__main__.__dict__} bigl = eval("dir()", namespace) + kwds = (s for s in keyword.kwlist + if s not in {'True', 'False', 'None'}) + bigl.extend(kwds) bigl.sort() if "__all__" in bigl: smalll = sorted(eval("__all__", namespace)) diff --git a/Lib/idlelib/autocomplete_w.py b/Lib/idlelib/autocomplete_w.py index fe7a6be83d586b..21b8a2472dd3f4 100644 --- a/Lib/idlelib/autocomplete_w.py +++ b/Lib/idlelib/autocomplete_w.py @@ -203,6 +203,7 @@ def show_window(self, comp_lists, index, complete, mode, userWantsWin): scrollbar.config(command=listbox.yview) scrollbar.pack(side=RIGHT, fill=Y) listbox.pack(side=LEFT, fill=BOTH, expand=True) + acw.update_idletasks() # Need for tk8.6.8 on macOS: #40128. acw.lift() # work around bug in Tk 8.5.18+ (issue #24570) # Initialize the listbox selection @@ -239,31 +240,46 @@ def winconfig_event(self, event): self.is_configuring = True if not self.is_active(): return - # Position the completion list window - text = self.widget - text.see(self.startindex) - x, y, cx, cy = text.bbox(self.startindex) - acw = self.autocompletewindow - acw.update() - acw_width, acw_height = acw.winfo_width(), acw.winfo_height() - text_width, text_height = text.winfo_width(), text.winfo_height() - new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width)) - new_y = text.winfo_rooty() + y - if (text_height - (y + cy) >= acw_height # enough height below - or y < acw_height): # not enough height above - # place acw below current line - new_y += cy - else: - # place acw above current line - new_y -= acw_height - acw.wm_geometry("+%d+%d" % (new_x, new_y)) - acw.update_idletasks() + + # Since the event may occur after the completion window is gone, + # catch potential TclError exceptions when accessing acw. See: bpo-41611. + try: + # Position the completion list window + text = self.widget + text.see(self.startindex) + x, y, cx, cy = text.bbox(self.startindex) + acw = self.autocompletewindow + if platform.system().startswith('Windows'): + # On Windows an update() call is needed for the completion + # list window to be created, so that we can fetch its width + # and height. However, this is not needed on other platforms + # (tested on Ubuntu and macOS) but at one point began + # causing freezes on macOS. See issues 37849 and 41611. + acw.update() + acw_width, acw_height = acw.winfo_width(), acw.winfo_height() + text_width, text_height = text.winfo_width(), text.winfo_height() + new_x = text.winfo_rootx() + min(x, max(0, text_width - acw_width)) + new_y = text.winfo_rooty() + y + if (text_height - (y + cy) >= acw_height # enough height below + or y < acw_height): # not enough height above + # place acw below current line + new_y += cy + else: + # place acw above current line + new_y -= acw_height + acw.wm_geometry("+%d+%d" % (new_x, new_y)) + acw.update_idletasks() + except TclError: + pass if platform.system().startswith('Windows'): - # See issue 15786. When on Windows platform, Tk will misbehave + # See issue 15786. When on Windows platform, Tk will misbehave # to call winconfig_event multiple times, we need to prevent this, # otherwise mouse button double click will not be able to used. - acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + try: + acw.unbind(WINCONFIG_SEQUENCE, self.winconfigid) + except TclError: + pass self.winconfigid = None self.is_configuring = False diff --git a/Lib/idlelib/calltip.py b/Lib/idlelib/calltip.py index d4092c7847186b..40bc5a0ad798fe 100644 --- a/Lib/idlelib/calltip.py +++ b/Lib/idlelib/calltip.py @@ -55,18 +55,50 @@ def refresh_calltip_event(self, event): self.open_calltip(False) def open_calltip(self, evalfuncs): - self.remove_calltip_window() + """Maybe close an existing calltip and maybe open a new calltip. + Called from (force_open|try_open|refresh)_calltip_event functions. + """ hp = HyperParser(self.editwin, "insert") sur_paren = hp.get_surrounding_brackets('(') + + # If not inside parentheses, no calltip. if not sur_paren: + self.remove_calltip_window() return + + # If a calltip is shown for the current parentheses, do + # nothing. + if self.active_calltip: + opener_line, opener_col = map(int, sur_paren[0].split('.')) + if ( + (opener_line, opener_col) == + (self.active_calltip.parenline, self.active_calltip.parencol) + ): + return + hp.set_index(sur_paren[0]) - expression = hp.get_expression() + try: + expression = hp.get_expression() + except ValueError: + expression = None if not expression: + # No expression before the opening parenthesis, e.g. + # because it's in a string or the opener for a tuple: + # Do nothing. return + + # At this point, the current index is after an opening + # parenthesis, in a section of code, preceded by a valid + # expression. If there is a calltip shown, it's not for the + # same index and should be closed. + self.remove_calltip_window() + + # Simple, fast heuristic: If the preceding expression includes + # an opening parenthesis, it likely includes a function call. if not evalfuncs and (expression.find('(') != -1): return + argspec = self.fetch_tip(expression) if not argspec: return @@ -118,7 +150,6 @@ def get_entity(expression): _first_param = re.compile(r'(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" _invalid_method = "invalid method signature" -_argument_positional = " # '/' marks preceding args as positional-only." def get_argspec(ob): '''Return a string describing the signature of a callable object, or ''. @@ -134,6 +165,7 @@ def get_argspec(ob): ob_call = ob.__call__ except BaseException: # Buggy user object could raise anything. return '' # No popup for non-callables. + # For Get_argspecTest.test_buggy_getattr_class, CallA() & CallB(). fob = ob_call if isinstance(ob_call, types.MethodType) else ob # Initialize argspec and wrap it to get lines. @@ -146,9 +178,6 @@ def get_argspec(ob): else: argspec = '' - if '/' in argspec and len(argspec) < _MAX_COLS - len(_argument_positional): - # Add explanation TODO remove after 3.7, before 3.9. - argspec += _argument_positional if isinstance(fob, type) and argspec == '()': # If fob has no argument, use default callable argspec. argspec = _default_callable_argspec @@ -157,10 +186,7 @@ def get_argspec(ob): if len(argspec) > _MAX_COLS else [argspec] if argspec else []) # Augment lines from docstring, if any, and join to get argspec. - if isinstance(ob_call, types.MethodType): - doc = ob_call.__doc__ - else: - doc = getattr(ob, "__doc__", "") + doc = inspect.getdoc(ob) if doc: for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]: line = line.strip() diff --git a/Lib/idlelib/codecontext.py b/Lib/idlelib/codecontext.py index 989b30e5994650..f2f44f5f8d4e61 100644 --- a/Lib/idlelib/codecontext.py +++ b/Lib/idlelib/codecontext.py @@ -7,11 +7,14 @@ enclosing block. The number of hint lines is determined by the maxlines variable in the codecontext section of config-extensions.def. Lines which do not open blocks are not shown in the context hints pane. + +For EditorWindows, <> is bound to CodeContext(self). +toggle_code_context_event. """ import re from sys import maxsize as INFINITY -import tkinter +from tkinter import Frame, Text, TclError from tkinter.constants import NSEW, SUNKEN from idlelib.config import idleConf @@ -83,7 +86,7 @@ def __del__(self): if self.t1 is not None: try: self.text.after_cancel(self.t1) - except tkinter.TclError: # pragma: no cover + except TclError: # pragma: no cover pass self.t1 = None @@ -111,7 +114,7 @@ def toggle_code_context_event(self, event=None): padx += widget.tk.getint(info['padx']) padx += widget.tk.getint(widget.cget('padx')) border += widget.tk.getint(widget.cget('border')) - context = self.context = tkinter.Text( + context = self.context = Text( self.editwin.text_frame, height=1, width=1, # Don't request more than we get. @@ -127,7 +130,7 @@ def toggle_code_context_event(self, event=None): line_number_colors = idleConf.GetHighlight(idleConf.CurrentTheme(), 'linenumber') - self.cell00 = tkinter.Frame(self.editwin.text_frame, + self.cell00 = Frame(self.editwin.text_frame, bg=line_number_colors['background']) self.cell00.grid(row=0, column=0, sticky=NSEW) menu_status = 'Hide' @@ -139,7 +142,7 @@ def toggle_code_context_event(self, event=None): self.text.after_cancel(self.t1) self._reset() menu_status = 'Show' - self.editwin.update_menu_label(menu='options', index='* Code Context', + self.editwin.update_menu_label(menu='options', index='*ode*ontext', label=f'{menu_status} Code Context') return "break" @@ -221,7 +224,7 @@ def jumptoline(self, event=None): """ try: self.context.index("sel.first") - except tkinter.TclError: + except TclError: lines = len(self.info) if lines == 1: # No context lines are showing. newtop = 1 diff --git a/Lib/idlelib/colorizer.py b/Lib/idlelib/colorizer.py index db1266fed3b691..0aae1778a580c0 100644 --- a/Lib/idlelib/colorizer.py +++ b/Lib/idlelib/colorizer.py @@ -8,15 +8,17 @@ DEBUG = False + def any(name, alternates): "Return a named group pattern matching list of alternates." return "(?P<%s>" % name + "|".join(alternates) + ")" + def make_pat(): kw = r"\b" + any("KEYWORD", keyword.kwlist) + r"\b" builtinlist = [str(name) for name in dir(builtins) - if not name.startswith('_') and \ - name not in keyword.kwlist] + if not name.startswith('_') and + name not in keyword.kwlist] builtin = r"([^.'\"\\#]\b|^)" + any("BUILTIN", builtinlist) + r"\b" comment = any("COMMENT", [r"#[^\n]*"]) stringprefix = r"(?i:r|u|f|fr|rf|b|br|rb)?" @@ -25,12 +27,14 @@ def make_pat(): sq3string = stringprefix + r"'''[^'\\]*((\\.|'(?!''))[^'\\]*)*(''')?" dq3string = stringprefix + r'"""[^"\\]*((\\.|"(?!""))[^"\\]*)*(""")?' string = any("STRING", [sq3string, dq3string, sqstring, dqstring]) - return kw + "|" + builtin + "|" + comment + "|" + string +\ - "|" + any("SYNC", [r"\n"]) + return (kw + "|" + builtin + "|" + comment + "|" + string + + "|" + any("SYNC", [r"\n"])) + prog = re.compile(make_pat(), re.S) idprog = re.compile(r"\s+(\w+)", re.S) + def color_config(text): """Set color options of Text widget. @@ -49,7 +53,7 @@ def color_config(text): selectforeground=select_colors['foreground'], selectbackground=select_colors['background'], inactiveselectbackground=select_colors['background'], # new in 8.5 - ) + ) class ColorDelegator(Delegator): @@ -120,14 +124,17 @@ def LoadTagDefs(self): "BUILTIN": idleConf.GetHighlight(theme, "builtin"), "STRING": idleConf.GetHighlight(theme, "string"), "DEFINITION": idleConf.GetHighlight(theme, "definition"), - "SYNC": {'background':None,'foreground':None}, - "TODO": {'background':None,'foreground':None}, + "SYNC": {'background': None, 'foreground': None}, + "TODO": {'background': None, 'foreground': None}, "ERROR": idleConf.GetHighlight(theme, "error"), - # The following is used by ReplaceDialog: + # "hit" is used by ReplaceDialog to mark matches. It shouldn't be changed by Colorizer, but + # that currently isn't technically possible. This should be moved elsewhere in the future + # when fixing the "hit" tag's visibility, or when the replace dialog is replaced with a + # non-modal alternative. "hit": idleConf.GetHighlight(theme, "hit"), } - if DEBUG: print('tagdefs',self.tagdefs) + if DEBUG: print('tagdefs', self.tagdefs) def insert(self, index, chars, tags=None): "Insert chars into widget at index and mark for colorizing." @@ -184,8 +191,8 @@ def toggle_colorize_event(self, event=None): if self.allow_colorizing and not self.colorizing: self.after_id = self.after(1, self.recolorize) if DEBUG: - print("auto colorizing turned",\ - self.allow_colorizing and "on" or "off") + print("auto colorizing turned", + "on" if self.allow_colorizing else "off") return "break" def recolorize(self): @@ -232,10 +239,7 @@ def recolorize_main(self): head, tail = item self.tag_remove("SYNC", head, tail) item = self.tag_prevrange("SYNC", head) - if item: - head = item[1] - else: - head = "1.0" + head = item[1] if item else "1.0" chars = "" next = head @@ -307,7 +311,7 @@ def _color_delegator(parent): # htest # "elif False: print(0)\n" "else: float(None)\n" "if iF + If + IF: 'keyword matching must respect case'\n" - "if'': x or'' # valid string-keyword no-space combinations\n" + "if'': x or'' # valid keyword-string no-space combinations\n" "async def f(): await g()\n" "# All valid prefixes for unicode and byte strings should be colored.\n" "'x', '''x''', \"x\", \"\"\"x\"\"\"\n" diff --git a/Lib/idlelib/config_key.py b/Lib/idlelib/config_key.py index 7510aa9f3d8786..9ca3a156f4b97f 100644 --- a/Lib/idlelib/config_key.py +++ b/Lib/idlelib/config_key.py @@ -4,6 +4,7 @@ from tkinter import Toplevel, Listbox, StringVar, TclError from tkinter.ttk import Frame, Button, Checkbutton, Entry, Label, Scrollbar from tkinter import messagebox +from tkinter.simpledialog import _setup_dialog import string import sys @@ -63,6 +64,7 @@ def __init__(self, parent, title, action, current_key_sequences, self.resizable(height=False, width=False) self.title(title) self.transient(parent) + _setup_dialog(self) self.grab_set() self.protocol("WM_DELETE_WINDOW", self.cancel) self.parent = parent diff --git a/Lib/idlelib/configdialog.py b/Lib/idlelib/configdialog.py index 82596498d34611..6d0893680274b3 100644 --- a/Lib/idlelib/configdialog.py +++ b/Lib/idlelib/configdialog.py @@ -15,11 +15,12 @@ StringVar, BooleanVar, IntVar, TRUE, FALSE, TOP, BOTTOM, RIGHT, LEFT, SOLID, GROOVE, NONE, BOTH, X, Y, W, E, EW, NS, NSEW, NW, - HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END) + HORIZONTAL, VERTICAL, ANCHOR, ACTIVE, END, TclError) from tkinter.ttk import (Frame, LabelFrame, Button, Checkbutton, Entry, Label, - OptionMenu, Notebook, Radiobutton, Scrollbar, Style) -import tkinter.colorchooser as tkColorChooser -import tkinter.font as tkFont + OptionMenu, Notebook, Radiobutton, Scrollbar, Style, + Spinbox, Combobox) +from tkinter import colorchooser +import tkinter.font as tkfont from tkinter import messagebox from idlelib.config import idleConf, ConfigChanges @@ -67,7 +68,6 @@ def __init__(self, parent, title='', *, _htest=False, _utest=False): if not _utest: self.withdraw() - self.configure(borderwidth=5) self.title(title or 'IDLE Preferences') x = parent.winfo_rootx() + 20 y = parent.winfo_rooty() + (30 if not _htest else 150) @@ -97,28 +97,35 @@ def create_widgets(self): """Create and place widgets for tabbed dialog. Widgets Bound to self: + frame: encloses all other widgets note: Notebook highpage: HighPage fontpage: FontPage keyspage: KeysPage - genpage: GenPage - extpage: self.create_page_extensions + winpage: WinPage + shedpage: ShedPage + extpage: ExtPage Methods: create_action_buttons load_configs: Load pages except for extensions. activate_config_changes: Tell editors to reload. """ - self.note = note = Notebook(self) - self.highpage = HighPage(note) + self.frame = frame = Frame(self, padding="5px") + self.frame.grid(sticky="nwes") + self.note = note = Notebook(frame) + self.extpage = ExtPage(note) + self.highpage = HighPage(note, self.extpage) self.fontpage = FontPage(note, self.highpage) - self.keyspage = KeysPage(note) - self.genpage = GenPage(note) - self.extpage = self.create_page_extensions() + self.keyspage = KeysPage(note, self.extpage) + self.winpage = WinPage(note) + self.shedpage = ShedPage(note) + note.add(self.fontpage, text='Fonts/Tabs') note.add(self.highpage, text='Highlights') note.add(self.keyspage, text=' Keys ') - note.add(self.genpage, text=' General ') + note.add(self.winpage, text=' Windows ') + note.add(self.shedpage, text=' Shell/Ed ') note.add(self.extpage, text='Extensions') note.enable_traversal() note.pack(side=TOP, expand=TRUE, fill=BOTH) @@ -148,7 +155,7 @@ def create_action_buttons(self): padding_args = {} else: padding_args = {'padding': (6, 3)} - outer = Frame(self, padding=2) + outer = Frame(self.frame, padding=2) buttons_frame = Frame(outer, padding=2) self.buttons = {} for txt, cmd in ( @@ -165,26 +172,15 @@ def create_action_buttons(self): return outer def ok(self): - """Apply config changes, then dismiss dialog. - - Methods: - apply - destroy: inherited - """ + """Apply config changes, then dismiss dialog.""" self.apply() self.destroy() def apply(self): - """Apply config changes and leave dialog open. - - Methods: - deactivate_current_config - save_all_changed_extensions - activate_config_changes - """ + """Apply config changes and leave dialog open.""" self.deactivate_current_config() changes.save_all() - self.save_all_changed_extensions() + self.extpage.save_all_changed_extensions() self.activate_config_changes() def cancel(self): @@ -242,190 +238,6 @@ def activate_config_changes(self): for klass in reloadables: klass.reload() - def create_page_extensions(self): - """Part of the config dialog used for configuring IDLE extensions. - - This code is generic - it works for any and all IDLE extensions. - - IDLE extensions save their configuration options using idleConf. - This code reads the current configuration using idleConf, supplies a - GUI interface to change the configuration values, and saves the - changes using idleConf. - - Not all changes take effect immediately - some may require restarting IDLE. - This depends on each extension's implementation. - - All values are treated as text, and it is up to the user to supply - reasonable values. The only exception to this are the 'enable*' options, - which are boolean, and can be toggled with a True/False button. - - Methods: - load_extensions: - extension_selected: Handle selection from list. - create_extension_frame: Hold widgets for one extension. - set_extension_value: Set in userCfg['extensions']. - save_all_changed_extensions: Call extension page Save(). - """ - parent = self.parent - frame = Frame(self.note) - self.ext_defaultCfg = idleConf.defaultCfg['extensions'] - self.ext_userCfg = idleConf.userCfg['extensions'] - self.is_int = self.register(is_int) - self.load_extensions() - # Create widgets - a listbox shows all available extensions, with the - # controls for the extension selected in the listbox to the right. - self.extension_names = StringVar(self) - frame.rowconfigure(0, weight=1) - frame.columnconfigure(2, weight=1) - self.extension_list = Listbox(frame, listvariable=self.extension_names, - selectmode='browse') - self.extension_list.bind('<>', self.extension_selected) - scroll = Scrollbar(frame, command=self.extension_list.yview) - self.extension_list.yscrollcommand=scroll.set - self.details_frame = LabelFrame(frame, width=250, height=250) - self.extension_list.grid(column=0, row=0, sticky='nws') - scroll.grid(column=1, row=0, sticky='ns') - self.details_frame.grid(column=2, row=0, sticky='nsew', padx=[10, 0]) - frame.configure(padding=10) - self.config_frame = {} - self.current_extension = None - - self.outerframe = self # TEMPORARY - self.tabbed_page_set = self.extension_list # TEMPORARY - - # Create the frame holding controls for each extension. - ext_names = '' - for ext_name in sorted(self.extensions): - self.create_extension_frame(ext_name) - ext_names = ext_names + '{' + ext_name + '} ' - self.extension_names.set(ext_names) - self.extension_list.selection_set(0) - self.extension_selected(None) - - return frame - - def load_extensions(self): - "Fill self.extensions with data from the default and user configs." - self.extensions = {} - for ext_name in idleConf.GetExtensions(active_only=False): - # Former built-in extensions are already filtered out. - self.extensions[ext_name] = [] - - for ext_name in self.extensions: - opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) - - # Bring 'enable' options to the beginning of the list. - enables = [opt_name for opt_name in opt_list - if opt_name.startswith('enable')] - for opt_name in enables: - opt_list.remove(opt_name) - opt_list = enables + opt_list - - for opt_name in opt_list: - def_str = self.ext_defaultCfg.Get( - ext_name, opt_name, raw=True) - try: - def_obj = {'True':True, 'False':False}[def_str] - opt_type = 'bool' - except KeyError: - try: - def_obj = int(def_str) - opt_type = 'int' - except ValueError: - def_obj = def_str - opt_type = None - try: - value = self.ext_userCfg.Get( - ext_name, opt_name, type=opt_type, raw=True, - default=def_obj) - except ValueError: # Need this until .Get fixed. - value = def_obj # Bad values overwritten by entry. - var = StringVar(self) - var.set(str(value)) - - self.extensions[ext_name].append({'name': opt_name, - 'type': opt_type, - 'default': def_str, - 'value': value, - 'var': var, - }) - - def extension_selected(self, event): - "Handle selection of an extension from the list." - newsel = self.extension_list.curselection() - if newsel: - newsel = self.extension_list.get(newsel) - if newsel is None or newsel != self.current_extension: - if self.current_extension: - self.details_frame.config(text='') - self.config_frame[self.current_extension].grid_forget() - self.current_extension = None - if newsel: - self.details_frame.config(text=newsel) - self.config_frame[newsel].grid(column=0, row=0, sticky='nsew') - self.current_extension = newsel - - def create_extension_frame(self, ext_name): - """Create a frame holding the widgets to configure one extension""" - f = VerticalScrolledFrame(self.details_frame, height=250, width=250) - self.config_frame[ext_name] = f - entry_area = f.interior - # Create an entry for each configuration option. - for row, opt in enumerate(self.extensions[ext_name]): - # Create a row with a label and entry/checkbutton. - label = Label(entry_area, text=opt['name']) - label.grid(row=row, column=0, sticky=NW) - var = opt['var'] - if opt['type'] == 'bool': - Checkbutton(entry_area, variable=var, - onvalue='True', offvalue='False', width=8 - ).grid(row=row, column=1, sticky=W, padx=7) - elif opt['type'] == 'int': - Entry(entry_area, textvariable=var, validate='key', - validatecommand=(self.is_int, '%P'), width=10 - ).grid(row=row, column=1, sticky=NSEW, padx=7) - - else: # type == 'str' - # Limit size to fit non-expanding space with larger font. - Entry(entry_area, textvariable=var, width=15 - ).grid(row=row, column=1, sticky=NSEW, padx=7) - return - - def set_extension_value(self, section, opt): - """Return True if the configuration was added or changed. - - If the value is the same as the default, then remove it - from user config file. - """ - name = opt['name'] - default = opt['default'] - value = opt['var'].get().strip() or default - opt['var'].set(value) - # if self.defaultCfg.has_section(section): - # Currently, always true; if not, indent to return. - if (value == default): - return self.ext_userCfg.RemoveOption(section, name) - # Set the option. - return self.ext_userCfg.SetOption(section, name, value) - - def save_all_changed_extensions(self): - """Save configuration changes to the user config file. - - Attributes accessed: - extensions - - Methods: - set_extension_value - """ - has_changes = False - for ext_name in self.extensions: - options = self.extensions[ext_name] - for opt in options: - if self.set_extension_value(ext_name, opt): - has_changes = True - if has_changes: - self.ext_userCfg.Save() - # class TabPage(Frame): # A template for Page classes. # def __init__(self, master): @@ -478,12 +290,11 @@ class FontPage(Frame): def __init__(self, master, highpage): super().__init__(master) self.highlight_sample = highpage.highlight_sample - self.create_page_font_tab() + self.create_page_font() self.load_font_cfg() - self.load_tab_cfg() - def create_page_font_tab(self): - """Return frame of widgets for Font/Tabs tab. + def create_page_font(self): + """Return frame of widgets for Font tab. Fonts: Enable users to provisionally change font face, size, or boldness and to see the consequence of proposed choices. Each @@ -507,11 +318,6 @@ def create_page_font_tab(self): Set_samples applies a new font constructed from the font vars to font_sample and to highlight_sample on the highlight page. - Tabs: Enable users to change spaces entered for indent tabs. - Changing indent_scale value with the mouse sets Var space_num, - which invokes the default callback to add an entry to - changes. Load_tab_cfg initializes space_num to default. - Widgets for FontPage(Frame): (*) widgets bound to self frame_font: LabelFrame frame_font_name: Frame @@ -524,23 +330,16 @@ def create_page_font_tab(self): (*)bold_toggle: Checkbutton - font_bold frame_sample: LabelFrame (*)font_sample: Label - frame_indent: LabelFrame - indent_title: Label - (*)indent_scale: Scale - space_num """ self.font_name = tracers.add(StringVar(self), self.var_changed_font) self.font_size = tracers.add(StringVar(self), self.var_changed_font) self.font_bold = tracers.add(BooleanVar(self), self.var_changed_font) - self.space_num = tracers.add(IntVar(self), ('main', 'Indent', 'num-spaces')) # Define frames and widgets. - frame_font = LabelFrame( - self, borderwidth=2, relief=GROOVE, text=' Shell/Editor Font ') - frame_sample = LabelFrame( - self, borderwidth=2, relief=GROOVE, - text=' Font Sample (Editable) ') - frame_indent = LabelFrame( - self, borderwidth=2, relief=GROOVE, text=' Indentation Width ') + frame_font = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Shell/Editor Font ') + frame_sample = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Font Sample (Editable) ') # frame_font. frame_font_name = Frame(frame_font) frame_font_param = Frame(frame_font) @@ -564,13 +363,6 @@ def create_page_font_tab(self): self.font_sample = font_sample_frame.text self.font_sample.config(wrap=NONE, width=1, height=1) self.font_sample.insert(END, font_sample_text) - # frame_indent. - indent_title = Label( - frame_indent, justify=LEFT, - text='Python Standard: 4 Spaces!') - self.indent_scale = Scale( - frame_indent, variable=self.space_num, - orient='horizontal', tickinterval=2, from_=2, to=16) # Grid and pack widgets: self.columnconfigure(1, weight=1) @@ -578,7 +370,6 @@ def create_page_font_tab(self): frame_font.grid(row=0, column=0, padx=5, pady=5) frame_sample.grid(row=0, column=1, rowspan=3, padx=5, pady=5, sticky='nsew') - frame_indent.grid(row=1, column=0, padx=5, pady=5, sticky='ew') # frame_font. frame_font_name.pack(side=TOP, padx=5, pady=5, fill=X) frame_font_param.pack(side=TOP, padx=5, pady=5, fill=X) @@ -590,9 +381,6 @@ def create_page_font_tab(self): self.bold_toggle.pack(side=LEFT, anchor=W, padx=20) # frame_sample. font_sample_frame.pack(expand=TRUE, fill=BOTH) - # frame_indent. - indent_title.pack(side=TOP, anchor=W, padx=5) - self.indent_scale.pack(side=TOP, padx=5, fill=X) def load_font_cfg(self): """Load current configuration settings for the font options. @@ -607,7 +395,7 @@ def load_font_cfg(self): font_bold = configured_font[2]=='bold' # Set sorted no-duplicate editor font selection list and font_name. - fonts = sorted(set(tkFont.families(self))) + fonts = sorted(set(tkfont.families(self))) for font in fonts: self.fontlist.insert(END, font) self.font_name.set(font_name) @@ -661,39 +449,24 @@ def set_samples(self, event=None): Updates font_sample and highlight page highlight_sample. """ font_name = self.font_name.get() - font_weight = tkFont.BOLD if self.font_bold.get() else tkFont.NORMAL + font_weight = tkfont.BOLD if self.font_bold.get() else tkfont.NORMAL new_font = (font_name, self.font_size.get(), font_weight) self.font_sample['font'] = new_font self.highlight_sample['font'] = new_font - def load_tab_cfg(self): - """Load current configuration settings for the tab options. - - Attributes updated: - space_num: Set to value from idleConf. - """ - # Set indent sizes. - space_num = idleConf.GetOption( - 'main', 'Indent', 'num-spaces', default=4, type='int') - self.space_num.set(space_num) - - def var_changed_space_num(self, *params): - "Store change to indentation size." - value = self.space_num.get() - changes.add_option('main', 'Indent', 'num-spaces', value) - class HighPage(Frame): - def __init__(self, master): + def __init__(self, master, extpage): super().__init__(master) - self.cd = master.master + self.extpage = extpage + self.cd = master.winfo_toplevel() self.style = Style(master) self.create_page_highlight() self.load_theme_cfg() def create_page_highlight(self): - """Return frame of widgets for Highlighting tab. + """Return frame of widgets for Highlights tab. Enable users to provisionally change foreground and background colors applied to textual tags. Color mappings are stored in @@ -1098,7 +871,7 @@ def get_color(self): target = self.highlight_target.get() prev_color = self.style.lookup(self.frame_color_set['style'], 'background') - rgbTuplet, color_string = tkColorChooser.askcolor( + rgbTuplet, color_string = colorchooser.askcolor( parent=self, title='Pick new color for : '+target, initialcolor=prev_color) if color_string and (color_string != prev_color): @@ -1337,16 +1110,17 @@ def delete_custom(self): self.builtin_name.set(idleConf.defaultCfg['main'].Get('Theme', 'name')) # User can't back out of these changes, they must be applied now. changes.save_all() - self.cd.save_all_changed_extensions() + self.extpage.save_all_changed_extensions() self.cd.activate_config_changes() self.set_theme_type() class KeysPage(Frame): - def __init__(self, master): + def __init__(self, master, extpage): super().__init__(master) - self.cd = master.master + self.extpage = extpage + self.cd = master.winfo_toplevel() self.create_page_keys() self.load_key_cfg() @@ -1769,19 +1543,19 @@ def delete_custom_keys(self): or idleConf.default_keys()) # User can't back out of these changes, they must be applied now. changes.save_all() - self.cd.save_all_changed_extensions() + self.extpage.save_all_changed_extensions() self.cd.activate_config_changes() self.set_keys_type() -class GenPage(Frame): +class WinPage(Frame): def __init__(self, master): super().__init__(master) self.init_validators() - self.create_page_general() - self.load_general_cfg() + self.create_page_windows() + self.load_windows_cfg() def init_validators(self): digits_or_empty_re = re.compile(r'[0-9]*') @@ -1790,76 +1564,45 @@ def is_digits_or_empty(s): return digits_or_empty_re.fullmatch(s) is not None self.digits_only = (self.register(is_digits_or_empty), '%P',) - def create_page_general(self): - """Return frame of widgets for General tab. - - Enable users to provisionally change general options. Function - load_general_cfg initializes tk variables and helplist using - idleConf. Radiobuttons startup_shell_on and startup_editor_on - set var startup_edit. Radiobuttons save_ask_on and save_auto_on - set var autosave. Entry boxes win_width_int and win_height_int - set var win_width and win_height. Setting var_name invokes the - default callback that adds option to changes. + def create_page_windows(self): + """Return frame of widgets for Windows tab. - Helplist: load_general_cfg loads list user_helplist with - name, position pairs and copies names to listbox helplist. - Clicking a name invokes help_source selected. Clicking - button_helplist_name invokes helplist_item_name, which also - changes user_helplist. These functions all call - set_add_delete_state. All but load call update_help_changes to - rewrite changes['main']['HelpFiles']. + Enable users to provisionally change general window options. + Function load_windows_cfg initializes tk variable idleConf. + Radiobuttons startup_shell_on and startup_editor_on set var + startup_edit. Entry boxes win_width_int and win_height_int set var + win_width and win_height. Setting var_name invokes the default + callback that adds option to changes. - Widgets for GenPage(Frame): (*) widgets bound to self + Widgets for WinPage(Frame): > vars, bound to self frame_window: LabelFrame frame_run: Frame startup_title: Label - (*)startup_editor_on: Radiobutton - startup_edit - (*)startup_shell_on: Radiobutton - startup_edit + startup_editor_on: Radiobutton > startup_edit + startup_shell_on: Radiobutton > startup_edit frame_win_size: Frame win_size_title: Label win_width_title: Label - (*)win_width_int: Entry - win_width + win_width_int: Entry > win_width win_height_title: Label - (*)win_height_int: Entry - win_height - frame_cursor_blink: Frame - cursor_blink_title: Label - (*)cursor_blink_bool: Checkbutton - cursor_blink + win_height_int: Entry > win_height + frame_cursor: Frame + indent_title: Label + indent_chooser: Spinbox (Combobox < 8.5.9) > indent_spaces + blink_on: Checkbutton > cursor_blink frame_autocomplete: Frame auto_wait_title: Label - (*)auto_wait_int: Entry - autocomplete_wait + auto_wait_int: Entry > autocomplete_wait frame_paren1: Frame paren_style_title: Label - (*)paren_style_type: OptionMenu - paren_style + paren_style_type: OptionMenu > paren_style frame_paren2: Frame paren_time_title: Label - (*)paren_flash_time: Entry - flash_delay - (*)bell_on: Checkbutton - paren_bell - frame_editor: LabelFrame - frame_save: Frame - run_save_title: Label - (*)save_ask_on: Radiobutton - autosave - (*)save_auto_on: Radiobutton - autosave + paren_flash_time: Entry > flash_delay + bell_on: Checkbutton > paren_bell frame_format: Frame format_width_title: Label - (*)format_width_int: Entry - format_width - frame_line_numbers_default: Frame - line_numbers_default_title: Label - (*)line_numbers_default_bool: Checkbutton - line_numbers_default - frame_context: Frame - context_title: Label - (*)context_int: Entry - context_lines - frame_shell: LabelFrame - frame_auto_squeeze_min_lines: Frame - auto_squeeze_min_lines_title: Label - (*)auto_squeeze_min_lines_int: Entry - auto_squeeze_min_lines - frame_help: LabelFrame - frame_helplist: Frame - frame_helplist_buttons: Frame - (*)button_helplist_edit - (*)button_helplist_add - (*)button_helplist_remove - (*)helplist: ListBox - scroll_helplist: Scrollbar + format_width_int: Entry > format_width """ # Integer values need StringVar because int('') raises. self.startup_edit = tracers.add( @@ -1868,6 +1611,8 @@ def create_page_general(self): StringVar(self), ('main', 'EditorWindow', 'width')) self.win_height = tracers.add( StringVar(self), ('main', 'EditorWindow', 'height')) + self.indent_spaces = tracers.add( + StringVar(self), ('main', 'Indent', 'num-spaces')) self.cursor_blink = tracers.add( BooleanVar(self), ('main', 'EditorWindow', 'cursor-blink')) self.autocomplete_wait = tracers.add( @@ -1878,31 +1623,13 @@ def create_page_general(self): StringVar(self), ('extensions', 'ParenMatch', 'flash-delay')) self.paren_bell = tracers.add( BooleanVar(self), ('extensions', 'ParenMatch', 'bell')) - - self.auto_squeeze_min_lines = tracers.add( - StringVar(self), ('main', 'PyShell', 'auto-squeeze-min-lines')) - - self.autosave = tracers.add( - IntVar(self), ('main', 'General', 'autosave')) self.format_width = tracers.add( StringVar(self), ('extensions', 'FormatParagraph', 'max-width')) - self.line_numbers_default = tracers.add( - BooleanVar(self), - ('main', 'EditorWindow', 'line-numbers-default')) - self.context_lines = tracers.add( - StringVar(self), ('extensions', 'CodeContext', 'maxlines')) # Create widgets: - # Section frames. frame_window = LabelFrame(self, borderwidth=2, relief=GROOVE, text=' Window Preferences') - frame_editor = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Editor Preferences') - frame_shell = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Shell Preferences') - frame_help = LabelFrame(self, borderwidth=2, relief=GROOVE, - text=' Additional Help Sources ') - # Frame_window. + frame_run = Frame(frame_window, borderwidth=0) startup_title = Label(frame_run, text='At Startup') self.startup_editor_on = Radiobutton( @@ -1926,19 +1653,28 @@ def create_page_general(self): validatecommand=self.digits_only, validate='key', ) - frame_cursor_blink = Frame(frame_window, borderwidth=0) - cursor_blink_title = Label(frame_cursor_blink, text='Cursor Blink') - self.cursor_blink_bool = Checkbutton(frame_cursor_blink, - variable=self.cursor_blink, width=1) + frame_cursor = Frame(frame_window, borderwidth=0) + indent_title = Label(frame_cursor, + text='Indent spaces (4 is standard)') + try: + self.indent_chooser = Spinbox( + frame_cursor, textvariable=self.indent_spaces, + from_=1, to=10, width=2, + validatecommand=self.digits_only, validate='key') + except TclError: + self.indent_chooser = Combobox( + frame_cursor, textvariable=self.indent_spaces, + state="readonly", values=list(range(1,11)), width=3) + cursor_blink_title = Label(frame_cursor, text='Cursor Blink') + self.cursor_blink_bool = Checkbutton(frame_cursor, text="Cursor blink", + variable=self.cursor_blink) frame_autocomplete = Frame(frame_window, borderwidth=0,) auto_wait_title = Label(frame_autocomplete, - text='Completions Popup Wait (milliseconds)') - self.auto_wait_int = Entry(frame_autocomplete, width=6, - textvariable=self.autocomplete_wait, - validatecommand=self.digits_only, - validate='key', - ) + text='Completions Popup Wait (milliseconds)') + self.auto_wait_int = Entry( + frame_autocomplete, textvariable=self.autocomplete_wait, + width=6, validatecommand=self.digits_only, validate='key') frame_paren1 = Frame(frame_window, borderwidth=0) paren_style_title = Label(frame_paren1, text='Paren Match Style') @@ -1950,79 +1686,20 @@ def create_page_general(self): frame_paren2, text='Time Match Displayed (milliseconds)\n' '(0 is until next input)') self.paren_flash_time = Entry( - frame_paren2, textvariable=self.flash_delay, width=6) + frame_paren2, textvariable=self.flash_delay, width=6, + validatecommand=self.digits_only, validate='key') self.bell_on = Checkbutton( frame_paren2, text="Bell on Mismatch", variable=self.paren_bell) - - # Frame_editor. - frame_save = Frame(frame_editor, borderwidth=0) - run_save_title = Label(frame_save, text='At Start of Run (F5) ') - self.save_ask_on = Radiobutton( - frame_save, variable=self.autosave, value=0, - text="Prompt to Save") - self.save_auto_on = Radiobutton( - frame_save, variable=self.autosave, value=1, - text='No Prompt') - - frame_format = Frame(frame_editor, borderwidth=0) + frame_format = Frame(frame_window, borderwidth=0) format_width_title = Label(frame_format, text='Format Paragraph Max Width') self.format_width_int = Entry( frame_format, textvariable=self.format_width, width=4, validatecommand=self.digits_only, validate='key', - ) - - frame_line_numbers_default = Frame(frame_editor, borderwidth=0) - line_numbers_default_title = Label( - frame_line_numbers_default, text='Show line numbers in new windows') - self.line_numbers_default_bool = Checkbutton( - frame_line_numbers_default, - variable=self.line_numbers_default, - width=1) - - frame_context = Frame(frame_editor, borderwidth=0) - context_title = Label(frame_context, text='Max Context Lines :') - self.context_int = Entry( - frame_context, textvariable=self.context_lines, width=3, - validatecommand=self.digits_only, validate='key', - ) - - # Frame_shell. - frame_auto_squeeze_min_lines = Frame(frame_shell, borderwidth=0) - auto_squeeze_min_lines_title = Label(frame_auto_squeeze_min_lines, - text='Auto-Squeeze Min. Lines:') - self.auto_squeeze_min_lines_int = Entry( - frame_auto_squeeze_min_lines, width=4, - textvariable=self.auto_squeeze_min_lines, - validatecommand=self.digits_only, validate='key', - ) - - # frame_help. - frame_helplist = Frame(frame_help) - frame_helplist_buttons = Frame(frame_helplist) - self.helplist = Listbox( - frame_helplist, height=5, takefocus=True, - exportselection=FALSE) - scroll_helplist = Scrollbar(frame_helplist) - scroll_helplist['command'] = self.helplist.yview - self.helplist['yscrollcommand'] = scroll_helplist.set - self.helplist.bind('', self.help_source_selected) - self.button_helplist_edit = Button( - frame_helplist_buttons, text='Edit', state='disabled', - width=8, command=self.helplist_item_edit) - self.button_helplist_add = Button( - frame_helplist_buttons, text='Add', - width=8, command=self.helplist_item_add) - self.button_helplist_remove = Button( - frame_helplist_buttons, text='Remove', state='disabled', - width=8, command=self.helplist_item_remove) + ) # Pack widgets: - # Body. frame_window.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_editor.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_shell.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - frame_help.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) # frame_run. frame_run.pack(side=TOP, padx=5, pady=0, fill=X) startup_title.pack(side=LEFT, anchor=W, padx=5, pady=5) @@ -2035,10 +1712,11 @@ def create_page_general(self): win_height_title.pack(side=RIGHT, anchor=E, pady=5) self.win_width_int.pack(side=RIGHT, anchor=E, padx=10, pady=5) win_width_title.pack(side=RIGHT, anchor=E, pady=5) - # frame_cursor_blink. - frame_cursor_blink.pack(side=TOP, padx=5, pady=0, fill=X) - cursor_blink_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.cursor_blink_bool.pack(side=LEFT, padx=5, pady=5) + # frame_cursor. + frame_cursor.pack(side=TOP, padx=5, pady=0, fill=X) + indent_title.pack(side=LEFT, anchor=W, padx=5) + self.indent_chooser.pack(side=LEFT, anchor=W, padx=10) + self.cursor_blink_bool.pack(side=RIGHT, anchor=E, padx=15, pady=5) # frame_autocomplete. frame_autocomplete.pack(side=TOP, padx=5, pady=0, fill=X) auto_wait_title.pack(side=LEFT, anchor=W, padx=5, pady=5) @@ -2051,41 +1729,12 @@ def create_page_general(self): paren_time_title.pack(side=LEFT, anchor=W, padx=5) self.bell_on.pack(side=RIGHT, anchor=E, padx=15, pady=5) self.paren_flash_time.pack(side=TOP, anchor=W, padx=15, pady=5) - - # frame_save. - frame_save.pack(side=TOP, padx=5, pady=0, fill=X) - run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) - self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) # frame_format. frame_format.pack(side=TOP, padx=5, pady=0, fill=X) format_width_title.pack(side=LEFT, anchor=W, padx=5, pady=5) self.format_width_int.pack(side=TOP, padx=10, pady=5) - # frame_line_numbers_default. - frame_line_numbers_default.pack(side=TOP, padx=5, pady=0, fill=X) - line_numbers_default_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.line_numbers_default_bool.pack(side=LEFT, padx=5, pady=5) - # frame_context. - frame_context.pack(side=TOP, padx=5, pady=0, fill=X) - context_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.context_int.pack(side=TOP, padx=5, pady=5) - # frame_auto_squeeze_min_lines - frame_auto_squeeze_min_lines.pack(side=TOP, padx=5, pady=0, fill=X) - auto_squeeze_min_lines_title.pack(side=LEFT, anchor=W, padx=5, pady=5) - self.auto_squeeze_min_lines_int.pack(side=TOP, padx=5, pady=5) - - # frame_help. - frame_helplist_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) - frame_helplist.pack(side=TOP, padx=5, pady=5, expand=TRUE, fill=BOTH) - scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) - self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) - self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) - self.button_helplist_add.pack(side=TOP, anchor=W) - self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) - - def load_general_cfg(self): - "Load current configuration settings for the general options." + def load_windows_cfg(self): # Set variables for all windows. self.startup_edit.set(idleConf.GetOption( 'main', 'General', 'editor-on-startup', type='bool')) @@ -2093,6 +1742,8 @@ def load_general_cfg(self): 'main', 'EditorWindow', 'width', type='int')) self.win_height.set(idleConf.GetOption( 'main', 'EditorWindow', 'height', type='int')) + self.indent_spaces.set(idleConf.GetOption( + 'main', 'Indent', 'num-spaces', type='int')) self.cursor_blink.set(idleConf.GetOption( 'main', 'EditorWindow', 'cursor-blink', type='bool')) self.autocomplete_wait.set(idleConf.GetOption( @@ -2103,27 +1754,394 @@ def load_general_cfg(self): 'extensions', 'ParenMatch', 'flash-delay', type='int')) self.paren_bell.set(idleConf.GetOption( 'extensions', 'ParenMatch', 'bell')) + self.format_width.set(idleConf.GetOption( + 'extensions', 'FormatParagraph', 'max-width', type='int')) + + +class ShedPage(Frame): + + def __init__(self, master): + super().__init__(master) + + self.init_validators() + self.create_page_shed() + self.load_shelled_cfg() + + def init_validators(self): + digits_or_empty_re = re.compile(r'[0-9]*') + def is_digits_or_empty(s): + "Return 's is blank or contains only digits'" + return digits_or_empty_re.fullmatch(s) is not None + self.digits_only = (self.register(is_digits_or_empty), '%P',) + + def create_page_shed(self): + """Return frame of widgets for Shell/Ed tab. + + Enable users to provisionally change shell and editor options. + Function load_shed_cfg initializes tk variables using idleConf. + Entry box auto_squeeze_min_lines_int sets + auto_squeeze_min_lines_int. Setting var_name invokes the + default callback that adds option to changes. + + Widgets for ShedPage(Frame): (*) widgets bound to self + frame_shell: LabelFrame + frame_auto_squeeze_min_lines: Frame + auto_squeeze_min_lines_title: Label + (*)auto_squeeze_min_lines_int: Entry - + auto_squeeze_min_lines + frame_editor: LabelFrame + frame_save: Frame + run_save_title: Label + (*)save_ask_on: Radiobutton - autosave + (*)save_auto_on: Radiobutton - autosave + frame_format: Frame + format_width_title: Label + (*)format_width_int: Entry - format_width + frame_line_numbers_default: Frame + line_numbers_default_title: Label + (*)line_numbers_default_bool: Checkbutton - line_numbers_default + frame_context: Frame + context_title: Label + (*)context_int: Entry - context_lines + """ + # Integer values need StringVar because int('') raises. + self.auto_squeeze_min_lines = tracers.add( + StringVar(self), ('main', 'PyShell', 'auto-squeeze-min-lines')) + + self.autosave = tracers.add( + IntVar(self), ('main', 'General', 'autosave')) + self.line_numbers_default = tracers.add( + BooleanVar(self), + ('main', 'EditorWindow', 'line-numbers-default')) + self.context_lines = tracers.add( + StringVar(self), ('extensions', 'CodeContext', 'maxlines')) + + # Create widgets: + frame_shell = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Shell Preferences') + frame_editor = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Editor Preferences') + # Frame_shell. + frame_auto_squeeze_min_lines = Frame(frame_shell, borderwidth=0) + auto_squeeze_min_lines_title = Label(frame_auto_squeeze_min_lines, + text='Auto-Squeeze Min. Lines:') + self.auto_squeeze_min_lines_int = Entry( + frame_auto_squeeze_min_lines, width=4, + textvariable=self.auto_squeeze_min_lines, + validatecommand=self.digits_only, validate='key', + ) + # Frame_editor. + frame_save = Frame(frame_editor, borderwidth=0) + run_save_title = Label(frame_save, text='At Start of Run (F5) ') + + self.save_ask_on = Radiobutton( + frame_save, variable=self.autosave, value=0, + text="Prompt to Save") + self.save_auto_on = Radiobutton( + frame_save, variable=self.autosave, value=1, + text='No Prompt') + + frame_line_numbers_default = Frame(frame_editor, borderwidth=0) + line_numbers_default_title = Label( + frame_line_numbers_default, text='Show line numbers in new windows') + self.line_numbers_default_bool = Checkbutton( + frame_line_numbers_default, + variable=self.line_numbers_default, + width=1) + + frame_context = Frame(frame_editor, borderwidth=0) + context_title = Label(frame_context, text='Max Context Lines :') + self.context_int = Entry( + frame_context, textvariable=self.context_lines, width=3, + validatecommand=self.digits_only, validate='key', + ) + + # Pack widgets: + frame_shell.pack(side=TOP, padx=5, pady=5, fill=BOTH) + Label(self).pack() # Spacer -- better solution? + frame_editor.pack(side=TOP, padx=5, pady=5, fill=BOTH) + # frame_auto_squeeze_min_lines + frame_auto_squeeze_min_lines.pack(side=TOP, padx=5, pady=0, fill=X) + auto_squeeze_min_lines_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.auto_squeeze_min_lines_int.pack(side=TOP, padx=5, pady=5) + # frame_save. + frame_save.pack(side=TOP, padx=5, pady=0, fill=X) + run_save_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.save_auto_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + self.save_ask_on.pack(side=RIGHT, anchor=W, padx=5, pady=5) + # frame_line_numbers_default. + frame_line_numbers_default.pack(side=TOP, padx=5, pady=0, fill=X) + line_numbers_default_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.line_numbers_default_bool.pack(side=LEFT, padx=5, pady=5) + # frame_context. + frame_context.pack(side=TOP, padx=5, pady=0, fill=X) + context_title.pack(side=LEFT, anchor=W, padx=5, pady=5) + self.context_int.pack(side=TOP, padx=5, pady=5) + def load_shelled_cfg(self): + # Set variables for shell windows. + self.auto_squeeze_min_lines.set(idleConf.GetOption( + 'main', 'PyShell', 'auto-squeeze-min-lines', type='int')) # Set variables for editor windows. self.autosave.set(idleConf.GetOption( 'main', 'General', 'autosave', default=0, type='bool')) - self.format_width.set(idleConf.GetOption( - 'extensions', 'FormatParagraph', 'max-width', type='int')) self.line_numbers_default.set(idleConf.GetOption( 'main', 'EditorWindow', 'line-numbers-default', type='bool')) self.context_lines.set(idleConf.GetOption( 'extensions', 'CodeContext', 'maxlines', type='int')) - # Set variables for shell windows. - self.auto_squeeze_min_lines.set(idleConf.GetOption( - 'main', 'PyShell', 'auto-squeeze-min-lines', type='int')) - # Set additional help sources. - self.user_helplist = idleConf.GetAllExtraHelpSourcesList() - self.helplist.delete(0, 'end') - for help_item in self.user_helplist: - self.helplist.insert(END, help_item[0]) - self.set_add_delete_state() +class ExtPage(Frame): + def __init__(self, master): + super().__init__(master) + self.ext_defaultCfg = idleConf.defaultCfg['extensions'] + self.ext_userCfg = idleConf.userCfg['extensions'] + self.is_int = self.register(is_int) + self.load_extensions() + self.create_page_extensions() # Requires extension names. + + def create_page_extensions(self): + """Configure IDLE feature extensions and help menu extensions. + + List the feature extensions and a configuration box for the + selected extension. Help menu extensions are in a HelpFrame. + + This code reads the current configuration using idleConf, + supplies a GUI interface to change the configuration values, + and saves the changes using idleConf. + + Some changes may require restarting IDLE. This depends on each + extension's implementation. + + All values are treated as text, and it is up to the user to + supply reasonable values. The only exception to this are the + 'enable*' options, which are boolean, and can be toggled with a + True/False button. + + Methods: + extension_selected: Handle selection from list. + create_extension_frame: Hold widgets for one extension. + set_extension_value: Set in userCfg['extensions']. + save_all_changed_extensions: Call extension page Save(). + """ + self.extension_names = StringVar(self) + + frame_ext = LabelFrame(self, borderwidth=2, relief=GROOVE, + text=' Feature Extensions ') + self.frame_help = HelpFrame(self, borderwidth=2, relief=GROOVE, + text=' Help Menu Extensions ') + + frame_ext.rowconfigure(0, weight=1) + frame_ext.columnconfigure(2, weight=1) + self.extension_list = Listbox(frame_ext, listvariable=self.extension_names, + selectmode='browse') + self.extension_list.bind('<>', self.extension_selected) + scroll = Scrollbar(frame_ext, command=self.extension_list.yview) + self.extension_list.yscrollcommand=scroll.set + self.details_frame = LabelFrame(frame_ext, width=250, height=250) + self.extension_list.grid(column=0, row=0, sticky='nws') + scroll.grid(column=1, row=0, sticky='ns') + self.details_frame.grid(column=2, row=0, sticky='nsew', padx=[10, 0]) + frame_ext.configure(padding=10) + self.config_frame = {} + self.current_extension = None + + self.outerframe = self # TEMPORARY + self.tabbed_page_set = self.extension_list # TEMPORARY + + # Create the frame holding controls for each extension. + ext_names = '' + for ext_name in sorted(self.extensions): + self.create_extension_frame(ext_name) + ext_names = ext_names + '{' + ext_name + '} ' + self.extension_names.set(ext_names) + self.extension_list.selection_set(0) + self.extension_selected(None) + + + frame_ext.grid(row=0, column=0, sticky='nsew') + Label(self).grid(row=1, column=0) # Spacer. Replace with config? + self.frame_help.grid(row=2, column=0, sticky='sew') + + def load_extensions(self): + "Fill self.extensions with data from the default and user configs." + self.extensions = {} + for ext_name in idleConf.GetExtensions(active_only=False): + # Former built-in extensions are already filtered out. + self.extensions[ext_name] = [] + + for ext_name in self.extensions: + opt_list = sorted(self.ext_defaultCfg.GetOptionList(ext_name)) + + # Bring 'enable' options to the beginning of the list. + enables = [opt_name for opt_name in opt_list + if opt_name.startswith('enable')] + for opt_name in enables: + opt_list.remove(opt_name) + opt_list = enables + opt_list + + for opt_name in opt_list: + def_str = self.ext_defaultCfg.Get( + ext_name, opt_name, raw=True) + try: + def_obj = {'True':True, 'False':False}[def_str] + opt_type = 'bool' + except KeyError: + try: + def_obj = int(def_str) + opt_type = 'int' + except ValueError: + def_obj = def_str + opt_type = None + try: + value = self.ext_userCfg.Get( + ext_name, opt_name, type=opt_type, raw=True, + default=def_obj) + except ValueError: # Need this until .Get fixed. + value = def_obj # Bad values overwritten by entry. + var = StringVar(self) + var.set(str(value)) + + self.extensions[ext_name].append({'name': opt_name, + 'type': opt_type, + 'default': def_str, + 'value': value, + 'var': var, + }) + + def extension_selected(self, event): + "Handle selection of an extension from the list." + newsel = self.extension_list.curselection() + if newsel: + newsel = self.extension_list.get(newsel) + if newsel is None or newsel != self.current_extension: + if self.current_extension: + self.details_frame.config(text='') + self.config_frame[self.current_extension].grid_forget() + self.current_extension = None + if newsel: + self.details_frame.config(text=newsel) + self.config_frame[newsel].grid(column=0, row=0, sticky='nsew') + self.current_extension = newsel + + def create_extension_frame(self, ext_name): + """Create a frame holding the widgets to configure one extension""" + f = VerticalScrolledFrame(self.details_frame, height=250, width=250) + self.config_frame[ext_name] = f + entry_area = f.interior + # Create an entry for each configuration option. + for row, opt in enumerate(self.extensions[ext_name]): + # Create a row with a label and entry/checkbutton. + label = Label(entry_area, text=opt['name']) + label.grid(row=row, column=0, sticky=NW) + var = opt['var'] + if opt['type'] == 'bool': + Checkbutton(entry_area, variable=var, + onvalue='True', offvalue='False', width=8 + ).grid(row=row, column=1, sticky=W, padx=7) + elif opt['type'] == 'int': + Entry(entry_area, textvariable=var, validate='key', + validatecommand=(self.is_int, '%P'), width=10 + ).grid(row=row, column=1, sticky=NSEW, padx=7) + + else: # type == 'str' + # Limit size to fit non-expanding space with larger font. + Entry(entry_area, textvariable=var, width=15 + ).grid(row=row, column=1, sticky=NSEW, padx=7) + return + + def set_extension_value(self, section, opt): + """Return True if the configuration was added or changed. + + If the value is the same as the default, then remove it + from user config file. + """ + name = opt['name'] + default = opt['default'] + value = opt['var'].get().strip() or default + opt['var'].set(value) + # if self.defaultCfg.has_section(section): + # Currently, always true; if not, indent to return. + if (value == default): + return self.ext_userCfg.RemoveOption(section, name) + # Set the option. + return self.ext_userCfg.SetOption(section, name, value) + + def save_all_changed_extensions(self): + """Save configuration changes to the user config file. + + Attributes accessed: + extensions + + Methods: + set_extension_value + """ + has_changes = False + for ext_name in self.extensions: + options = self.extensions[ext_name] + for opt in options: + if self.set_extension_value(ext_name, opt): + has_changes = True + if has_changes: + self.ext_userCfg.Save() + + +class HelpFrame(LabelFrame): + + def __init__(self, master, **cfg): + super().__init__(master, **cfg) + self.create_frame_help() + self.load_helplist() + + def create_frame_help(self): + """Create LabelFrame for additional help menu sources. + + load_helplist loads list user_helplist with + name, position pairs and copies names to listbox helplist. + Clicking a name invokes help_source selected. Clicking + button_helplist_name invokes helplist_item_name, which also + changes user_helplist. These functions all call + set_add_delete_state. All but load call update_help_changes to + rewrite changes['main']['HelpFiles']. + + Widgets for HelpFrame(LabelFrame): (*) widgets bound to self + frame_helplist: Frame + (*)helplist: ListBox + scroll_helplist: Scrollbar + frame_buttons: Frame + (*)button_helplist_edit + (*)button_helplist_add + (*)button_helplist_remove + """ + # self = frame_help in dialog (until ExtPage class). + frame_helplist = Frame(self) + self.helplist = Listbox( + frame_helplist, height=5, takefocus=True, + exportselection=FALSE) + scroll_helplist = Scrollbar(frame_helplist) + scroll_helplist['command'] = self.helplist.yview + self.helplist['yscrollcommand'] = scroll_helplist.set + self.helplist.bind('', self.help_source_selected) + + frame_buttons = Frame(self) + self.button_helplist_edit = Button( + frame_buttons, text='Edit', state='disabled', + width=8, command=self.helplist_item_edit) + self.button_helplist_add = Button( + frame_buttons, text='Add', + width=8, command=self.helplist_item_add) + self.button_helplist_remove = Button( + frame_buttons, text='Remove', state='disabled', + width=8, command=self.helplist_item_remove) + + # Pack frame_help. + frame_helplist.pack(side=LEFT, padx=5, pady=5, expand=TRUE, fill=BOTH) + self.helplist.pack(side=LEFT, anchor=E, expand=TRUE, fill=BOTH) + scroll_helplist.pack(side=RIGHT, anchor=W, fill=Y) + frame_buttons.pack(side=RIGHT, padx=5, pady=5, fill=Y) + self.button_helplist_edit.pack(side=TOP, anchor=W, pady=5) + self.button_helplist_add.pack(side=TOP, anchor=W) + self.button_helplist_remove.pack(side=TOP, anchor=W, pady=5) def help_source_selected(self, event): "Handle event for selecting additional help." @@ -2193,6 +2211,14 @@ def update_help_changes(self): 'main', 'HelpFiles', str(num), ';'.join(self.user_helplist[num-1][:2])) + def load_helplist(self): + # Set additional help sources. + self.user_helplist = idleConf.GetAllExtraHelpSourcesList() + self.helplist.delete(0, 'end') + for help_item in self.user_helplist: + self.helplist.insert(END, help_item[0]) + self.set_add_delete_state() + class VarTrace: """Maintain Tk variables trace state.""" @@ -2314,7 +2340,15 @@ def detach(self): Shell Preferences: Auto-Squeeze Min. Lines is the minimum number of lines of output to automatically "squeeze". -''' +''', + 'Extensions': ''' +ZzDummy: This extension is provided as an example for how to create and +use an extension. Enable indicates whether the extension is active or +not; likewise enable_editor and enable_shell indicate which windows it +will be active on. For this extension, z-text is the text that will be +inserted at or removed from the beginning of the lines of selected text, +or the current line if no selection. +''', } diff --git a/Lib/idlelib/debugger_r.py b/Lib/idlelib/debugger_r.py index 9dcfc56414c050..26204438858d8a 100644 --- a/Lib/idlelib/debugger_r.py +++ b/Lib/idlelib/debugger_r.py @@ -19,7 +19,7 @@ barrier, in particular frame and traceback objects. """ - +import reprlib import types from idlelib import debugger @@ -170,7 +170,7 @@ def dict_keys_list(self, did): def dict_item(self, did, key): dict = dicttable[did] value = dict[key] - value = repr(value) ### can't pickle module 'builtins' + value = reprlib.repr(value) ### can't pickle module 'builtins' return value #----------end class IdbAdapter---------- @@ -390,4 +390,4 @@ def restart_subprocess_debugger(rpcclt): if __name__ == "__main__": from unittest import main - main('idlelib.idle_test.test_debugger', verbosity=2, exit=False) + main('idlelib.idle_test.test_debugger_r', verbosity=2, exit=False) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index b0f88b5463d1b6..b9cb50264ff06f 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -12,8 +12,8 @@ from tkinter import * from tkinter.font import Font from tkinter.ttk import Scrollbar -import tkinter.simpledialog as tkSimpleDialog -import tkinter.messagebox as tkMessageBox +from tkinter import simpledialog +from tkinter import messagebox from idlelib.config import idleConf from idlelib import configdialog @@ -46,7 +46,7 @@ def _sphinx_version(): return release -class EditorWindow(object): +class EditorWindow: from idlelib.percolator import Percolator from idlelib.colorizer import ColorDelegator, color_config from idlelib.undo import UndoDelegator @@ -295,9 +295,9 @@ def __init__(self, flist=None, filename=None, key=None, root=None): window.register_callback(self.postwindowsmenu) # Some abstractions so IDLE extensions are cross-IDE - self.askyesno = tkMessageBox.askyesno - self.askinteger = tkSimpleDialog.askinteger - self.showerror = tkMessageBox.showerror + self.askinteger = simpledialog.askinteger + self.askyesno = messagebox.askyesno + self.showerror = messagebox.showerror # Add pseudoevents for former extension fixed keys. # (This probably needs to be done once in the process.) @@ -339,7 +339,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): text.bind("<>", self.code_context.toggle_code_context_event) else: - self.update_menu_state('options', '*Code Context', 'disabled') + self.update_menu_state('options', '*ode*ontext', 'disabled') if self.allow_line_numbers: self.line_numbers = self.LineNumbers(self) if idleConf.GetOption('main', 'EditorWindow', @@ -347,7 +347,7 @@ def __init__(self, flist=None, filename=None, key=None, root=None): self.toggle_line_numbers_event() text.bind("<>", self.toggle_line_numbers_event) else: - self.update_menu_state('options', '*Line Numbers', 'disabled') + self.update_menu_state('options', '*ine*umbers', 'disabled') def handle_winconfig(self, event=None): self.set_width() @@ -450,7 +450,9 @@ def createmenubar(self): self.menudict = menudict = {} for name, label in self.menu_specs: underline, label = prepstr(label) - menudict[name] = menu = Menu(mbar, name=name, tearoff=0) + postcommand = getattr(self, f'{name}_menu_postcommand', None) + menudict[name] = menu = Menu(mbar, name=name, tearoff=0, + postcommand=postcommand) mbar.add_cascade(label=label, menu=menu, underline=underline) if macosx.isCarbonTk(): # Insert the application menu @@ -499,15 +501,23 @@ def handle_yview(self, event, *args): rmenu = None def right_menu_event(self, event): - self.text.tag_remove("sel", "1.0", "end") - self.text.mark_set("insert", "@%d,%d" % (event.x, event.y)) + text = self.text + newdex = text.index(f'@{event.x},{event.y}') + try: + in_selection = (text.compare('sel.first', '<=', newdex) and + text.compare(newdex, '<=', 'sel.last')) + except TclError: + in_selection = False + if not in_selection: + text.tag_remove("sel", "1.0", "end") + text.mark_set("insert", newdex) if not self.rmenu: self.make_rmenu() rmenu = self.rmenu self.event = event iswin = sys.platform[:3] == 'win' if iswin: - self.text.config(cursor="arrow") + text.config(cursor="arrow") for item in self.rmenu_specs: try: @@ -520,7 +530,6 @@ def right_menu_event(self, event): state = getattr(self, verify_state)() rmenu.entryconfigure(label, state=state) - rmenu.tk_popup(event.x_root, event.y_root) if iswin: self.text.config(cursor="ibeam") @@ -589,7 +598,7 @@ def python_docs(self, event=None): try: os.startfile(self.help_url) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(self.help_url) @@ -920,7 +929,7 @@ def display_extra_help(helpfile=helpfile): try: os.startfile(helpfile) except OSError as why: - tkMessageBox.showerror(title='Document Start Failure', + messagebox.showerror(title='Document Start Failure', message=str(why), parent=self.text) else: webbrowser.open(helpfile) @@ -956,7 +965,7 @@ def update_recent_files_list(self, new_file=None): except OSError as err: if not getattr(self.root, "recentfiles_message", False): self.root.recentfiles_message = True - tkMessageBox.showwarning(title='IDLE Warning', + messagebox.showwarning(title='IDLE Warning', message="Cannot save Recent Files list to disk.\n" f" {err}\n" "Select OK to continue.", @@ -1520,7 +1529,7 @@ def toggle_line_numbers_event(self, event=None): else: self.line_numbers.show_sidebar() menu_label = "Hide" - self.update_menu_label(menu='options', index='*Line Numbers', + self.update_menu_label(menu='options', index='*ine*umbers', label=f'{menu_label} Line Numbers') # "line.col" -> line, as an int @@ -1539,7 +1548,7 @@ def get_line_indent(line, tabwidth): return m.end(), len(m.group().expandtabs(tabwidth)) -class IndentSearcher(object): +class IndentSearcher: # .run() chews over the Text widget, looking for a block opener # and the stmt following it. Returns a pair, diff --git a/Lib/idlelib/extend.txt b/Lib/idlelib/extend.txt index c9cb2e8297eb35..b482f76c4fb0f7 100644 --- a/Lib/idlelib/extend.txt +++ b/Lib/idlelib/extend.txt @@ -28,8 +28,8 @@ variables: (There are a few more, but they are rarely useful.) The extension class must not directly bind Window Manager (e.g. X) events. -Rather, it must define one or more virtual events, e.g. <>, and -corresponding methods, e.g. zoom_height_event(). The virtual events will be +Rather, it must define one or more virtual events, e.g. <>, and +corresponding methods, e.g. z_in_event(). The virtual events will be bound to the corresponding methods, and Window Manager events can then be bound to the virtual events. (This indirection is done so that the key bindings can easily be changed, and so that other sources of virtual events can exist, such @@ -54,21 +54,21 @@ Extensions are not required to define menu entries for all the events they implement. (They are also not required to create keybindings, but in that case there must be empty bindings in cofig-extensions.def) -Here is a complete example: +Here is a partial example from zzdummy.py: -class ZoomHeight: +class ZzDummy: menudefs = [ - ('edit', [ - None, # Separator - ('_Zoom Height', '<>'), - ]) + ('format', [ + ('Z in', '<>'), + ('Z out', '<>'), + ] ) ] def __init__(self, editwin): self.editwin = editwin - def zoom_height_event(self, event): + def z_in_event(self, event=None): "...Do what you want here..." The final piece of the puzzle is the file "config-extensions.def", which is diff --git a/Lib/idlelib/filelist.py b/Lib/idlelib/filelist.py index 0d200854ef0007..254f5caf6b81b0 100644 --- a/Lib/idlelib/filelist.py +++ b/Lib/idlelib/filelist.py @@ -1,7 +1,7 @@ "idlelib.filelist" import os -from tkinter import messagebox as tkMessageBox +from tkinter import messagebox class FileList: @@ -20,7 +20,7 @@ def open(self, filename, action=None): filename = self.canonize(filename) if os.path.isdir(filename): # This can happen when bad filename is passed on command line: - tkMessageBox.showerror( + messagebox.showerror( "File Error", "%r is a directory." % (filename,), master=self.root) @@ -88,7 +88,7 @@ def filename_changed_edit(self, edit): if newkey in self.dict: conflict = self.dict[newkey] self.inversedict[conflict] = None - tkMessageBox.showerror( + messagebox.showerror( "Name Conflict", "You now have multiple edit windows open for %r" % (filename,), master=self.root) diff --git a/Lib/idlelib/help.html b/Lib/idlelib/help.html index 424c6b50f339e1..3f87e89f675c4e 100644 --- a/Lib/idlelib/help.html +++ b/Lib/idlelib/help.html @@ -1,23 +1,24 @@ - + - IDLE — Python 3.9.0a4 documentation + + IDLE — Python 3.11.0a0 documentation - - - - - + + + + + - + @@ -31,7 +32,6 @@ -