diff --git a/.github/appveyor.yml b/.github/appveyor.yml index b7d40787318985a..9d47c9975e5a995 100644 --- a/.github/appveyor.yml +++ b/.github/appveyor.yml @@ -7,11 +7,30 @@ branches: - buildbot-custom cache: - externals -> PCbuild\* +before_build: + - ps: |+ + if ($env:APPVEYOR_RE_BUILD) { + echo 'Doing full build due to re-build request.' + } elseif (!$env:APPVEYOR_PULL_REQUEST_HEAD_COMMIT) { + echo 'Not a PR, doing full build.' + } else { + git fetch -q origin +refs/heads/$env:APPVEYOR_REPO_BRANCH + $mergebase = git merge-base HEAD FETCH_HEAD + $changes = git diff --name-only HEAD $mergebase | grep -vE '(\.rst$)|(^Doc)|(^Misc)' + If (!$changes) { + echo 'Only docs were updated, stopping build process.' + Exit-AppveyorBuild + } + echo 'Doing full build due to non-doc changes in these files:' + echo $changes + } + + build_script: -- cmd: PCbuild\build.bat -e -- cmd: PCbuild\win32\python.exe -m test.pythoninfo + - cmd: PCbuild\build.bat -e + - cmd: PCbuild\win32\python.exe -m test.pythoninfo test_script: -- cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 --fail-env-changed -j0 + - cmd: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 --fail-env-changed -j0 environment: HOST_PYTHON: C:\Python36\python.exe image: diff --git a/.gitignore b/.gitignore index 05fb6cba0875a1c..58f8bf72f2b9bd5 100644 --- a/.gitignore +++ b/.gitignore @@ -113,3 +113,4 @@ Tools/ssl/amd64 Tools/ssl/win32 .vs/ .vscode/ +gmon.out diff --git a/.hgeol b/.hgeol deleted file mode 100644 index eb19a6c88d28d05..000000000000000 --- a/.hgeol +++ /dev/null @@ -1,58 +0,0 @@ -[patterns] - -# Non human-editable files are binary - -**.dsp = BIN -**.dsw = BIN -**.mk = BIN -**.sln = BIN -**.vcproj = BIN -**.vsprops = BIN - -**.aif = BIN -**.aifc = BIN -**.aiff = BIN -**.au = BIN -**.bmp = BIN -**.db = BIN -**.exe = BIN -**.icns = BIN -**.gif = BIN -**.ico = BIN -**.info = BIN -**.jpg = BIN -**.pck = BIN -**.png = BIN -**.psd = BIN -**.tar = BIN -**.wav = BIN -**.whl = BIN -**.xar = BIN -**.zip = BIN - -Lib/test/cjkencodings/* = BIN -Lib/test/decimaltestdata/*.decTest = BIN -Lib/test/sndhdrdata/sndhdr.* = BIN -Lib/test/test_email/data/msg_26.txt = BIN -Lib/test/xmltestdata/* = BIN - -Lib/venv/scripts/nt/* = BIN - -Lib/test/coding20731.py = BIN - -# Windows only zlib upstream file -Modules/zlib/zlib.map = CRLF - -# Windows batch files work best with CRLF, there can be subtle problems with LF -**.bat = CRLF - -# The Windows readme is likely to be read in Notepad, so make it readable -PCbuild/readme.txt = CRLF - -# All other files (which presumably are human-editable) are "native". -# This must be the last rule! - -** = native - -[repository] -native = LF diff --git a/.hgignore b/.hgignore deleted file mode 100644 index 68c607f2e8d420c..000000000000000 --- a/.hgignore +++ /dev/null @@ -1,106 +0,0 @@ -.gdb_history -.purify -.svn/ -^.idea/ -^.vscode/ -.DS_Store -Makefile$ -Makefile.pre$ -TAGS$ -autom4te.cache$ -^build/ -^Doc/build/ -^Doc/venv/ -buildno$ -config.cache -config.log -config.status -config.status.lineno -db_home -platform$ -pyconfig.h$ -python$ -python.bat$ -python.exe$ -python-config$ -python-config.py$ -reflog.txt$ -tags$ -Misc/python.pc -Misc/python-config.sh$ -Modules/Setup$ -Modules/Setup.config -Modules/Setup.local -Modules/config.c -Modules/ld_so_aix$ -^lcov-report/ -^core -^python-gdb.py -^python.exe-gdb.py -^pybuilddir.txt - -syntax: glob -libpython*.a -libpython*.so* -libpython*.dylib -libpython*.dll -*.swp -*.o -*.pyc -*.pyo -*.pyd -*.cover -*~ -*.gc?? -*.profclang? -*.profraw -*.dyn -Include/pydtrace_probes.h -Lib/distutils/command/*.pdb -Lib/lib2to3/*.pickle -Lib/test/data/* -Misc/*.wpu -PC/python_nt*.h -PC/pythonnt_rc*.h -PC/*/*.exe -PC/*/*.exp -PC/*/*.lib -PC/*/*.bsc -PC/*/*.dll -PC/*/*.pdb -PC/*/*.user -PC/*/*.ncb -PC/*/*.suo -PC/*/Win32-temp-* -PC/*/x64-temp-* -PC/*/amd64 -PCbuild/*.user -PCbuild/*.suo -PCbuild/*.*sdf -PCbuild/*-pgi -PCbuild/*-pgo -PCbuild/.vs -PCbuild/amd64 -PCbuild/obj -PCbuild/win32 -Tools/unicode/build/ -Tools/unicode/MAPPINGS/ -BuildLog.htm -__pycache__ -Parser/pgen{,.exe} -Programs/_freeze_importlib{,.exe} -Programs/_testembed{,.exe} -.coverage -coverage/ -externals/ -htmlcov/ -*.gcda -*.gcno -*.gcov -ipch/ -coverage.info -Tools/msi/obj -Tools/ssl/amd64 -Tools/ssl/win32 -.vs/ -.vscode/ diff --git a/.hgtags b/.hgtags deleted file mode 100644 index 8f51c2ced49aed4..000000000000000 --- a/.hgtags +++ /dev/null @@ -1,182 +0,0 @@ -64cc5439e10a6fdf984effaf0141e94fa4cc1004 v0.9.8 -78a7ed6953025e7ecdde9585099b01a6ae40b76a v0.9.9 -b15b8cc9b8d10e0352a0b8b7e8d51fa309db6df3 v1.0.1 -0326b5d61445ee3a8d3de28119f9652cb72d2e3f v1.0.2 -832615ec07646e310c85316b8ba6bc9b17ad3547 v1.1 -9895475d18c7b5f32adaf78f71886ae041e4d10c v1.1.1 -16eb4c51ee97169046340998e850a63c65225b0a v1.2b1 -b45c688756d04fb84d4a0d518fc3d7e3cb25fa8d v1.2b2 -9e82daf7605bad7976a9abc997cb5e0abe434078 v1.2b3 -065e31cf5862e27521cf5725b003aed211f091b2 v1.2b4 -e72257e655454d569468da8b1189e0ec336f3536 v1.2 -e63d83f8275853aaaa3d1972cb86564505e65583 v1.3b1 -7d743c865a9aa6bde8b603e32e0542031bba3c33 v1.3 -4fc85c82cc222554ae6b9c0b87776ed5f2b70c6e v1.4b1 -129f1299d4e97e884bbbbdd00baf101d178973e6 v1.4b2 -44a82ac654a4175569deed8e8a94b0cc8edee08d v1.4b3 -db49494c93dc73de06d5721c74eab533a947a92c v1.4 -062aed8a4ce2c91c81b80e29f02faff1cf5a761b v1.5a1 -c9498ac988372575cf7028b86395b900c9b0a840 v1.5a2 -dc5c968ec992aab3d40a7189df0c443d1c7a1a68 v1.5a3 -746654a0af680c7d9b814b210a026eb91bec9533 v1.5a4 -8ff58b5730f06be08fbbdc2bf592226f7a736201 v1.5b1 -eb78658d819fb0af09a8e6f9bedcb670805ed5f6 v1.5b2 -84461011a1a0ab402e352f06748f29fb5b5559e5 v1.5 -44aba4d26b01fbae0403efe654f9fd0347606732 v1.5.1 -fed63ccbe6dc3ac663bfe97a2f7006b1b28568f9 v1.5.2a1 -21d71f2e27248a0f4e393d0fc321ecf9b89321d2 v1.5.2a2 -f08c7a2a56f80741f5f192fd0ebe0b0967a203cf v1.5.2b1 -8fe7ec4b4fc1518fcac89e6bf674fbbce16150a9 v1.5.2b2 -39fb0dcc83dc375c1565ba65dbce0ed59b1359c9 v1.5.2c1 -61c91c7f101bab3149adfcd5646ae40e048de712 v1.5.2 -605eb9326ffe1fd1e43f40e2338d6652ab449fdf v1.6a1 -011bee8fd9f7f4da457ec71596484fb0882c0614 v1.6a2 -35c4fc1414a59888614b9be784a25f233ba67984 v2.0b1 -55bba197d4870cdae62aeca00e20240a756b84f8 v2.0b2 -e276329cce036a5f9e9d3451256dca5984e543dc v2.0c1 -2fa4e35083e02342ca014bf5bfba46aecb816c31 v2.0 -b60831eeab5a06dd3c5e8297a99e39297aa8794b v2.1a1 -b382f1f07ec6b2c95551658b30c6139eeb32077a v2.1a2 -d0c830db5e68edd4aaa3401216e610c9ff145826 v2.1b1 -b59a536ae1ef3774fd85c17f623e8926b7b6c095 v2.1b2 -d611276e9ad53b5d32d1e8065e1d811c32f7d96f v2.1c1 -ff065e674af6c9ab895bd9eff7d9e9039a376c7d v2.1c2 -020e95d8180d7943fe54701e1db0a7d7d87e2b1e v2.1 -08796a137f1ada2462f7a3177306df5f67a767e1 v2.2a3 -d054c29647f90bccb8345bd779bca1eecf2dd7f2 v2.3c1 -fce5c9e9abc722394cb2e909b7e2a39080d4448e v2.3c2 -92ca658fd420095b6284c9ce6e9082a80285ec9c v2.4a1 -055fc6955f3c6522bfeb7ed4c671c97d5baaaac2 v2.4a2 -186b72550e53533ef6175f6411f932c1298193d7 v2.4a3 -53cff04283855adf88ed0c0fd3698827ca843637 v2.4b1 -7e387a9dcc79954a77695adef8b593da35be1214 v2.4b2 -ff80d8bbef6e13426c8a85d7f9d837b8f8f89834 v2.4c1 -f31e18d313c7a4fc66914b2d27e130a0f72c0b69 v2.4 -cd3f783cd08a16781e236c0b9cb5717d1d995fa9 v3.0a1 -65e82140e281bf26f2e22eda05a7f9956c420f8b v3.0a2 -df15827f34881b9af0936350813ced5c123c8230 v3.0a3 -15f773f7300e372c56a21d59fe49ca26955a6477 v3.0a4 -e35935475153377d6727d64e6c52f72c3b84015b v3.0a5 -a335c4d643b1cfe14197a9ef195c9b2804f608fc v3.0b1 -16ec4bb14a68ea428acf09ebf0c92981da2646f3 v3.0b2 -509e30a7968e01be329ec121540b3e755fc4e566 v3.0b3 -507ede9c7f7f475dfafbd4a52c22d767d10a2bc0 v3.0rc1 -8fae465a39627b590385462e6095eb63af45240a v3.0rc2 -e83a60c69d53f5551a306e77a6d38e9b11485496 v3.0rc3 -bc1ce368986e45b1faf96f93995df46bcd75e7b8 v3.1a1 -ee430e5075db2adf8124e6b94916a89ca41d3171 v3.1a2 -b63020797f9678adaf4d2c3e26574a9eef2ef028 v3.1b1 -4353fd0843cb31b356adc50f93d220e2e7255ef2 v3.1rc1 -0b87e438e1b53e3f812cad963a7fdb65d198ba2f v3.1rc2 -a69a031ac1402dede8b1ef80096436bca6d371f3 v3.1 -35efb1054ec6ceca72017a587263cb6a9257340b v3.1.1rc1 -8b9c0f573ab29c41c6c5fdcca82a1fe0ff5355af v3.1.1 -149b8b87514d10416b598884db5f74651f625b38 v3.1.2rc1 -960efa327c5d9c18df995437b0ac550cb89c9f85 v3.1.2 -d18e9d71f369d8211f6ac87252c6d3211f9bd09f v3.1.3rc1 -a4f75773c0060cee38b0bb651a7aba6f56b0e996 v3.1.3 -32fcb9e94985cb19ce37ba9543f091c0dbe9d7dd v3.1.4rc1 -c918ec9f3a76d6afedfbb5d455004de880443a3d v3.1.4 -ee26aca3219cf4bb0b93352e83edcc9cb28c7802 v3.1.5rc1 -75db2bc69fc9a3e4801e94e3e19801cb096208d8 v3.1.5rc2 -7395330e495ec3316862ca1f6ce0aaf7bdf6785b v3.1.5 -b37b7834757492d009b99cf0ca4d42d2153d7fac v3.2a1 -56d4373cecb73c8b45126ba7b045b3c7b3f94b0b v3.2a2 -da012d9a2c23d144e399d2e01a55b8a83ad94573 v3.2a3 -d92a5b850f5e56808bedc01723906ed64c5e6e2e v3.2a4 -b635cea94195780c8716e236479af319bcc26253 v3.2b1 -e3af5f3a7904c0d5343ec9633ea66e7acfd23a66 v3.2b2 -865d5b24bf28ca41b536befc326407c03e74a4d5 v3.2rc1 -acf3e24dd0d0dfd1e20c907d696d3da965a8f56f v3.2rc2 -18c1f52896501c7ee13b038454a39acb45a87979 v3.2rc3 -a222a015e28d8ae9af3899258dc6c15c3d40add0 v3.2 -8ffac2337a3323323d02153ac919fd1483176652 v3.2.1b1 -cfa9364997c7f2e67b9cbb45c3a5fa3bba4e4999 v3.2.1rc1 -5df549718fb4841ff521fe051f6b54f290fad5d8 v3.2.1rc2 -ac1f7e5c05104d557d5acd922e95625ba5d1fe10 v3.2.1 -c860feaa348d663e598986894ee4680480577e15 v3.2.2rc1 -137e45f15c0bd262c9ad4c032d97425bc0589456 v3.2.2 -7085403daf439adb3f9e70ef13f6bedb1c447376 v3.2.3rc1 -428f05cb7277e1d42bb9dd8d1af6b6270ebc6112 v3.2.3rc2 -3d0686d90f55a78f96d9403da2c52dc2411419d0 v3.2.3 -b2cb7bc1edb8493c0a78f9331eae3e8fba6a881d v3.2.4rc1 -1e10bdeabe3de02f038a63c001911561ac1d13a7 v3.2.4 -cef745775b6583446572cffad704100983db2bea v3.2.5 -51382a5598ec96119cb84594572901c9c964dc3c v3.2.6rc1 -0bd5f4f14de965ca8e44c6e3965fee106176cfc4 v3.2.6 -f1a9a6505731714f0e157453ff850e3b71615c45 v3.3.0a1 -2f69db52d6de306cdaef0a0cc00cc823fb350b01 v3.3.0a2 -0b53b70a40a00013505eb35e3660057b62be77be v3.3.0a3 -7c51388a3aa7ce76a8541bbbdfc05d2d259a162c v3.3.0a4 -e15c554cd43eb23bc0a528a4e8741da9bbec9607 v3.3.0b1 -4972a8f1b2aa3d7cdd64dc96aa7fa112fe1ea343 v3.3.0b2 -8bb5c7bc46ba43804480f3e328e1fa956672c885 v3.3.0rc1 -88a0792e8ba3e4916b24c7e7a522c277d326d66e v3.3.0rc2 -c191d21cefafb3832c45570e84854e309aa62eaa v3.3.0rc3 -bd8afb90ebf28ba4edc901d4a235f75e7bbc79fd v3.3.0 -92c2cfb924055ce68c4f78f836dcfe688437ceb8 v3.3.1rc1 -d9893d13c6289aa03d33559ec67f97dcbf5c9e3c v3.3.1 -d047928ae3f6314a13b6137051315453d0ae89b6 v3.3.2 -fd53c500f8b80f54f3ecedec9da2e8c7e52a6888 v3.3.3rc1 -d32442c0e60dfbd71234e807d3d1dedd227495a9 v3.3.3rc2 -c3896275c0f61b2510a6c7e6c458a750359a91b8 v3.3.3 -fa92f5f940c6c0d839d7f0611e4b717606504a3c v3.3.4rc1 -7ff62415e4263c432c8acf6e424224209211eadb v3.3.4 -9ec811df548ed154a9bf9815383a916d6df31b98 v3.3.5rc1 -ca5635efe090f78806188ac2758f9948596aa8b2 v3.3.5rc2 -62cf4e77f78564714e7ea3d4bf1479ca1fbd0758 v3.3.5 -51317c9786f54267975abf2e9c502e6aaaa4a249 v3.3.6rc1 -971fec30da1fc5bf2b9fb28e09812a5127014211 v3.3.6 -46535f65e7f3bcdcf176f36d34bc1fed719ffd2b v3.4.0a1 -9265a2168e2cb2a84785d8717792acc661e6b692 v3.4.0a2 -dd9cdf90a5073510877e9dd5112f8e6cf20d5e89 v3.4.0a3 -e245b0d7209bb6d0e19316e1e2af1aa9c2139104 v3.4.0a4 -3405dc9a6afaa0a06dd1f6f182ec5c998dce6f5f v3.4.0b1 -ba32913eb13ec545a46dd0ce18035b6c416f0d78 v3.4.0b2 -a97ce3ecc96af79bd2e1ac66ce48d9138e0ca749 v3.4.0b3 -5e088cea8660677969113741c1313d570d977e02 v3.4.0rc1 -a300712ed38c9a242b736c44e806caea25a6dc05 v3.4.0rc2 -8a81cdab3e9d521daaef989fade94b16455fc3b8 v3.4.0rc3 -04f714765c13824c3bc2835d7b008908862e083a v3.4.0 -c67a19e11a7191baf30f313bf55e2e0b6c6f574e v3.4.1rc1 -c0e311e010fcb5bae8d87ca22051cd0845ea0ca0 v3.4.1 -8711a09513848cfc48c689d983495ee64f4668ca v3.4.2rc1 -ab2c023a9432f16652e89c404bbc84aa91bf55af v3.4.2 -69dd528ca6255a66c37cc5cf680e8357d108b036 v3.4.3rc1 -b4cbecbc0781e89a309d03b60a1f75f8499250e6 v3.4.3 -04f3f725896c6961212c3a12e8ac25be6958f4fa v3.4.4rc1 -737efcadf5a678b184e0fa431aae11276bf06648 v3.4.4 -3631bb4a2490292ebf81d3e947ae36da145da564 v3.4.5rc1 -619b61e505d0e2ccc8516b366e4ddd1971b46a6f v3.4.5 -e199a272ccdac5a8c073d4690f60c13e0b6d86b0 v3.4.6rc1 -b662f47769213f23325144b80782c05764d0f053 v3.4.6 -5d4b6a57d5fd7564bf73f3db0e46fe5eeb00bcd8 v3.5.0a1 -0337bd7ebcb6559d69679bc7025059ad1ce4f432 v3.5.0a2 -82656e28b5e5c4ae48d8dd8b5f0d7968908a82b6 v3.5.0a3 -413e0e0004f4f954331cb8122aa55fe208984955 v3.5.0a4 -071fefbb5e3db770c6c19fba9994699f121b1cea v3.5.0b1 -7a088af5615bf04024e9912068f4bd8f43ed3917 v3.5.0b2 -0035fcd9b9243ae52c2e830204fd9c1f7d528534 v3.5.0b3 -c0d64105463581f85d0e368e8d6e59b7fd8f12b1 v3.5.0b4 -1a58b1227501e046eee13d90f113417b60843301 v3.5.0rc1 -cc15d736d860303b9da90d43cd32db39bab048df v3.5.0rc2 -66ed52375df802f9d0a34480daaa8ce79fc41313 v3.5.0rc3 -2d033fedfa7f1e325fd14ccdaa9cb42155da206f v3.5.0rc4 -374f501f4567b7595f2ad7798aa09afa2456bb28 v3.5.0 -948ef16a69513ba1ff15c9d7d0b012b949df4c80 v3.5.1rc1 -37a07cee5969e6d3672583187a73cf636ff28e1b v3.5.1 -68feec6488b26327a85a634605dd28eca4daa5f1 v3.5.2rc1 -4def2a2901a5618ea45bcc8f2a1411ef33af18ad v3.5.2 -de530d7f21c0398bb2a2b67716e0638e5fadf727 v3.5.3rc1 -1880cb95a742cd001c67677de5c4efeab169416c v3.5.3 -5896da372fb044e38595fb74495de1e1e7c8fb3c v3.6.0a1 -37889342355223e2fc1438de3dc7ffcd625c60f7 v3.6.0a2 -f3edf13dc339b8942ae6b309771ab197dd8ce6fa v3.6.0a3 -017cf260936b444788c9b671d195b7bfd83dbd25 v3.6.0a4 -5b0ca4ed5e2f0669d76ece7ef975c544580f12b4 v3.6.0b1 -b9fadc7d1c3f9c3c77f32f35afbe1a1cc38070e6 v3.6.0b2 -8345e066c0ed713c3e510cbc8fafc1c38d6d306b v3.6.0b3 -18496abdb3d5c2730a659b747a89261b2219fecf v3.6.0b4 -29a273eee9a523ee178f6a66c4ac9d317c8fc84f v3.6.0rc1 -800a67f7806de45a7abd5273359e704bf147c079 v3.6.0rc2 -41df79263a11f2429d1dd0cfe12553de3dcb5508 v3.6.0 diff --git a/.travis.yml b/.travis.yml index d7387e5f9831b2d..ab43318975fb558 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,17 +5,18 @@ group: beta # To cache doc-building dependencies and C compiler output. cache: - - pip - - ccache - - directories: - - $HOME/multissl + - pip + - ccache + - directories: + - $HOME/multissl env: global: - OPENSSL=1.1.0g - OPENSSL_DIR="$HOME/multissl/openssl/${OPENSSL}" - PATH="${OPENSSL_DIR}/bin:$PATH" - - CFLAGS="-I${OPENSSL_DIR}/include" + # Use -O3 because we don't use debugger on Travis-CI + - CFLAGS="-I${OPENSSL_DIR}/include -O3" - LDFLAGS="-L${OPENSSL_DIR}/lib" # Set rpath with env var instead of -Wl,-rpath linker flag # OpenSSL ignores LDFLAGS when linking bin/openssl @@ -25,6 +26,7 @@ branches: only: - master - /^\d\.\d$/ + - buildbot-custom matrix: fast_finish: true @@ -55,22 +57,12 @@ matrix: compiler: gcc env: OPTIONAL=true before_script: - - | - if ! git diff --name-only $TRAVIS_COMMIT_RANGE | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' - then - echo "Only docs were updated, stopping build process." - exit - fi - python3 Tools/ssl/multissltests.py --steps=library \ - --base-directory ${HOME}/multissl \ - --openssl ${OPENSSL} >/dev/null - openssl version - ./configure - make -s -j4 - # Need a venv that can parse covered code. - ./python -m venv venv - ./venv/bin/python -m pip install -U coverage - ./venv/bin/python -m test.pythoninfo + - ./configure + - make -s -j4 + # Need a venv that can parse covered code. + - ./python -m venv venv + - ./venv/bin/python -m pip install -U coverage + - ./venv/bin/python -m test.pythoninfo script: # Skip tests that re-run the entire test suite. - ./venv/bin/python -m coverage run --pylib -m test --fail-env-changed -uall,-cpu -x test_multiprocessing_fork -x test_multiprocessing_forkserver -x test_multiprocessing_spawn -x test_concurrent_futures @@ -79,47 +71,60 @@ matrix: - source ./venv/bin/activate - bash <(curl -s https://codecov.io/bash) -# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. -before_script: + +before_install: + - set -e - | - set -e - if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then - files_changed=$(git diff --name-only $TRAVIS_COMMIT_RANGE) - else - # Pull requests are slightly complicated because merging the PR commit without - # rebasing causes it to retain its old commit date. Meaning in history if any - # commits have been made on master that post-date it, they will be accidentally - # included in the diff if we use the TRAVIS_COMMIT_RANGE variable. - files_changed=$(git diff --name-only HEAD $(git merge-base HEAD $TRAVIS_BRANCH)) + # Check short-circuit conditions + if [ "${TESTING}" != "docs" ] + then + if [ "$TRAVIS_PULL_REQUEST" = "false" ] + then + echo "Not a PR, doing full build." + else + # Pull requests are slightly complicated because $TRAVIS_COMMIT_RANGE + # may include more changes than desired if the history is convoluted. + # Instead, explicitly fetch the base branch and compare against the + # merge-base commit. + git fetch -q origin +refs/heads/$TRAVIS_BRANCH + changes=$(git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)) + echo "Files changed:" + echo "$changes" + if ! echo "$changes" | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' + then + echo "Only docs were updated, stopping build process." + exit + fi + fi fi - # Prints changed files in this commit to help debug doc-only build issues. - echo "Files changed: " - echo $files_changed - - if ! echo $files_changed | grep -qvE '(\.rst$)|(^Doc)|(^Misc)' +install: + - | + # Install OpenSSL as necessary + if [ "${TESTING}" != "docs" ] then - echo "Only docs were updated, stopping build process." - exit - fi - if [ "${TESTING}" != "docs" ]; then # clang complains about unused-parameter a lot, redirect stderr python3 Tools/ssl/multissltests.py --steps=library \ --base-directory ${HOME}/multissl \ --openssl ${OPENSSL} >/dev/null 2>&1 fi - openssl version - ./configure --with-pydebug - make -j4 - make -j4 regen-all clinic - changes=`git status --porcelain` + - openssl version + +# Travis provides only 2 cores, so don't overdo the parallelism and waste memory. +before_script: + - ./configure --with-pydebug + - make -j4 regen-all + - changes=`git status --porcelain` + - | + # Check for changes in regenerated files if ! test -z "$changes" then echo "Generated files not up to date" echo "$changes" exit 1 fi - make pythoninfo + - make -j4 + - make pythoninfo script: # Using the built Python as patchcheck.py is built around the idea of using @@ -127,10 +132,10 @@ script: # should be compared against. # Only run on Linux as the check only needs to be run once. - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then ./python Tools/scripts/patchcheck.py --travis $TRAVIS_PULL_REQUEST; fi - # `-r -w` implicitly provided through `make buildbottest`. - - make buildbottest TESTOPTS="-j4 -uall,-cpu" # Check that all symbols exported by libpython start with "Py" or "_Py" - make smelly + # `-r -w` implicitly provided through `make buildbottest`. + - make buildbottest TESTOPTS="-j4 -uall,-cpu" notifications: email: false diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index e4b48e66bc2d1c4..b41130ede416e39 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -151,19 +151,35 @@ which disallows mutable objects such as :class:`bytearray`. Previously, :exc:`TypeError` was raised when embedded null code points were encountered in the Python string. + .. deprecated-removed:: 3.3 4.0 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`PyUnicode_AsWideCharString`. + ``u#`` (:class:`str`) [const Py_UNICODE \*, int] This variant on ``u`` stores into two C variables, the first one a pointer to a Unicode data buffer, the second one its length. This variant allows null code points. + .. deprecated-removed:: 3.3 4.0 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`PyUnicode_AsWideCharString`. + ``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*] Like ``u``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to *NULL*. + .. deprecated-removed:: 3.3 4.0 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`PyUnicode_AsWideCharString`. + ``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, int] Like ``u#``, but the Python object may also be ``None``, in which case the :c:type:`Py_UNICODE` pointer is set to *NULL*. + .. deprecated-removed:: 3.3 4.0 + Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using + :c:func:`PyUnicode_AsWideCharString`. + ``U`` (:class:`str`) [PyObject \*] Requires that the Python object is a Unicode object, without attempting any conversion. Raises :exc:`TypeError` if the object is not a Unicode @@ -552,12 +568,13 @@ Building values ``z#`` (:class:`str` or ``None``) [const char \*, int] Same as ``s#``. - ``u`` (:class:`str`) [const Py_UNICODE \*] - Convert a null-terminated buffer of Unicode (UCS-2 or UCS-4) data to a Python - Unicode object. If the Unicode buffer pointer is *NULL*, ``None`` is returned. + ``u`` (:class:`str`) [const wchar_t \*] + Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) + data to a Python Unicode object. If the Unicode buffer pointer is *NULL*, + ``None`` is returned. - ``u#`` (:class:`str`) [const Py_UNICODE \*, int] - Convert a Unicode (UCS-2 or UCS-4) data buffer and its length to a Python + ``u#`` (:class:`str`) [const wchar_t \*, int] + Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python Unicode object. If the Unicode buffer pointer is *NULL*, the length is ignored and ``None`` is returned. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst index 8c2de9691f3e1a1..5a9a46fc67e8995 100644 --- a/Doc/c-api/buffer.rst +++ b/Doc/c-api/buffer.rst @@ -473,6 +473,15 @@ Buffer-related functions (*order* is ``'A'``). Return ``0`` otherwise. +.. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) + + Copy *len* bytes from *src* to its contiguous representation in *buf*. + *order* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). + ``0`` is returned on success, ``-1`` on error. + + This function fails if *len* != *src->len*. + + .. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if @@ -497,6 +506,3 @@ Buffer-related functions If this function is used as part of a :ref:`getbufferproc `, *exporter* MUST be set to the exporting object and *flags* must be passed unmodified. Otherwise, *exporter* MUST be NULL. - - - diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index bae49d5ba8125e6..694b4669eea8979 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -31,6 +31,9 @@ The following functions can be safely called before Python is initialized: * :c:func:`Py_SetProgramName` * :c:func:`Py_SetPythonHome` * :c:func:`Py_SetStandardStreamEncoding` + * :c:func:`PySys_AddWarnOption` + * :c:func:`PySys_AddXOption` + * :c:func:`PySys_ResetWarnOptions` * Informative functions: @@ -273,8 +276,8 @@ Initializing and finalizing the interpreter the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory allocated by the Python interpreter. This is a no-op when called for a second time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is 0. If there were errors during finalization - (flushing buffered data), -1 is returned. + return value is ``0``. If there were errors during finalization + (flushing buffered data), ``-1`` is returned. This function is provided for a number of reasons. An embedding application might want to restart Python without having to restart the application itself. @@ -1018,7 +1021,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. .. c:function:: PY_INT64_T PyInterpreterState_GetID(PyInterpreterState *interp) Return the interpreter's unique ID. If there was any error in doing - so then -1 is returned and an error is set. + so then ``-1`` is returned and an error is set. .. versionadded:: 3.7 diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst index 4f16b578eb5999a..5b1f386fb7e58fa 100644 --- a/Doc/c-api/long.rst +++ b/Doc/c-api/long.rst @@ -137,7 +137,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Raise :exc:`OverflowError` if the value of *obj* is out of range for a :c:type:`long`. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) @@ -151,7 +151,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long long PyLong_AsLongLong(PyObject *obj) @@ -166,7 +166,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Raise :exc:`OverflowError` if the value of *obj* is out of range for a :c:type:`long`. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) @@ -180,7 +180,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. versionadded:: 3.2 @@ -197,7 +197,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`Py_ssize_t`. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) @@ -259,7 +259,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long`, return the reduction of that value modulo ``ULONG_MAX + 1``. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) @@ -271,7 +271,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. If the value of *obj* is out of range for an :c:type:`unsigned long long`, return the reduction of that value modulo ``PY_ULLONG_MAX + 1``. - Returns -1 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: double PyLong_AsDouble(PyObject *pylong) @@ -282,7 +282,7 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. Raise :exc:`OverflowError` if the value of *pylong* is out of range for a :c:type:`double`. - Returns -1.0 on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns ``-1.0`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. .. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) @@ -292,4 +292,4 @@ distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. is only assured to produce a usable :c:type:`void` pointer for values created with :c:func:`PyLong_FromVoidPtr`. - Returns NULL on error. Use :c:func:`PyErr_Occurred` to disambiguate. + Returns *NULL* on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst index 2af0c46d451f056..73ae6db498cf2c7 100644 --- a/Doc/c-api/memory.rst +++ b/Doc/c-api/memory.rst @@ -530,7 +530,7 @@ tracemalloc C API Track an allocated memory block in the :mod:`tracemalloc` module. - Return 0 on success, return ``-1`` on error (failed to allocate memory to + Return ``0`` on success, return ``-1`` on error (failed to allocate memory to store the trace). Return ``-2`` if tracemalloc is disabled. If memory block is already tracked, update the existing trace. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 7efab28af724aa4..017b656854a8cd1 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -196,17 +196,23 @@ or request "multi-phase initialization" by returning the definition struct itsel .. c:member:: traverseproc m_traverse A traversal function to call during GC traversal of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: inquiry m_clear A clear function to call during GC clearing of the module object, or - *NULL* if not needed. + *NULL* if not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. .. c:member:: freefunc m_free A function to call during deallocation of the module object, or *NULL* if - not needed. + not needed. This function may be called before module state + is allocated (:c:func:`PyModule_GetState()` may return `NULL`), + and before the :c:member:`Py_mod_exec` function is executed. Single-phase initialization ........................... diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst index f1825f079be4746..81f8557ea6e665d 100644 --- a/Doc/c-api/sequence.rst +++ b/Doc/c-api/sequence.rst @@ -17,9 +17,8 @@ Sequence Protocol .. index:: builtin: len - Returns the number of objects in sequence *o* on success, and ``-1`` on failure. - For objects that do not provide sequence protocol, this is equivalent to the - Python expression ``len(o)``. + Returns the number of objects in sequence *o* on success, and ``-1`` on + failure. This is equivalent to the Python expression ``len(o)``. .. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index e4da96c493cd888..994509aa50f2ad3 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -205,16 +205,24 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_ResetWarnOptions() - Reset :data:`sys.warnoptions` to an empty list. + Reset :data:`sys.warnoptions` to an empty list. This function may be + called prior to :c:func:`Py_Initialize`. .. c:function:: void PySys_AddWarnOption(const wchar_t *s) - Append *s* to :data:`sys.warnoptions`. + Append *s* to :data:`sys.warnoptions`. This function must be called prior + to :c:func:`Py_Initialize` in order to affect the warnings filter list. .. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) Append *unicode* to :data:`sys.warnoptions`. + Note: this function is not currently usable from outside the CPython + implementation, as it must be called prior to the implicit import of + :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be + called until enough of the runtime has been initialized to permit the + creation of Unicode objects. + .. c:function:: void PySys_SetPath(const wchar_t *path) Set :data:`sys.path` to a list object of paths found in *path* which should @@ -260,7 +268,8 @@ accessible to C code. They all work with the current interpreter thread's .. c:function:: void PySys_AddXOption(const wchar_t *s) Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. + options mapping as returned by :c:func:`PySys_GetXOptions`. This function + may be called prior to :c:func:`Py_Initialize`. .. versionadded:: 3.2 diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst index 21f1acdace5f194..cd10a7fdf31513e 100644 --- a/Doc/distutils/configfile.rst +++ b/Doc/distutils/configfile.rst @@ -36,7 +36,9 @@ consequences: * installers can override anything in :file:`setup.cfg` using the command-line options to :file:`setup.py` -The basic syntax of the configuration file is simple:: +The basic syntax of the configuration file is simple: + +.. code-block:: ini [command] option=value @@ -51,9 +53,11 @@ option values can be split across multiple lines simply by indenting the continuation lines. You can find out the list of options supported by a particular command with the -universal :option:`!--help` option, e.g. :: +universal :option:`!--help` option, e.g. + +.. code-block:: shell-session - > python setup.py --help build_ext + $ python setup.py --help build_ext [...] Options for 'build_ext' command: --build-lib (-b) directory for compiled extension modules @@ -75,14 +79,18 @@ For example, say you want your extensions to be built "in-place"---that is, you have an extension :mod:`pkg.ext`, and you want the compiled extension file (:file:`ext.so` on Unix, say) to be put in the same source directory as your pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the -:option:`!--inplace` option on the command-line to ensure this:: +:option:`!--inplace` option on the command-line to ensure this: + +.. code-block:: sh python setup.py build_ext --inplace But this requires that you always specify the :command:`build_ext` command explicitly, and remember to provide :option:`!--inplace`. An easier way is to "set and forget" this option, by encoding it in :file:`setup.cfg`, the -configuration file for this distribution:: +configuration file for this distribution: + +.. code-block:: ini [build_ext] inplace=1 @@ -103,7 +111,9 @@ information comes from the setup script, and some is automatically generated by the Distutils (such as the list of files installed). But some of it has to be supplied as options to :command:`bdist_rpm`, which would be very tedious to do on the command-line for every run. Hence, here is a snippet from the Distutils' -own :file:`setup.cfg`:: +own :file:`setup.cfg`: + +.. code-block:: ini [bdist_rpm] release = 1 diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst index 44556e3df9c5b1b..086e14eb255a081 100644 --- a/Doc/distutils/packageindex.rst +++ b/Doc/distutils/packageindex.rst @@ -156,7 +156,9 @@ The :command:`register` and :command:`upload` commands both check for the existence of a :file:`.pypirc` file at the location :file:`$HOME/.pypirc`. If this file exists, the command uses the username, password, and repository URL configured in the file. The format of a :file:`.pypirc` file is as -follows:: +follows: + +.. code-block:: ini [distutils] index-servers = @@ -179,7 +181,9 @@ Each section describing a repository defines three variables: will be prompt to type it when needed. If you want to define another server a new section can be created and -listed in the *index-servers* variable:: +listed in the *index-servers* variable: + +.. code-block:: ini [distutils] index-servers = diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst index ab2f61614afc4ec..7e4fc19db83b2a6 100644 --- a/Doc/extending/embedding.rst +++ b/Doc/extending/embedding.rst @@ -323,7 +323,7 @@ options. In this case, the :mod:`sysconfig` module is a useful tool to programmatically extract the configuration values that you will want to combine together. For example: -.. code-block:: python +.. code-block:: pycon >>> import sysconfig >>> sysconfig.get_config_var('LIBS') diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst index e02f7837b69ed28..82b689e064c1d50 100644 --- a/Doc/extending/extending.rst +++ b/Doc/extending/extending.rst @@ -43,7 +43,9 @@ Let's create an extension module called ``spam`` (the favorite food of Monty Python fans...) and let's say we want to create a Python interface to the C library function :c:func:`system` [#]_. This function takes a null-terminated character string as argument and returns an integer. We want this function to -be callable from Python as follows:: +be callable from Python as follows: + +.. code-block:: pycon >>> import spam >>> status = spam.system("ls -l") @@ -439,7 +441,9 @@ part of the Python interpreter, you will have to change the configuration setup and rebuild the interpreter. Luckily, this is very simple on Unix: just place your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory of an unpacked source distribution, add a line to the file -:file:`Modules/Setup.local` describing your file:: +:file:`Modules/Setup.local` describing your file: + +.. code-block:: sh spam spammodule.o @@ -450,7 +454,9 @@ subdirectory, but then you must first rebuild :file:`Makefile` there by running :file:`Setup` file.) If your module requires additional libraries to link with, these can be listed -on the line in the configuration file as well, for instance:: +on the line in the configuration file as well, for instance: + +.. code-block:: sh spam spammodule.o -lX11 diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst index 80594e357fd497d..0994e3e8627dfae 100644 --- a/Doc/extending/index.rst +++ b/Doc/extending/index.rst @@ -26,9 +26,11 @@ Recommended third party tools ============================= This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like Cython, -``cffi``, SWIG and Numba offer both simpler and more sophisticated -approaches to creating C and C++ extensions for Python. +as part of this version of CPython. Third party tools like +`Cython `_, `cffi `_, +`SWIG `_ and `Numba `_ +offer both simpler and more sophisticated approaches to creating C and C++ +extensions for Python. .. seealso:: @@ -52,6 +54,7 @@ C extensions. :numbered: extending.rst + newtypes_tutorial.rst newtypes.rst building.rst windows.rst diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst index 62fbdb87a530007..d0d2ec1f88207c9 100644 --- a/Doc/extending/newtypes.rst +++ b/Doc/extending/newtypes.rst @@ -1,889 +1,11 @@ .. highlightlang:: c - -.. _defining-new-types: - -****************** -Defining New Types -****************** - -.. sectionauthor:: Michael Hudson -.. sectionauthor:: Dave Kuhlman -.. sectionauthor:: Jim Fulton - - -As mentioned in the last chapter, Python allows the writer of an extension -module to define new types that can be manipulated from Python code, much like -strings and lists in core Python. - -This is not hard; the code for all extension types follows a pattern, but there -are some details that you need to understand before you can get started. - - -.. _dnt-basics: - -The Basics -========== - -The Python runtime sees all Python objects as variables of type -:c:type:`PyObject\*`, which serves as a "base type" for all Python objects. -:c:type:`PyObject` itself only contains the refcount and a pointer to the -object's "type object". This is where the action is; the type object determines -which (C) functions get called when, for instance, an attribute gets looked -up on an object or it is multiplied by another object. These C functions -are called "type methods". - -So, if you want to define a new object type, you need to create a new type -object. - -This sort of thing can only be explained by example, so here's a minimal, but -complete, module that defines a new type: - -.. literalinclude:: ../includes/noddy.c - - -Now that's quite a bit to take in at once, but hopefully bits will seem familiar -from the last chapter. - -The first bit that will be new is:: - - typedef struct { - PyObject_HEAD - } noddy_NoddyObject; - -This is what a Noddy object will contain---in this case, nothing more than what -every Python object contains---a field called ``ob_base`` of type -:c:type:`PyObject`. :c:type:`PyObject` in turn, contains an ``ob_refcnt`` -field and a pointer to a type object. These can be accessed using the macros -:c:macro:`Py_REFCNT` and :c:macro:`Py_TYPE` respectively. These are the fields -the :c:macro:`PyObject_HEAD` macro brings in. The reason for the macro is to -standardize the layout and to enable special debugging fields in debug builds. - -Note that there is no semicolon after the :c:macro:`PyObject_HEAD` macro; -one is included in the macro definition. Be wary of adding one by -accident; it's easy to do from habit, and your compiler might not complain, -but someone else's probably will! (On Windows, MSVC is known to call this an -error and refuse to compile the code.) - -For contrast, let's take a look at the corresponding definition for standard -Python floats:: - - typedef struct { - PyObject_HEAD - double ob_fval; - } PyFloatObject; - -Moving on, we come to the crunch --- the type object. :: - - static PyTypeObject noddy_NoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "noddy.Noddy", /* tp_name */ - sizeof(noddy_NoddyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - }; - -Now if you go and look up the definition of :c:type:`PyTypeObject` in -:file:`object.h` you'll see that it has many more fields that the definition -above. The remaining fields will be filled with zeros by the C compiler, and -it's common practice to not specify them explicitly unless you need them. - -This is so important that we're going to pick the top of it apart still -further:: - - PyVarObject_HEAD_INIT(NULL, 0) - -This line is a bit of a wart; what we'd like to write is:: - - PyVarObject_HEAD_INIT(&PyType_Type, 0) - -as the type of a type object is "type", but this isn't strictly conforming C and -some compilers complain. Fortunately, this member will be filled in for us by -:c:func:`PyType_Ready`. :: - - "noddy.Noddy", /* tp_name */ - -The name of our type. This will appear in the default textual representation of -our objects and in some error messages, for example:: - - >>> "" + noddy.new_noddy() - Traceback (most recent call last): - File "", line 1, in - TypeError: cannot add type "noddy.Noddy" to string - -Note that the name is a dotted name that includes both the module name and the -name of the type within the module. The module in this case is :mod:`noddy` and -the type is :class:`Noddy`, so we set the type name to :class:`noddy.Noddy`. -One side effect of using an undotted name is that the pydoc documentation tool -will not list the new type in the module documentation. :: - - sizeof(noddy_NoddyObject), /* tp_basicsize */ - -This is so that Python knows how much memory to allocate when you call -:c:func:`PyObject_New`. - -.. note:: - - If you want your type to be subclassable from Python, and your type has the same - :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple - inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's - :meth:`__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its - base type does. Most of the time, this will be true anyway, because either your - base type will be :class:`object`, or else you will be adding data members to - your base type, and therefore increasing its size. - -:: - - 0, /* tp_itemsize */ - -This has to do with variable length objects like lists and strings. Ignore this -for now. - -Skipping a number of type methods that we don't provide, we set the class flags -to :const:`Py_TPFLAGS_DEFAULT`. :: - - Py_TPFLAGS_DEFAULT, /* tp_flags */ - -All types should include this constant in their flags. It enables all of the -members defined until at least Python 3.3. If you need further members, -you will need to OR the corresponding flags. - -We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: - - "Noddy objects", /* tp_doc */ - -Now we get into the type methods, the things that make your objects different -from the others. We aren't going to implement any of these in this version of -the module. We'll expand this example later to have more interesting behavior. - -For now, all we want to be able to do is to create new :class:`Noddy` objects. -To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` implementation. -In this case, we can just use the default implementation provided by the API -function :c:func:`PyType_GenericNew`. :: - - PyType_GenericNew, /* tp_new */ - -All the other type methods are *NULL*, so we'll go over them later --- that's -for a later section! - -Everything else in the file should be familiar, except for some code in -:c:func:`PyInit_noddy`:: - - if (PyType_Ready(&noddy_NoddyType) < 0) - return; - -This initializes the :class:`Noddy` type, filing in a number of members, -including :attr:`ob_type` that we initially set to *NULL*. :: - - PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType); - -This adds the type to the module dictionary. This allows us to create -:class:`Noddy` instances by calling the :class:`Noddy` class:: - - >>> import noddy - >>> mynoddy = noddy.Noddy() - -That's it! All that remains is to build it; put the above code in a file called -:file:`noddy.c` and :: - - from distutils.core import setup, Extension - setup(name="noddy", version="1.0", - ext_modules=[Extension("noddy", ["noddy.c"])]) - -in a file called :file:`setup.py`; then typing - -.. code-block:: shell-session - - $ python setup.py build - -at a shell should produce a file :file:`noddy.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import noddy`` and -play around with Noddy objects. - -That wasn't so hard, was it? - -Of course, the current Noddy type is pretty uninteresting. It has no data and -doesn't do anything. It can't even be subclassed. - - -Adding data and methods to the Basic example --------------------------------------------- - -Let's extend the basic example to add some data and methods. Let's also make -the type usable as a base class. We'll create a new module, :mod:`noddy2` that -adds these capabilities: - -.. literalinclude:: ../includes/noddy2.c - - -This version of the module has a number of changes. - -We've added an extra include:: - - #include - -This include provides declarations that we use to handle attributes, as -described a bit later. - -The name of the :class:`Noddy` object structure has been shortened to -:class:`Noddy`. The type object name has been shortened to :class:`NoddyType`. - -The :class:`Noddy` type now has three data attributes, *first*, *last*, and -*number*. The *first* and *last* variables are Python strings containing first -and last names. The *number* attribute is an integer. - -The object structure is updated accordingly:: - - typedef struct { - PyObject_HEAD - PyObject *first; - PyObject *last; - int number; - } Noddy; - -Because we now have data to manage, we have to be more careful about object -allocation and deallocation. At a minimum, we need a deallocation method:: - - static void - Noddy_dealloc(Noddy* self) - { - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject*)self); - } - -which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: - - (destructor)Noddy_dealloc, /*tp_dealloc*/ - -This method decrements the reference counts of the two Python attributes. We use -:c:func:`Py_XDECREF` here because the :attr:`first` and :attr:`last` members -could be *NULL*. It then calls the :c:member:`~PyTypeObject.tp_free` member of the object's type -to free the object's memory. Note that the object's type might not be -:class:`NoddyType`, because the object may be an instance of a subclass. - -We want to make sure that the first and last names are initialized to empty -strings, so we provide a new method:: - - static PyObject * - Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) - { - Noddy *self; - - self = (Noddy *)type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - - self->number = 0; - } - - return (PyObject *)self; - } - -and install it in the :c:member:`~PyTypeObject.tp_new` member:: - - Noddy_new, /* tp_new */ - -The new member is responsible for creating (as opposed to initializing) objects -of the type. It is exposed in Python as the :meth:`__new__` method. See the -paper titled "Unifying types and classes in Python" for a detailed discussion of -the :meth:`__new__` method. One reason to implement a new method is to assure -the initial values of instance variables. In this case, we use the new method -to make sure that the initial values of the members :attr:`first` and -:attr:`last` are not *NULL*. If we didn't care whether the initial values were -*NULL*, we could have used :c:func:`PyType_GenericNew` as our new method, as we -did before. :c:func:`PyType_GenericNew` initializes all of the instance variable -members to *NULL*. - -The new method is a static method that is passed the type being instantiated and -any arguments passed when the type was called, and that returns the new object -created. New methods always accept positional and keyword arguments, but they -often ignore the arguments, leaving the argument handling to initializer -methods. Note that if the type supports subclassing, the type passed may not be -the type being defined. The new method calls the :c:member:`~PyTypeObject.tp_alloc` slot to -allocate memory. We don't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather -:c:func:`PyType_Ready` fills it for us by inheriting it from our base class, -which is :class:`object` by default. Most types use the default allocation. - -.. note:: - - If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one that calls a base type's - :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), you must *not* try to determine what method - to call using method resolution order at runtime. Always statically determine - what type you are going to call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via - ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your - type that also inherit from other Python-defined classes may not work correctly. - (Specifically, you may not be able to create instances of such subclasses - without getting a :exc:`TypeError`.) - -We provide an initialization function:: - - static int - Noddy_init(Noddy *self, PyObject *args, PyObject *kwds) - { - PyObject *first=NULL, *last=NULL, *tmp; - - static char *kwlist[] = {"first", "last", "number", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - - return 0; - } - -by filling the :c:member:`~PyTypeObject.tp_init` slot. :: - - (initproc)Noddy_init, /* tp_init */ - -The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the :meth:`__init__` method. It -is used to initialize an object after it's created. Unlike the new method, we -can't guarantee that the initializer is called. The initializer isn't called -when unpickling objects and it can be overridden. Our initializer accepts -arguments to provide initial values for our instance. Initializers always accept -positional and keyword arguments. Initializers should return either 0 on -success or -1 on error. - -Initializers can be called multiple times. Anyone can call the :meth:`__init__` -method on our objects. For this reason, we have to be extra careful when -assigning the new values. We might be tempted, for example to assign the -:attr:`first` member like this:: - - if (first) { - Py_XDECREF(self->first); - Py_INCREF(first); - self->first = first; - } - -But this would be risky. Our type doesn't restrict the type of the -:attr:`first` member, so it could be any kind of object. It could have a -destructor that causes code to be executed that tries to access the -:attr:`first` member. To be paranoid and protect ourselves against this -possibility, we almost always reassign members before decrementing their -reference counts. When don't we have to do this? - -* when we absolutely know that the reference count is greater than 1 - -* when we know that deallocation of the object [#]_ will not cause any calls - back into our type's code - -* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` handler when - garbage-collections is not supported [#]_ - -We want to expose our instance variables as attributes. There are a -number of ways to do that. The simplest way is to define member definitions:: - - static PyMemberDef Noddy_members[] = { - {"first", T_OBJECT_EX, offsetof(Noddy, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(Noddy, last), 0, - "last name"}, - {"number", T_INT, offsetof(Noddy, number), 0, - "noddy number"}, - {NULL} /* Sentinel */ - }; - -and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: - - Noddy_members, /* tp_members */ - -Each member definition has a member name, type, offset, access flags and -documentation string. See the :ref:`Generic-Attribute-Management` section below for -details. - -A disadvantage of this approach is that it doesn't provide a way to restrict the -types of objects that can be assigned to the Python attributes. We expect the -first and last names to be strings, but any Python objects can be assigned. -Further, the attributes can be deleted, setting the C pointers to *NULL*. Even -though we can make sure the members are initialized to non-*NULL* values, the -members can be set to *NULL* if the attributes are deleted. - -We define a single method, :meth:`name`, that outputs the objects name as the -concatenation of the first and last names. :: - - static PyObject * - Noddy_name(Noddy* self) - { - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - - return PyUnicode_FromFormat("%S %S", self->first, self->last); - } - -The method is implemented as a C function that takes a :class:`Noddy` (or -:class:`Noddy` subclass) instance as the first argument. Methods always take an -instance as the first argument. Methods often take positional and keyword -arguments as well, but in this case we don't take any and don't need to accept -a positional argument tuple or keyword argument dictionary. This method is -equivalent to the Python method:: - - def name(self): - return "%s %s" % (self.first, self.last) - -Note that we have to check for the possibility that our :attr:`first` and -:attr:`last` members are *NULL*. This is because they can be deleted, in which -case they are set to *NULL*. It would be better to prevent deletion of these -attributes and to restrict the attribute values to be strings. We'll see how to -do that in the next section. - -Now that we've defined the method, we need to create an array of method -definitions:: - - static PyMethodDef Noddy_methods[] = { - {"name", (PyCFunction)Noddy_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ - }; - -and assign them to the :c:member:`~PyTypeObject.tp_methods` slot:: - - Noddy_methods, /* tp_methods */ - -Note that we used the :const:`METH_NOARGS` flag to indicate that the method is -passed no arguments. - -Finally, we'll make our type usable as a base class. We've written our methods -carefully so far so that they don't make any assumptions about the type of the -object being created or used, so all we need to do is to add the -:const:`Py_TPFLAGS_BASETYPE` to our class flag definition:: - - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ - -We rename :c:func:`PyInit_noddy` to :c:func:`PyInit_noddy2` and update the module -name in the :c:type:`PyModuleDef` struct. - -Finally, we update our :file:`setup.py` file to build the new module:: - - from distutils.core import setup, Extension - setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - ]) - - -Providing finer control over data attributes --------------------------------------------- - -In this section, we'll provide finer control over how the :attr:`first` and -:attr:`last` attributes are set in the :class:`Noddy` example. In the previous -version of our module, the instance variables :attr:`first` and :attr:`last` -could be set to non-string values or even deleted. We want to make sure that -these attributes always contain strings. - -.. literalinclude:: ../includes/noddy3.c - - -To provide greater control, over the :attr:`first` and :attr:`last` attributes, -we'll use custom getter and setter functions. Here are the functions for -getting and setting the :attr:`first` attribute:: - - Noddy_getfirst(Noddy *self, void *closure) - { - Py_INCREF(self->first); - return self->first; - } - - static int - Noddy_setfirst(Noddy *self, PyObject *value, void *closure) - { - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - - if (! PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a str"); - return -1; - } - - Py_DECREF(self->first); - Py_INCREF(value); - self->first = value; - - return 0; - } - -The getter function is passed a :class:`Noddy` object and a "closure", which is -void pointer. In this case, the closure is ignored. (The closure supports an -advanced usage in which definition data is passed to the getter and setter. This -could, for example, be used to allow a single set of getter and setter functions -that decide the attribute to get or set based on data in the closure.) - -The setter function is passed the :class:`Noddy` object, the new value, and the -closure. The new value may be *NULL*, in which case the attribute is being -deleted. In our setter, we raise an error if the attribute is deleted or if the -attribute value is not a string. - -We create an array of :c:type:`PyGetSetDef` structures:: - - static PyGetSetDef Noddy_getseters[] = { - {"first", - (getter)Noddy_getfirst, (setter)Noddy_setfirst, - "first name", - NULL}, - {"last", - (getter)Noddy_getlast, (setter)Noddy_setlast, - "last name", - NULL}, - {NULL} /* Sentinel */ - }; - -and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: - - Noddy_getseters, /* tp_getset */ - -to register our attribute getters and setters. - -The last item in a :c:type:`PyGetSetDef` structure is the closure mentioned -above. In this case, we aren't using the closure, so we just pass *NULL*. - -We also remove the member definitions for these attributes:: - - static PyMemberDef Noddy_members[] = { - {"number", T_INT, offsetof(Noddy, number), 0, - "noddy number"}, - {NULL} /* Sentinel */ - }; - -We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only allow strings [#]_ to -be passed:: - - static int - Noddy_init(Noddy *self, PyObject *args, PyObject *kwds) - { - PyObject *first=NULL, *last=NULL, *tmp; - - static char *kwlist[] = {"first", "last", "number", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - - return 0; - } - -With these changes, we can assure that the :attr:`first` and :attr:`last` -members are never *NULL* so we can remove checks for *NULL* values in almost all -cases. This means that most of the :c:func:`Py_XDECREF` calls can be converted to -:c:func:`Py_DECREF` calls. The only place we can't change these calls is in the -deallocator, where there is the possibility that the initialization of these -members failed in the constructor. - -We also rename the module initialization function and module name in the -initialization function, as we did before, and we add an extra definition to the -:file:`setup.py` file. - - -Supporting cyclic garbage collection ------------------------------------- - -Python has a cyclic-garbage collector that can identify unneeded objects even -when their reference counts are not zero. This can happen when objects are -involved in cycles. For example, consider:: - - >>> l = [] - >>> l.append(l) - >>> del l - -In this example, we create a list that contains itself. When we delete it, it -still has a reference from itself. Its reference count doesn't drop to zero. -Fortunately, Python's cyclic-garbage collector will eventually figure out that -the list is garbage and free it. - -In the second version of the :class:`Noddy` example, we allowed any kind of -object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. This -means that :class:`Noddy` objects can participate in cycles:: - - >>> import noddy2 - >>> n = noddy2.Noddy() - >>> l = [n] - >>> n.first = l - -This is pretty silly, but it gives us an excuse to add support for the -cyclic-garbage collector to the :class:`Noddy` example. To support cyclic -garbage collection, types need to fill two slots and set a class flag that -enables these slots: - -.. literalinclude:: ../includes/noddy4.c - - -The traversal method provides access to subobjects that could participate in -cycles:: - - static int - Noddy_traverse(Noddy *self, visitproc visit, void *arg) - { - int vret; - - if (self->first) { - vret = visit(self->first, arg); - if (vret != 0) - return vret; - } - if (self->last) { - vret = visit(self->last, arg); - if (vret != 0) - return vret; - } - - return 0; - } - -For each subobject that can participate in cycles, we need to call the -:c:func:`visit` function, which is passed to the traversal method. The -:c:func:`visit` function takes as arguments the subobject and the extra argument -*arg* passed to the traversal method. It returns an integer value that must be -returned if it is non-zero. - -Python provides a :c:func:`Py_VISIT` macro that automates calling visit -functions. With :c:func:`Py_VISIT`, :c:func:`Noddy_traverse` can be simplified:: - - static int - Noddy_traverse(Noddy *self, visitproc visit, void *arg) - { - Py_VISIT(self->first); - Py_VISIT(self->last); - return 0; - } - -.. note:: - - Note that the :c:member:`~PyTypeObject.tp_traverse` implementation must name its arguments exactly - *visit* and *arg* in order to use :c:func:`Py_VISIT`. This is to encourage - uniformity across these boring implementations. - -We also need to provide a method for clearing any subobjects that can -participate in cycles. - -:: - - static int - Noddy_clear(Noddy *self) - { - PyObject *tmp; - - tmp = self->first; - self->first = NULL; - Py_XDECREF(tmp); - - tmp = self->last; - self->last = NULL; - Py_XDECREF(tmp); - - return 0; - } - -Notice the use of a temporary variable in :c:func:`Noddy_clear`. We use the -temporary variable so that we can set each member to *NULL* before decrementing -its reference count. We do this because, as was discussed earlier, if the -reference count drops to zero, we might cause code to run that calls back into -the object. In addition, because we now support garbage collection, we also -have to worry about code being run that triggers garbage collection. If garbage -collection is run, our :c:member:`~PyTypeObject.tp_traverse` handler could get called. We can't -take a chance of having :c:func:`Noddy_traverse` called when a member's reference -count has dropped to zero and its value hasn't been set to *NULL*. - -Python provides a :c:func:`Py_CLEAR` that automates the careful decrementing of -reference counts. With :c:func:`Py_CLEAR`, the :c:func:`Noddy_clear` function can -be simplified:: - - static int - Noddy_clear(Noddy *self) - { - Py_CLEAR(self->first); - Py_CLEAR(self->last); - return 0; - } - -Note that :c:func:`Noddy_dealloc` may call arbitrary functions through -``__del__`` method or weakref callback. It means circular GC can be -triggered inside the function. Since GC assumes reference count is not zero, -we need to untrack the object from GC by calling :c:func:`PyObject_GC_UnTrack` -before clearing members. Here is reimplemented deallocator which uses -:c:func:`PyObject_GC_UnTrack` and :c:func:`Noddy_clear`. - -:: - - static void - Noddy_dealloc(Noddy* self) - { - PyObject_GC_UnTrack(self); - Noddy_clear(self); - Py_TYPE(self)->tp_free((PyObject*)self); - } - -Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: - - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - -That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or -:c:member:`~PyTypeObject.tp_free` slots, we'd need to modify them for cyclic-garbage collection. -Most extensions will use the versions automatically provided. - - -Subclassing other types ------------------------ - -It is possible to create new extension types that are derived from existing -types. It is easiest to inherit from the built in types, since an extension can -easily use the :class:`PyTypeObject` it needs. It can be difficult to share -these :class:`PyTypeObject` structures between extension modules. - -In this example we will create a :class:`Shoddy` type that inherits from the -built-in :class:`list` type. The new type will be completely compatible with -regular lists, but will have an additional :meth:`increment` method that -increases an internal counter. :: - - >>> import shoddy - >>> s = shoddy.Shoddy(range(3)) - >>> s.extend(s) - >>> print(len(s)) - 6 - >>> print(s.increment()) - 1 - >>> print(s.increment()) - 2 - -.. literalinclude:: ../includes/shoddy.c - - -As you can see, the source code closely resembles the :class:`Noddy` examples in -previous sections. We will break down the main differences between them. :: - - typedef struct { - PyListObject list; - int state; - } Shoddy; - -The primary difference for derived type objects is that the base type's object -structure must be the first value. The base type will already include the -:c:func:`PyObject_HEAD` at the beginning of its structure. - -When a Python object is a :class:`Shoddy` instance, its *PyObject\** pointer can -be safely cast to both *PyListObject\** and *Shoddy\**. :: - - static int - Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) - { - if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; - } - -In the :attr:`__init__` method for our type, we can see how to call through to -the :attr:`__init__` method of the base type. - -This pattern is important when writing a type with custom :attr:`new` and -:attr:`dealloc` methods. The :attr:`new` method should not actually create the -memory for the object with :c:member:`~PyTypeObject.tp_alloc`, that will be handled by the base -class when calling its :c:member:`~PyTypeObject.tp_new`. - -When filling out the :c:func:`PyTypeObject` for the :class:`Shoddy` type, you see -a slot for :c:func:`tp_base`. Due to cross platform compiler issues, you can't -fill that field directly with the :c:func:`PyList_Type`; it can be done later in -the module's :c:func:`init` function. :: - - PyMODINIT_FUNC - PyInit_shoddy(void) - { - PyObject *m; - - ShoddyType.tp_base = &PyList_Type; - if (PyType_Ready(&ShoddyType) < 0) - return NULL; - - m = PyModule_Create(&shoddymodule); - if (m == NULL) - return NULL; - - Py_INCREF(&ShoddyType); - PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType); - return m; - } - -Before calling :c:func:`PyType_Ready`, the type structure must have the -:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving a new type, it is not -necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` slot with :c:func:`PyType_GenericNew` --- the allocate function from the base type will be inherited. - -After that, calling :c:func:`PyType_Ready` and adding the type object to the -module is the same as with the basic :class:`Noddy` examples. - +***************************************** +Defining Extension Types: Assorted Topics +***************************************** .. _dnt-type-methods: -Type Methods -============ - This section aims to give a quick fly-by on the various type methods you can implement and what they do. @@ -893,21 +15,20 @@ debug builds omitted: .. literalinclude:: ../includes/typestruct.h -Now that's a *lot* of methods. Don't worry too much though - if you have a type -you want to define, the chances are very good that you will only implement a -handful of these. +Now that's a *lot* of methods. Don't worry too much though -- if you have +a type you want to define, the chances are very good that you will only +implement a handful of these. As you probably expect by now, we're going to go over this and give more information about the various handlers. We won't go in the order they are defined in the structure, because there is a lot of historical baggage that -impacts the ordering of the fields; be sure your type initialization keeps the -fields in the right order! It's often easiest to find an example that includes -all the fields you need (even if they're initialized to ``0``) and then change -the values to suit your new type. :: +impacts the ordering of the fields. It's often easiest to find an example +that includes the fields you need and then change the values to suit your new +type. :: const char *tp_name; /* For printing */ -The name of the type - as mentioned in the last section, this will appear in +The name of the type -- as mentioned in the previous chapter, this will appear in various places, almost entirely for diagnostic purposes. Try to choose something that will be helpful in such a situation! :: @@ -915,7 +36,7 @@ that will be helpful in such a situation! :: These fields tell the runtime how much memory to allocate when new objects of this type are created. Python has some built-in support for variable length -structures (think: strings, lists) which is where the :c:member:`~PyTypeObject.tp_itemsize` field +structures (think: strings, tuples) which is where the :c:member:`~PyTypeObject.tp_itemsize` field comes in. This will be dealt with later. :: const char *tp_doc; @@ -923,7 +44,7 @@ comes in. This will be dealt with later. :: Here you can put a string (or its address) that you want returned when the Python script references ``obj.__doc__`` to retrieve the doc string. -Now we come to the basic type methods---the ones most extension types will +Now we come to the basic type methods -- the ones most extension types will implement. @@ -947,7 +68,7 @@ object itself needs to be freed here as well. Here is an example of this function:: static void - newdatatype_dealloc(newdatatypeobject * obj) + newdatatype_dealloc(newdatatypeobject *obj) { free(obj->obj_UnderlyingDatatypePtr); Py_TYPE(obj)->tp_free(obj); @@ -1035,7 +156,7 @@ example:: static PyObject * newdatatype_repr(newdatatypeobject * obj) { - return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:\%d}}", + return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); } @@ -1055,7 +176,7 @@ Here is a simple example:: static PyObject * newdatatype_str(newdatatypeobject * obj) { - return PyUnicode_FromFormat("Stringified_newdatatype{{size:\%d}}", + return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", obj->obj_UnderlyingDatatypePtr->size); } @@ -1236,7 +357,7 @@ example that simply raises an exception; if this were really all you wanted, the static int newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) { - (void)PyErr_Format(PyExc_RuntimeError, "Read-only attribute: \%s", name); + PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name); return -1; } @@ -1321,17 +442,23 @@ these in the :file:`Objects` directory of the Python source distribution. :: hashfunc tp_hash; This function, if you choose to provide it, should return a hash number for an -instance of your data type. Here is a moderately pointless example:: +instance of your data type. Here is a simple example:: - static long + static Py_hash_t newdatatype_hash(newdatatypeobject *obj) { - long result; - result = obj->obj_UnderlyingDatatypePtr->size; - result = result * 3; + Py_hash_t result; + result = obj->some_size + 32767 * obj->some_number; + if (result == -1) + result = -2; return result; } +:c:type:`Py_hash_t` is a signed integer type with a platform-varying width. +Returning ``-1`` from :c:member:`~PyTypeObject.tp_hash` indicates an error, +which is why you should be careful to avoid returning it when hash computation +is successful, as seen above. + :: ternaryfunc tp_call; @@ -1342,27 +469,22 @@ contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is inv This function takes three arguments: -#. *arg1* is the instance of the data type which is the subject of the call. If - the call is ``obj1('hello')``, then *arg1* is ``obj1``. +#. *self* is the instance of the data type which is the subject of the call. + If the call is ``obj1('hello')``, then *self* is ``obj1``. -#. *arg2* is a tuple containing the arguments to the call. You can use +#. *args* is a tuple containing the arguments to the call. You can use :c:func:`PyArg_ParseTuple` to extract the arguments. -#. *arg3* is a dictionary of keyword arguments that were passed. If this is +#. *kwds* is a dictionary of keyword arguments that were passed. If this is non-*NULL* and you support keyword arguments, use - :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you do not - want to support keyword arguments and this is non-*NULL*, raise a + :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you + do not want to support keyword arguments and this is non-*NULL*, raise a :exc:`TypeError` with a message saying that keyword arguments are not supported. -Here is a desultory example of the implementation of the call function. :: +Here is a toy ``tp_call`` implementation:: - /* Implement the call function. - * obj1 is the instance receiving the call. - * obj2 is a tuple containing the arguments to the call, in this - * case 3 strings. - */ static PyObject * - newdatatype_call(newdatatypeobject *obj, PyObject *args, PyObject *other) + newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds) { PyObject *result; const char *arg1; @@ -1373,7 +495,7 @@ Here is a desultory example of the implementation of the call function. :: return NULL; } result = PyUnicode_FromFormat( - "Returning -- value: [\%d] arg1: [\%s] arg2: [\%s] arg3: [\%s]\n", + "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n", obj->obj_UnderlyingDatatypePtr->size, arg1, arg2, arg3); return result; @@ -1385,32 +507,36 @@ Here is a desultory example of the implementation of the call function. :: getiterfunc tp_iter; iternextfunc tp_iternext; -These functions provide support for the iterator protocol. Any object which -wishes to support iteration over its contents (which may be generated during -iteration) must implement the ``tp_iter`` handler. Objects which are returned -by a ``tp_iter`` handler must implement both the ``tp_iter`` and ``tp_iternext`` -handlers. Both handlers take exactly one parameter, the instance for which they -are being called, and return a new reference. In the case of an error, they -should set an exception and return *NULL*. - -For an object which represents an iterable collection, the ``tp_iter`` handler -must return an iterator object. The iterator object is responsible for -maintaining the state of the iteration. For collections which can support -multiple iterators which do not interfere with each other (as lists and tuples -do), a new iterator should be created and returned. Objects which can only be -iterated over once (usually due to side effects of iteration) should implement -this handler by returning a new reference to themselves, and should also -implement the ``tp_iternext`` handler. File objects are an example of such an -iterator. - -Iterator objects should implement both handlers. The ``tp_iter`` handler should -return a new reference to the iterator (this is the same as the ``tp_iter`` -handler for objects which can only be iterated over destructively). The -``tp_iternext`` handler should return a new reference to the next object in the -iteration if there is one. If the iteration has reached the end, it may return -*NULL* without setting an exception or it may set :exc:`StopIteration`; avoiding -the exception can yield slightly better performance. If an actual error occurs, -it should set an exception and return *NULL*. +These functions provide support for the iterator protocol. Both handlers +take exactly one parameter, the instance for which they are being called, +and return a new reference. In the case of an error, they should set an +exception and return *NULL*. :c:member:`~PyTypeObject.tp_iter` corresponds +to the Python :meth:`__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` +corresponds to the Python :meth:`~iterator.__next__` method. + +Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter` +handler, which must return an :term:`iterator` object. Here the same guidelines +apply as for Python classes: + +* For collections (such as lists and tuples) which can support multiple + independent iterators, a new iterator should be created and returned by + each call to :c:member:`~PyTypeObject.tp_iter`. +* Objects which can only be iterated over once (usually due to side effects of + iteration, such as file objects) can implement :c:member:`~PyTypeObject.tp_iter` + by returning a new reference to themselves -- and should also therefore + implement the :c:member:`~PyTypeObject.tp_iternext` handler. + +Any :term:`iterator` object should implement both :c:member:`~PyTypeObject.tp_iter` +and :c:member:`~PyTypeObject.tp_iternext`. An iterator's +:c:member:`~PyTypeObject.tp_iter` handler should return a new reference +to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should +return a new reference to the next object in the iteration, if there is one. +If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext` +may return *NULL* without setting an exception, or it may set +:exc:`StopIteration` *in addition* to returning *NULL*; avoiding +the exception can yield slightly better performance. If an actual error +occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception +and return *NULL*. .. _weakref-support: @@ -1418,110 +544,76 @@ it should set an exception and return *NULL*. Weak Reference Support ---------------------- -One of the goals of Python's weak-reference implementation is to allow any type +One of the goals of Python's weak reference implementation is to allow any type to participate in the weak reference mechanism without incurring the overhead on -those objects which do not benefit by weak referencing (such as numbers). +performance-critical objects (such as numbers). -For an object to be weakly referencable, the extension must include a -:c:type:`PyObject\*` field in the instance structure for the use of the weak -reference mechanism; it must be initialized to *NULL* by the object's -constructor. It must also set the :c:member:`~PyTypeObject.tp_weaklistoffset` field of the -corresponding type object to the offset of the field. For example, the instance -type is defined with the following structure:: +.. seealso:: + Documentation for the :mod:`weakref` module. - typedef struct { - PyObject_HEAD - PyClassObject *in_class; /* The class object */ - PyObject *in_dict; /* A dictionary */ - PyObject *in_weakreflist; /* List of weak references */ - } PyInstanceObject; - -The statically-declared type object for instances is defined this way:: - - PyTypeObject PyInstance_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - 0, - "module.instance", - - /* Lots of stuff omitted for brevity... */ - - Py_TPFLAGS_DEFAULT, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(PyInstanceObject, in_weakreflist), /* tp_weaklistoffset */ - }; +For an object to be weakly referencable, the extension type must do two things: -The type constructor is responsible for initializing the weak reference list to -*NULL*:: +#. Include a :c:type:`PyObject\*` field in the C object structure dedicated to + the weak reference mechanism. The object's constructor should leave it + *NULL* (which is automatic when using the default + :c:member:`~PyTypeObject.tp_alloc`). - static PyObject * - instance_new() { - /* Other initialization stuff omitted for brevity */ +#. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member + to the offset of the aforementioned field in the C object structure, + so that the interpreter knows how to access and modify that field. - self->in_weakreflist = NULL; +Concretely, here is how a trivial object structure would be augmented +with the required field:: - return (PyObject *) self; - } + typedef struct { + PyObject_HEAD + PyObject *weakreflist; /* List of weak references */ + } TrivialObject; -The only further addition is that the destructor needs to call the weak -reference manager to clear any weak references. This is only required if the -weak reference list is non-*NULL*:: +And the corresponding member in the statically-declared type object:: - static void - instance_dealloc(PyInstanceObject *inst) - { - /* Allocate temporaries if needed, but do not begin - destruction just yet. - */ + static PyTypeObject TrivialType = { + PyVarObject_HEAD_INIT(NULL, 0) + /* ... other members omitted for brevity ... */ + .tp_weaklistoffset = offsetof(TrivialObject, weakreflist), + }; - if (inst->in_weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) inst); +The only further addition is that ``tp_dealloc`` needs to clear any weak +references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is +non-*NULL*:: - /* Proceed with object destruction normally. */ + static void + Trivial_dealloc(TrivialObject *self) + { + /* Clear weakrefs first before calling any destructors */ + if (self->weakreflist != NULL) + PyObject_ClearWeakRefs((PyObject *) self); + /* ... remainder of destruction code omitted for brevity ... */ + Py_TYPE(self)->tp_free((PyObject *) self); } More Suggestions ---------------- -Remember that you can omit most of these functions, in which case you provide -``0`` as a value. There are type definitions for each of the functions you must -provide. They are in :file:`object.h` in the Python include directory that -comes with the source distribution of Python. - In order to learn how to implement any specific method for your new data type, -do the following: Download and unpack the Python source distribution. Go to -the :file:`Objects` directory, then search the C source files for ``tp_`` plus -the function you want (for example, ``tp_richcompare``). You will find examples -of the function you want to implement. +get the :term:`CPython` source code. Go to the :file:`Objects` directory, +then search the C source files for ``tp_`` plus the function you want +(for example, ``tp_richcompare``). You will find examples of the function +you want to implement. -When you need to verify that an object is an instance of the type you are -implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of its use -might be something like the following:: +When you need to verify that an object is a concrete instance of the type you +are implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of +its use might be something like the following:: - if (! PyObject_TypeCheck(some_object, &MyType)) { + if (!PyObject_TypeCheck(some_object, &MyType)) { PyErr_SetString(PyExc_TypeError, "arg #1 not a mything"); return NULL; } -.. rubric:: Footnotes - -.. [#] This is true when we know that the object is a basic type, like a string or a - float. - -.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler in this example, because our - type doesn't support garbage collection. Even if a type supports garbage - collection, there are calls that can be made to "untrack" the object from - garbage collection, however, these calls are advanced and not covered here. - -.. [#] We now know that the first and last members are strings, so perhaps we could be - less careful about decrementing their reference counts, however, we accept - instances of string subclasses. Even though deallocating normal strings won't - call back into our objects, we can't guarantee that deallocating an instance of - a string subclass won't call back into our objects. +.. seealso:: + Download CPython source releases. + https://www.python.org/downloads/source/ -.. [#] Even in the third version, we aren't guaranteed to avoid cycles. Instances of - string subclasses are allowed and string subclasses could allow cycles even if - normal strings don't. + The CPython project on GitHub, where the CPython source code is developed. + https://github.com/python/cpython diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst new file mode 100644 index 000000000000000..ac48637bbee9bea --- /dev/null +++ b/Doc/extending/newtypes_tutorial.rst @@ -0,0 +1,896 @@ +.. highlightlang:: c + +.. _defining-new-types: + +********************************** +Defining Extension Types: Tutorial +********************************** + +.. sectionauthor:: Michael Hudson +.. sectionauthor:: Dave Kuhlman +.. sectionauthor:: Jim Fulton + + +Python allows the writer of a C extension module to define new types that +can be manipulated from Python code, much like the built-in :class:`str` +and :class:`list` types. The code for all extension types follows a +pattern, but there are some details that you need to understand before you +can get started. This document is a gentle introduction to the topic. + + +.. _dnt-basics: + +The Basics +========== + +The :term:`CPython` runtime sees all Python objects as variables of type +:c:type:`PyObject\*`, which serves as a "base type" for all Python objects. +The :c:type:`PyObject` structure itself only contains the object's +:term:`reference count` and a pointer to the object's "type object". +This is where the action is; the type object determines which (C) functions +get called by the interpreter when, for instance, an attribute gets looked up +on an object, a method called, or it is multiplied by another object. These +C functions are called "type methods". + +So, if you want to define a new extension type, you need to create a new type +object. + +This sort of thing can only be explained by example, so here's a minimal, but +complete, module that defines a new type named :class:`Custom` inside a C +extension module :mod:`custom`: + +.. note:: + What we're showing here is the traditional way of defining *static* + extension types. It should be adequate for most uses. The C API also + allows defining heap-allocated extension types using the + :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. + +.. literalinclude:: ../includes/custom.c + +Now that's quite a bit to take in at once, but hopefully bits will seem familiar +from the previous chapter. This file defines three things: + +#. What a :class:`Custom` **object** contains: this is the ``CustomObject`` + struct, which is allocated once for each :class:`Custom` instance. +#. How the :class:`Custom` **type** behaves: this is the ``CustomType`` struct, + which defines a set of flags and function pointers that the interpreter + inspects when specific operations are requested. +#. How to initialize the :mod:`custom` module: this is the ``PyInit_custom`` + function and the associated ``custommodule`` struct. + +The first bit is:: + + typedef struct { + PyObject_HEAD + } CustomObject; + +This is what a Custom object will contain. ``PyObject_HEAD`` is mandatory +at the start of each object struct and defines a field called ``ob_base`` +of type :c:type:`PyObject`, containing a pointer to a type object and a +reference count (these can be accessed using the macros :c:macro:`Py_REFCNT` +and :c:macro:`Py_TYPE` respectively). The reason for the macro is to +abstract away the layout and to enable additional fields in debug builds. + +.. note:: + There is no semicolon above after the :c:macro:`PyObject_HEAD` macro. + Be wary of adding one by accident: some compilers will complain. + +Of course, objects generally store additional data besides the standard +``PyObject_HEAD`` boilerplate; for example, here is the definition for +standard Python floats:: + + typedef struct { + PyObject_HEAD + double ob_fval; + } PyFloatObject; + +The second bit is the definition of the type object. :: + + static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_new = PyType_GenericNew, + }; + +.. note:: + We recommend using C99-style designated initializers as above, to + avoid listing all the :c:type:`PyTypeObject` fields that you don't care + about and also to avoid caring about the fields' declaration order. + +The actual definition of :c:type:`PyTypeObject` in :file:`object.h` has +many more :ref:`fields ` than the definition above. The +remaining fields will be filled with zeros by the C compiler, and it's +common practice to not specify them explicitly unless you need them. + +We're going to pick it apart, one field at a time:: + + PyVarObject_HEAD_INIT(NULL, 0) + +This line is mandatory boilerplate to initialize the ``ob_base`` +field mentioned above. :: + + .tp_name = "custom.Custom", + +The name of our type. This will appear in the default textual representation of +our objects and in some error messages, for example: + +.. code-block:: pycon + + >>> "" + custom.Custom() + Traceback (most recent call last): + File "", line 1, in + TypeError: can only concatenate str (not "custom.Custom") to str + +Note that the name is a dotted name that includes both the module name and the +name of the type within the module. The module in this case is :mod:`custom` and +the type is :class:`Custom`, so we set the type name to :class:`custom.Custom`. +Using the real dotted import path is important to make your type compatible +with the :mod:`pydoc` and :mod:`pickle` modules. :: + + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + +This is so that Python knows how much memory to allocate when creating +new :class:`Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is +only used for variable-sized objects and should otherwise be zero. + +.. note:: + + If you want your type to be subclassable from Python, and your type has the same + :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple + inheritance. A Python subclass of your type will have to list your type first + in its :attr:`~class.__bases__`, or else it will not be able to call your type's + :meth:`__new__` method without getting an error. You can avoid this problem by + ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its + base type does. Most of the time, this will be true anyway, because either your + base type will be :class:`object`, or else you will be adding data members to + your base type, and therefore increasing its size. + +We set the class flags to :const:`Py_TPFLAGS_DEFAULT`. :: + + .tp_flags = Py_TPFLAGS_DEFAULT, + +All types should include this constant in their flags. It enables all of the +members defined until at least Python 3.3. If you need further members, +you will need to OR the corresponding flags. + +We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: + + .tp_doc = "Custom objects", + +To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` +handler. This is the equivalent of the Python method :meth:`__new__`, but +has to be specified explicitly. In this case, we can just use the default +implementation provided by the API function :c:func:`PyType_GenericNew`. :: + + .tp_new = PyType_GenericNew, + +Everything else in the file should be familiar, except for some code in +:c:func:`PyInit_custom`:: + + if (PyType_Ready(&CustomType) < 0) + return; + +This initializes the :class:`Custom` type, filling in a number of members +to the appropriate default values, including :attr:`ob_type` that we initially +set to *NULL*. :: + + PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + +This adds the type to the module dictionary. This allows us to create +:class:`Custom` instances by calling the :class:`Custom` class: + +.. code-block:: pycon + + >>> import custom + >>> mycustom = custom.Custom() + +That's it! All that remains is to build it; put the above code in a file called +:file:`custom.c` and: + +.. code-block:: python + + from distutils.core import setup, Extension + setup(name="custom", version="1.0", + ext_modules=[Extension("custom", ["custom.c"])]) + +in a file called :file:`setup.py`; then typing + +.. code-block:: shell-session + + $ python setup.py build + +at a shell should produce a file :file:`custom.so` in a subdirectory; move to +that directory and fire up Python --- you should be able to ``import custom`` and +play around with Custom objects. + +That wasn't so hard, was it? + +Of course, the current Custom type is pretty uninteresting. It has no data and +doesn't do anything. It can't even be subclassed. + +.. note:: + While this documentation showcases the standard :mod:`distutils` module + for building C extensions, it is recommended in real-world use cases to + use the newer and better-maintained ``setuptools`` library. Documentation + on how to do this is out of scope for this document and can be found in + the `Python Packaging User's Guide `_. + + +Adding data and methods to the Basic example +============================================ + +Let's extend the basic example to add some data and methods. Let's also make +the type usable as a base class. We'll create a new module, :mod:`custom2` that +adds these capabilities: + +.. literalinclude:: ../includes/custom2.c + + +This version of the module has a number of changes. + +We've added an extra include:: + + #include + +This include provides declarations that we use to handle attributes, as +described a bit later. + +The :class:`Custom` type now has three data attributes in its C struct, +*first*, *last*, and *number*. The *first* and *last* variables are Python +strings containing first and last names. The *number* attribute is a C integer. + +The object structure is updated accordingly:: + + typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; + } CustomObject; + +Because we now have data to manage, we have to be more careful about object +allocation and deallocation. At a minimum, we need a deallocation method:: + + static void + Custom_dealloc(CustomObject *self) + { + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); + } + +which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: + + .tp_dealloc = (destructor) Custom_dealloc, + +This method first clears the reference counts of the two Python attributes. +:c:func:`Py_XDECREF` correctly handles the case where its argument is +*NULL* (which might happen here if ``tp_new`` failed midway). It then +calls the :c:member:`~PyTypeObject.tp_free` member of the object's type +(computed by ``Py_TYPE(self)``) to free the object's memory. Note that +the object's type might not be :class:`CustomType`, because the object may +be an instance of a subclass. + +.. note:: + The explicit cast to ``destructor`` above is needed because we defined + ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc`` + function pointer expects to receive a ``PyObject *`` argument. Otherwise, + the compiler will emit a warning. This is object-oriented polymorphism, + in C! + +We want to make sure that the first and last names are initialized to empty +strings, so we provide a ``tp_new`` implementation:: + + static PyObject * + Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) + { + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } + return (PyObject *) self; + } + +and install it in the :c:member:`~PyTypeObject.tp_new` member:: + + .tp_new = Custom_new, + +The ``tp_new`` handler is responsible for creating (as opposed to initializing) +objects of the type. It is exposed in Python as the :meth:`__new__` method. +It is not required to define a ``tp_new`` member, and indeed many extension +types will simply reuse :c:func:`PyType_GenericNew` as done in the first +version of the ``Custom`` type above. In this case, we use the ``tp_new`` +handler to initialize the ``first`` and ``last`` attributes to non-*NULL* +default values. + +``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``, +if a subclass is instantiated) and any arguments passed when the type was +called, and is expected to return the instance created. ``tp_new`` handlers +always accept positional and keyword arguments, but they often ignore the +arguments, leaving the argument handling to initializer (a.k.a. ``tp_init`` +in C or ``__init__`` in Python) methods. + +.. note:: + ``tp_new`` shouldn't call ``tp_init`` explicitly, as the interpreter + will do it itself. + +The ``tp_new`` implementation calls the :c:member:`~PyTypeObject.tp_alloc` +slot to allocate memory:: + + self = (CustomObject *) type->tp_alloc(type, 0); + +Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc` +result against *NULL* before proceeding. + +.. note:: + We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather + :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, + which is :class:`object` by default. Most types use the default allocation + strategy. + +.. note:: + If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one + that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`__new__`), + you must *not* try to determine what method to call using method resolution + order at runtime. Always statically determine what type you are going to + call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via + ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your + type that also inherit from other Python-defined classes may not work correctly. + (Specifically, you may not be able to create instances of such subclasses + without getting a :exc:`TypeError`.) + +We also define an initialization function which accepts arguments to provide +initial values for our instance:: + + static int + Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + { + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_XDECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_XDECREF(tmp); + } + return 0; + } + +by filling the :c:member:`~PyTypeObject.tp_init` slot. :: + + .tp_init = (initproc) Custom_init, + +The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the +:meth:`__init__` method. It is used to initialize an object after it's +created. Initializers always accept positional and keyword arguments, +and they should return either ``0`` on success or ``-1`` on error. + +Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init`` +is called at all (for example, the :mod:`pickle` module by default +doesn't call :meth:`__init__` on unpickled instances). It can also be +called multiple times. Anyone can call the :meth:`__init__` method on +our objects. For this reason, we have to be extra careful when assigning +the new attribute values. We might be tempted, for example to assign the +``first`` member like this:: + + if (first) { + Py_XDECREF(self->first); + Py_INCREF(first); + self->first = first; + } + +But this would be risky. Our type doesn't restrict the type of the +``first`` member, so it could be any kind of object. It could have a +destructor that causes code to be executed that tries to access the +``first`` member; or that destructor could release the +:term:`Global interpreter Lock` and let arbitrary code run in other +threads that accesses and modifies our object. + +To be paranoid and protect ourselves against this possibility, we almost +always reassign members before decrementing their reference counts. When +don't we have to do this? + +* when we absolutely know that the reference count is greater than 1; + +* when we know that deallocation of the object [#]_ will neither release + the :term:`GIL` nor cause any calls back into our type's code; + +* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` + handler on a type which doesn't support cyclic garbage collection [#]_. + +We want to expose our instance variables as attributes. There are a +number of ways to do that. The simplest way is to define member definitions:: + + static PyMemberDef Custom_members[] = { + {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, + "first name"}, + {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, + "last name"}, + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL} /* Sentinel */ + }; + +and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: + + .tp_members = Custom_members, + +Each member definition has a member name, type, offset, access flags and +documentation string. See the :ref:`Generic-Attribute-Management` section +below for details. + +A disadvantage of this approach is that it doesn't provide a way to restrict the +types of objects that can be assigned to the Python attributes. We expect the +first and last names to be strings, but any Python objects can be assigned. +Further, the attributes can be deleted, setting the C pointers to *NULL*. Even +though we can make sure the members are initialized to non-*NULL* values, the +members can be set to *NULL* if the attributes are deleted. + +We define a single method, :meth:`Custom.name()`, that outputs the objects name as the +concatenation of the first and last names. :: + + static PyObject * + Custom_name(CustomObject *self) + { + if (self->first == NULL) { + PyErr_SetString(PyExc_AttributeError, "first"); + return NULL; + } + if (self->last == NULL) { + PyErr_SetString(PyExc_AttributeError, "last"); + return NULL; + } + return PyUnicode_FromFormat("%S %S", self->first, self->last); + } + +The method is implemented as a C function that takes a :class:`Custom` (or +:class:`Custom` subclass) instance as the first argument. Methods always take an +instance as the first argument. Methods often take positional and keyword +arguments as well, but in this case we don't take any and don't need to accept +a positional argument tuple or keyword argument dictionary. This method is +equivalent to the Python method: + +.. code-block:: python + + def name(self): + return "%s %s" % (self.first, self.last) + +Note that we have to check for the possibility that our :attr:`first` and +:attr:`last` members are *NULL*. This is because they can be deleted, in which +case they are set to *NULL*. It would be better to prevent deletion of these +attributes and to restrict the attribute values to be strings. We'll see how to +do that in the next section. + +Now that we've defined the method, we need to create an array of method +definitions:: + + static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {NULL} /* Sentinel */ + }; + +(note that we used the :const:`METH_NOARGS` flag to indicate that the method +is expecting no arguments other than *self*) + +and assign it to the :c:member:`~PyTypeObject.tp_methods` slot:: + + .tp_methods = Custom_methods, + +Finally, we'll make our type usable as a base class for subclassing. We've +written our methods carefully so far so that they don't make any assumptions +about the type of the object being created or used, so all we need to do is +to add the :const:`Py_TPFLAGS_BASETYPE` to our class flag definition:: + + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + +We rename :c:func:`PyInit_custom` to :c:func:`PyInit_custom2`, update the +module name in the :c:type:`PyModuleDef` struct, and update the full class +name in the :c:type:`PyTypeObject` struct. + +Finally, we update our :file:`setup.py` file to build the new module: + +.. code-block:: python + + from distutils.core import setup, Extension + setup(name="custom", version="1.0", + ext_modules=[ + Extension("custom", ["custom.c"]), + Extension("custom2", ["custom2.c"]), + ]) + + +Providing finer control over data attributes +============================================ + +In this section, we'll provide finer control over how the :attr:`first` and +:attr:`last` attributes are set in the :class:`Custom` example. In the previous +version of our module, the instance variables :attr:`first` and :attr:`last` +could be set to non-string values or even deleted. We want to make sure that +these attributes always contain strings. + +.. literalinclude:: ../includes/custom3.c + + +To provide greater control, over the :attr:`first` and :attr:`last` attributes, +we'll use custom getter and setter functions. Here are the functions for +getting and setting the :attr:`first` attribute:: + + static PyObject * + Custom_getfirst(CustomObject *self, void *closure) + { + Py_INCREF(self->first); + return self->first; + } + + static int + Custom_setfirst(CustomObject *self, PyObject *value, void *closure) + { + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The first attribute value must be a string"); + return -1; + } + tmp = self->first; + Py_INCREF(value); + self->first = value; + Py_DECREF(tmp); + return 0; + } + +The getter function is passed a :class:`Custom` object and a "closure", which is +a void pointer. In this case, the closure is ignored. (The closure supports an +advanced usage in which definition data is passed to the getter and setter. This +could, for example, be used to allow a single set of getter and setter functions +that decide the attribute to get or set based on data in the closure.) + +The setter function is passed the :class:`Custom` object, the new value, and the +closure. The new value may be *NULL*, in which case the attribute is being +deleted. In our setter, we raise an error if the attribute is deleted or if its +new value is not a string. + +We create an array of :c:type:`PyGetSetDef` structures:: + + static PyGetSetDef Custom_getsetters[] = { + {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + "first name", NULL}, + {"last", (getter) Custom_getlast, (setter) Custom_setlast, + "last name", NULL}, + {NULL} /* Sentinel */ + }; + +and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: + + .tp_getset = Custom_getsetters, + +The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned +above. In this case, we aren't using a closure, so we just pass *NULL*. + +We also remove the member definitions for these attributes:: + + static PyMemberDef Custom_members[] = { + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL} /* Sentinel */ + }; + +We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only +allow strings [#]_ to be passed:: + + static int + Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) + { + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_DECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_DECREF(tmp); + } + return 0; + } + +With these changes, we can assure that the ``first`` and ``last`` members are +never *NULL* so we can remove checks for *NULL* values in almost all cases. +This means that most of the :c:func:`Py_XDECREF` calls can be converted to +:c:func:`Py_DECREF` calls. The only place we can't change these calls is in +the ``tp_dealloc`` implementation, where there is the possibility that the +initialization of these members failed in ``tp_new``. + +We also rename the module initialization function and module name in the +initialization function, as we did before, and we add an extra definition to the +:file:`setup.py` file. + + +Supporting cyclic garbage collection +==================================== + +Python has a :term:`cyclic garbage collector (GC) ` that +can identify unneeded objects even when their reference counts are not zero. +This can happen when objects are involved in cycles. For example, consider: + +.. code-block:: pycon + + >>> l = [] + >>> l.append(l) + >>> del l + +In this example, we create a list that contains itself. When we delete it, it +still has a reference from itself. Its reference count doesn't drop to zero. +Fortunately, Python's cyclic garbage collector will eventually figure out that +the list is garbage and free it. + +In the second version of the :class:`Custom` example, we allowed any kind of +object to be stored in the :attr:`first` or :attr:`last` attributes [#]_. +Besides, in the second and third versions, we allowed subclassing +:class:`Custom`, and subclasses may add arbitrary attributes. For any of +those two reasons, :class:`Custom` objects can participate in cycles: + +.. code-block:: pycon + + >>> import custom3 + >>> class Derived(custom3.Custom): pass + ... + >>> n = Derived() + >>> n.some_attribute = n + +To allow a :class:`Custom` instance participating in a reference cycle to +be properly detected and collected by the cyclic GC, our :class:`Custom` type +needs to fill two additional slots and to enable a flag that enables these slots: + +.. literalinclude:: ../includes/custom4.c + + +First, the traversal method lets the cyclic GC know about subobjects that could +participate in cycles:: + + static int + Custom_traverse(CustomObject *self, visitproc visit, void *arg) + { + int vret; + if (self->first) { + vret = visit(self->first, arg); + if (vret != 0) + return vret; + } + if (self->last) { + vret = visit(self->last, arg); + if (vret != 0) + return vret; + } + return 0; + } + +For each subobject that can participate in cycles, we need to call the +:c:func:`visit` function, which is passed to the traversal method. The +:c:func:`visit` function takes as arguments the subobject and the extra argument +*arg* passed to the traversal method. It returns an integer value that must be +returned if it is non-zero. + +Python provides a :c:func:`Py_VISIT` macro that automates calling visit +functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate +in ``Custom_traverse``:: + + static int + Custom_traverse(CustomObject *self, visitproc visit, void *arg) + { + Py_VISIT(self->first); + Py_VISIT(self->last); + return 0; + } + +.. note:: + The :c:member:`~PyTypeObject.tp_traverse` implementation must name its + arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. + +Second, we need to provide a method for clearing any subobjects that can +participate in cycles:: + + static int + Custom_clear(CustomObject *self) + { + Py_CLEAR(self->first); + Py_CLEAR(self->last); + return 0; + } + +Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe +way to clear data attributes of arbitrary types while decrementing +their reference counts. If you were to call :c:func:`Py_XDECREF` instead +on the attribute before setting it to *NULL*, there is a possibility +that the attribute's destructor would call back into code that reads the +attribute again (*especially* if there is a reference cycle). + +.. note:: + You could emulate :c:func:`Py_CLEAR` by writing:: + + PyObject *tmp; + tmp = self->first; + self->first = NULL; + Py_XDECREF(tmp); + + Nevertheless, it is much easier and less error-prone to always + use :c:func:`Py_CLEAR` when deleting an attribute. Don't + try to micro-optimize at the expense of robustness! + +The deallocator ``Custom_dealloc`` may call arbitrary code when clearing +attributes. It means the circular GC can be triggered inside the function. +Since the GC assumes reference count is not zero, we need to untrack the object +from the GC by calling :c:func:`PyObject_GC_UnTrack` before clearing members. +Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack` +and ``Custom_clear``:: + + static void + Custom_dealloc(CustomObject *self) + { + PyObject_GC_UnTrack(self); + Custom_clear(self); + Py_TYPE(self)->tp_free((PyObject *) self); + } + +Finally, we add the :const:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: + + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + +That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or +:c:member:`~PyTypeObject.tp_free` handlers, we'd need to modify them for cyclic +garbage collection. Most extensions will use the versions automatically provided. + + +Subclassing other types +======================= + +It is possible to create new extension types that are derived from existing +types. It is easiest to inherit from the built in types, since an extension can +easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share +these :c:type:`PyTypeObject` structures between extension modules. + +In this example we will create a :class:`SubList` type that inherits from the +built-in :class:`list` type. The new type will be completely compatible with +regular lists, but will have an additional :meth:`increment` method that +increases an internal counter: + +.. code-block:: pycon + + >>> import sublist + >>> s = sublist.SubList(range(3)) + >>> s.extend(s) + >>> print(len(s)) + 6 + >>> print(s.increment()) + 1 + >>> print(s.increment()) + 2 + +.. literalinclude:: ../includes/sublist.c + + +As you can see, the source code closely resembles the :class:`Custom` examples in +previous sections. We will break down the main differences between them. :: + + typedef struct { + PyListObject list; + int state; + } SubListObject; + +The primary difference for derived type objects is that the base type's +object structure must be the first value. The base type will already include +the :c:func:`PyObject_HEAD` at the beginning of its structure. + +When a Python object is a :class:`SubList` instance, its ``PyObject *`` pointer +can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: + + static int + SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) + { + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) + return -1; + self->state = 0; + return 0; + } + +We see above how to call through to the :attr:`__init__` method of the base +type. + +This pattern is important when writing a type with custom +:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_dealloc` +members. The :c:member:`~PyTypeObject.tp_new` handler should not actually +create the memory for the object with its :c:member:`~PyTypeObject.tp_alloc`, +but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_new`. + +The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` +specifying the type's concrete base class. Due to cross-platform compiler +issues, you can't fill that field directly with a reference to +:c:type:`PyList_Type`; it should be done later in the module initialization +function:: + + PyMODINIT_FUNC + PyInit_sublist(void) + { + PyObject* m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) + return NULL; + + m = PyModule_Create(&sublistmodule); + if (m == NULL) + return NULL; + + Py_INCREF(&SubListType); + PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + return m; + } + +Before calling :c:func:`PyType_Ready`, the type structure must have the +:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving an +existing type, it is not necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` +slot with :c:func:`PyType_GenericNew` -- the allocation function from the base +type will be inherited. + +After that, calling :c:func:`PyType_Ready` and adding the type object to the +module is the same as with the basic :class:`Custom` examples. + + +.. rubric:: Footnotes + +.. [#] This is true when we know that the object is a basic type, like a string or a + float. + +.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler + in this example, because our type doesn't support garbage collection. + +.. [#] We now know that the first and last members are strings, so perhaps we + could be less careful about decrementing their reference counts, however, + we accept instances of string subclasses. Even though deallocating normal + strings won't call back into our objects, we can't guarantee that deallocating + an instance of a string subclass won't call back into our objects. + +.. [#] Also, even with our attributes restricted to strings instances, the user + could pass arbitrary :class:`str` subclasses and therefore still create + reference cycles. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst index 88996e48035b245..fd04a83df33c3df 100644 --- a/Doc/faq/extending.rst +++ b/Doc/faq/extending.rst @@ -62,7 +62,7 @@ How can I execute arbitrary Python statements from C? The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes a single string argument to be executed in the context of the module -``__main__`` and returns 0 for success and -1 when an exception occurred +``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred (including ``SyntaxError``). If you want more control, use :c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in ``Python/pythonrun.c``. diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst index f84feadd7802345..aec4bf9b85d56e6 100644 --- a/Doc/faq/library.rst +++ b/Doc/faq/library.rst @@ -74,7 +74,9 @@ interpreter. Occasionally, a user's environment is so full that the :program:`/usr/bin/env` program fails; or there's no env program at all. In that case, you can try the -following hack (due to Alex Rezinsky):: +following hack (due to Alex Rezinsky): + +.. code-block:: sh #! /bin/sh """:" diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst index d703f2862221a66..7792cfaa16a19dd 100644 --- a/Doc/faq/windows.rst +++ b/Doc/faq/windows.rst @@ -1,5 +1,7 @@ :tocdepth: 2 +.. highlightlang:: none + .. _windows-faq: ===================== @@ -39,12 +41,16 @@ or "Command prompt window". Usually you can create such a window from your Start menu; under Windows 7 the menu selection is :menuselection:`Start --> Programs --> Accessories --> Command Prompt`. You should be able to recognize when you have started such a window because you will see a Windows "command -prompt", which usually looks like this:: +prompt", which usually looks like this: + +.. code-block:: doscon C:\> The letter may be different, and there might be other things after it, so you -might just as easily see something like:: +might just as easily see something like: + +.. code-block:: doscon D:\YourName\Projects\Python> @@ -60,11 +66,15 @@ program. So, how do you arrange for the interpreter to handle your Python? First, you need to make sure that your command window recognises the word "python" as an instruction to start the interpreter. If you have opened a command window, you should try entering the command ``python`` and hitting -return.:: +return: + +.. code-block:: doscon C:\Users\YourName> python -You should then see something like:: +You should then see something like: + +.. code-block:: pycon Python 3.3.0 (v3.3.0:bd8afb90ebf2, Sep 29 2012, 10:55:48) [MSC v.1600 32 bit (Intel)] on win32 Type "help", "copyright", "credits" or "license" for more information. @@ -73,7 +83,9 @@ You should then see something like:: You have started the interpreter in "interactive mode". That means you can enter Python statements or expressions interactively and have them executed or evaluated while you wait. This is one of Python's strongest features. Check it -by entering a few expressions of your choice and seeing the results:: +by entering a few expressions of your choice and seeing the results: + +.. code-block:: pycon >>> print("Hello") Hello @@ -317,7 +329,9 @@ present, and ``getch()`` which gets one character without echoing it. How do I emulate os.kill() in Windows? -------------------------------------- -Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`:: +Prior to Python 2.7 and 3.2, to terminate a process, you can use :mod:`ctypes`: + +.. code-block:: python import ctypes diff --git a/Doc/glossary.rst b/Doc/glossary.rst index dcfe086b38b12ff..2eab003146829b5 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -463,7 +463,7 @@ Glossary hash-based pyc - A bytecode cache file that uses the the hash rather than the last-modified + A bytecode cache file that uses the hash rather than the last-modified time of the corresponding source file to determine its validity. See :ref:`pyc-invalidation`. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst index 9d770f5232b4402..e78a022b372faad 100644 --- a/Doc/howto/argparse.rst +++ b/Doc/howto/argparse.rst @@ -24,7 +24,7 @@ Concepts Let's show the sort of functionality that we are going to explore in this introductory tutorial by making use of the :command:`ls` command: -.. code-block:: sh +.. code-block:: shell-session $ ls cpython devguide prog.py pypy rm-unused-function.patch @@ -77,7 +77,7 @@ Let us start with a very simple example which does (almost) nothing:: Following is a result of running the code: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py $ python3 prog.py --help @@ -119,7 +119,7 @@ An example:: And running the code: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py usage: prog.py [-h] echo @@ -164,7 +164,7 @@ by reading the source code. So, let's make it a bit more useful:: And we get: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py -h usage: prog.py [-h] echo @@ -185,7 +185,7 @@ Now, how about doing something even more useful:: Following is a result of running the code: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 Traceback (most recent call last): @@ -206,7 +206,7 @@ give it as strings, unless we tell it otherwise. So, let's tell Following is a result of running the code: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 16 @@ -233,7 +233,7 @@ have a look on how to add optional ones:: And the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py --verbosity 1 verbosity turned on @@ -279,7 +279,7 @@ Let's modify the code accordingly:: And the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py --verbose verbosity turned on @@ -325,7 +325,7 @@ versions of the options. It's quite simple:: And here goes: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py -v verbosity turned on @@ -359,7 +359,7 @@ Our program keeps growing in complexity:: And now the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py usage: prog.py [-h] [-v] square @@ -395,7 +395,7 @@ multiple verbosity values, and actually get to use them:: And the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 16 @@ -429,7 +429,7 @@ Let's fix it by restricting the values the ``--verbosity`` option can accept:: And the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 -v 3 usage: prog.py [-h] [-v {0,1,2}] square @@ -470,7 +470,7 @@ verbosity argument (check the output of ``python --help``):: We have introduced another action, "count", to count the number of occurrences of a specific optional arguments: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 16 @@ -537,7 +537,7 @@ Let's fix:: And this is what it gives: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 -vvv the square of 4 equals 16 @@ -581,7 +581,7 @@ it gets the ``None`` value, and that cannot be compared to an int value And: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 16 @@ -614,7 +614,7 @@ not just squares:: Output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py usage: prog.py [-h] [-v] x y @@ -652,7 +652,7 @@ to display *more* text instead:: Output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 2 16 @@ -695,7 +695,7 @@ which will be the opposite of the ``--verbose`` one:: Our program is now simpler, and we've lost some functionality for the sake of demonstration. Anyways, here's the output: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py 4 2 4^2 == 16 @@ -739,7 +739,7 @@ Note that slight difference in the usage text. Note the ``[-v | -q]``, which tells us that we can either use ``-v`` or ``-q``, but not both at the same time: -.. code-block:: sh +.. code-block:: shell-session $ python3 prog.py --help usage: prog.py [-h] [-v | -q] x y diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst index d3c7d668959e56e..788a0eee2ba1198 100644 --- a/Doc/howto/clinic.rst +++ b/Doc/howto/clinic.rst @@ -267,12 +267,16 @@ Let's dive in! should get its own line. All the parameter lines should be indented from the function name and the docstring. - The general form of these parameter lines is as follows:: + The general form of these parameter lines is as follows: + + .. code-block:: none name_of_parameter: converter If the parameter has a default value, add that after the - converter:: + converter: + + .. code-block:: none name_of_parameter: converter = default_value @@ -925,13 +929,17 @@ Parameter default values ------------------------ Default values for parameters can be any of a number of values. -At their simplest, they can be string, int, or float literals:: +At their simplest, they can be string, int, or float literals: + +.. code-block:: none foo: str = "abc" bar: int = 123 bat: float = 45.6 -They can also use any of Python's built-in constants:: +They can also use any of Python's built-in constants: + +.. code-block:: none yep: bool = True nope: bool = False @@ -959,7 +967,9 @@ It can be an entire expression, using math operators and looking up attributes on objects. However, this support isn't exactly simple, because of some non-obvious semantics. -Consider the following example:: +Consider the following example: + +.. code-block:: none foo: Py_ssize_t = sys.maxsize - 1 @@ -970,7 +980,9 @@ runtime, when the user asks for the function's signature. What namespace is available when the expression is evaluated? It's evaluated in the context of the module the builtin came from. So, if your module has an -attribute called "``max_widgets``", you may simply use it:: +attribute called "``max_widgets``", you may simply use it: + +.. code-block:: none foo: Py_ssize_t = max_widgets @@ -982,7 +994,9 @@ it's best to restrict yourself to modules that are preloaded by Python itself.) Evaluating default values only at runtime means Argument Clinic can't compute the correct equivalent C default value. So you need to tell it explicitly. When you use an expression, you must also specify the equivalent expression -in C, using the ``c_default`` parameter to the converter:: +in C, using the ``c_default`` parameter to the converter: + +.. code-block:: none foo: Py_ssize_t(c_default="PY_SSIZE_T_MAX - 1") = sys.maxsize - 1 @@ -1359,7 +1373,9 @@ Let's start with defining some terminology: A field, in this context, is a subsection of Clinic's output. For example, the ``#define`` for the ``PyMethodDef`` structure is a field, called ``methoddef_define``. Clinic has seven - different fields it can output per function definition:: + different fields it can output per function definition: + + .. code-block:: none docstring_prototype docstring_definition @@ -1416,7 +1432,9 @@ Let's start with defining some terminology: Clinic defines five new directives that let you reconfigure its output. -The first new directive is ``dump``:: +The first new directive is ``dump``: + +.. code-block:: none dump @@ -1425,7 +1443,9 @@ the current block, and empties it. This only works with ``buffer`` and ``two-pass`` destinations. The second new directive is ``output``. The most basic form of ``output`` -is like this:: +is like this: + +.. code-block:: none output @@ -1433,7 +1453,9 @@ This tells Clinic to output *field* to *destination*. ``output`` also supports a special meta-destination, called ``everything``, which tells Clinic to output *all* fields to that *destination*. -``output`` has a number of other functions:: +``output`` has a number of other functions: + +.. code-block:: none output push output pop @@ -1508,7 +1530,9 @@ preset configurations, as follows: Suppresses the ``impl_prototype``, write the ``docstring_definition`` and ``parser_definition`` to ``buffer``, write everything else to ``block``. -The third new directive is ``destination``:: +The third new directive is ``destination``: + +.. code-block:: none destination [...] @@ -1516,7 +1540,9 @@ This performs an operation on the destination named ``name``. There are two defined subcommands: ``new`` and ``clear``. -The ``new`` subcommand works like this:: +The ``new`` subcommand works like this: + +.. code-block:: none destination new @@ -1564,7 +1590,9 @@ There are five destination types: A two-pass buffer, like the "two-pass" builtin destination above. -The ``clear`` subcommand works like this:: +The ``clear`` subcommand works like this: + +.. code-block:: none destination clear @@ -1572,7 +1600,9 @@ It removes all the accumulated text up to this point in the destination. (I don't know what you'd need this for, but I thought maybe it'd be useful while someone's experimenting.) -The fourth new directive is ``set``:: +The fourth new directive is ``set``: + +.. code-block:: none set line_prefix "string" set line_suffix "string" @@ -1590,7 +1620,9 @@ Both of these support two format strings: Turns into the string ``*/``, the end-comment text sequence for C files. The final new directive is one you shouldn't need to use directly, -called ``preserve``:: +called ``preserve``: + +.. code-block:: none preserve @@ -1638,7 +1670,9 @@ like so:: #endif /* HAVE_FUNCTIONNAME */ Then, remove those three lines from the ``PyMethodDef`` structure, -replacing them with the macro Argument Clinic generated:: +replacing them with the macro Argument Clinic generated: + +.. code-block:: none MODULE_FUNCTIONNAME_METHODDEF diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst index b9c51a4a4538aba..b63c43c81f71d20 100644 --- a/Doc/howto/instrumentation.rst +++ b/Doc/howto/instrumentation.rst @@ -254,11 +254,15 @@ and the remainder indicates the call/return hierarchy as the script executes. For a `--enable-shared` build of CPython, the markers are contained within the libpython shared library, and the probe's dotted path needs to reflect this. For -example, this line from the above example:: +example, this line from the above example: + +.. code-block:: none probe process("python").mark("function__entry") { -should instead read:: +should instead read: + +.. code-block:: none probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") { diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 4d2d052d2917880..fdf7874883eef8e 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -72,7 +72,9 @@ Here is the auxiliary module:: def some_function(): module_logger.info('received a call to "some_function"') -The output looks like this:: +The output looks like this: + +.. code-block:: none 2005-03-23 23:47:11,663 - spam_application - INFO - creating an instance of auxiliary_module.Auxiliary @@ -127,7 +129,9 @@ shows logging from the main (initial) thread and another thread:: if __name__ == '__main__': main() -When run, the script should print something like the following:: +When run, the script should print something like the following: + +.. code-block:: none 0 Thread-1 Hi from myfunc 3 MainThread Hello from main @@ -240,14 +244,18 @@ messages should not. Here's how you can achieve this:: logger2.warning('Jail zesty vixen who grabbed pay from quack.') logger2.error('The five boxing wizards jump quickly.') -When you run this, on the console you will see :: +When you run this, on the console you will see + +.. code-block:: none root : INFO Jackdaws love my big sphinx of quartz. myapp.area1 : INFO How quickly daft jumping zebras vex. myapp.area2 : WARNING Jail zesty vixen who grabbed pay from quack. myapp.area2 : ERROR The five boxing wizards jump quickly. -and in the file you will see something like :: +and in the file you will see something like + +.. code-block:: none 10-22 22:19 root INFO Jackdaws love my big sphinx of quartz. 10-22 22:19 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim. @@ -515,7 +523,9 @@ module. Here is a basic working example:: main() First run the server, and then the client. On the client side, nothing is -printed on the console; on the server side, you should see something like:: +printed on the console; on the server side, you should see something like: + +.. code-block:: none About to start TCP server... 59 root INFO Jackdaws love my big sphinx of quartz. @@ -675,7 +685,9 @@ script:: lvlname = logging.getLevelName(lvl) a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') -which, when run, produces something like:: +which, when run, produces something like: + +.. code-block:: none 2010-09-06 22:38:15,292 a.b.c DEBUG IP: 123.231.231.123 User: fred A debug message 2010-09-06 22:38:15,300 a.b.c INFO IP: 192.168.0.1 User: sheila An info message with some parameters @@ -976,7 +988,9 @@ logging package provides a :class:`~handlers.RotatingFileHandler`:: print(filename) The result should be 6 separate files, each with part of the log history for the -application:: +application: + +.. code-block:: none logging_rotatingfile_example.out logging_rotatingfile_example.out.1 @@ -1706,7 +1720,9 @@ which uses JSON to serialise the event in a machine-parseable manner:: logging.basicConfig(level=logging.INFO, format='%(message)s') logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456)) -If the above script is run, it prints:: +If the above script is run, it prints: + +.. code-block:: none message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"} @@ -1753,7 +1769,9 @@ as in the following complete example:: if __name__ == '__main__': main() -When the above script is run, it prints:: +When the above script is run, it prints: + +.. code-block:: none message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]} @@ -2083,7 +2101,9 @@ most obvious, but you can provide any callable which returns a This example shows how you can pass configuration data to the callable which constructs the instance, in the form of keyword parameters. When run, the above -script will print:: +script will print: + +.. code-block:: none changed: hello @@ -2150,7 +2170,9 @@ class, as shown in the following example:: if __name__ == '__main__': main() -When run, this produces a file with exactly two lines:: +When run, this produces a file with exactly two lines: + +.. code-block:: none 28/01/2015 07:21:23|INFO|Sample message| 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| @@ -2312,7 +2334,9 @@ Here's the script:: write_line('Calling decorated foo with True') assert decorated_foo(True) -When this script is run, the following output should be observed:: +When this script is run, the following output should be observed: + +.. code-block:: none Calling undecorated foo with False about to log at DEBUG ... @@ -2408,7 +2432,9 @@ the following complete example:: logging.config.dictConfig(LOGGING) logging.warning('The local time is %s', time.asctime()) -When this script is run, it should print something like:: +When this script is run, it should print something like: + +.. code-block:: none 2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015 2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015 diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst index 4ee68b4747eb009..f8b78b618569868 100644 --- a/Doc/howto/logging.rst +++ b/Doc/howto/logging.rst @@ -134,7 +134,9 @@ interpreter, and don't just continue from the session described above:: logging.warning('And this, too') And now if we open the file and look at what we have, we should find the log -messages:: +messages: + +.. code-block:: none DEBUG:root:This message should go to the log file INFO:root:So should this @@ -144,7 +146,9 @@ This example also shows how you can set the logging level which acts as the threshold for tracking. In this case, because we set the threshold to ``DEBUG``, all of the messages were printed. -If you want to set the logging level from a command-line option such as:: +If you want to set the logging level from a command-line option such as: + +.. code-block:: none --log=INFO @@ -208,7 +212,9 @@ could organize logging in it:: def do_something(): logging.info('Doing something') -If you run *myapp.py*, you should see this in *myapp.log*:: +If you run *myapp.py*, you should see this in *myapp.log*: + +.. code-block:: none INFO:root:Started INFO:root:Doing something @@ -258,7 +264,9 @@ specify the format you want to use:: logging.info('So should this') logging.warning('And this, too') -which would print:: +which would print: + +.. code-block:: none DEBUG:This message should appear on the console INFO:So should this @@ -282,7 +290,9 @@ your format string:: logging.basicConfig(format='%(asctime)s %(message)s') logging.warning('is when this event was logged.') -which should print something like this:: +which should print something like this: + +.. code-block:: none 2010-12-12 11:41:42,612 is when this event was logged. @@ -294,7 +304,9 @@ argument to ``basicConfig``, as in this example:: logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') logging.warning('is when this event was logged.') -which would display something like this:: +which would display something like this: + +.. code-block:: none 12/12/2010 11:46:36 AM is when this event was logged. @@ -376,7 +388,9 @@ if no destination is set; and if one is not set, they will set a destination of the console (``sys.stderr``) and a default format for the displayed message before delegating to the root logger to do the actual message output. -The default format set by :func:`basicConfig` for messages is:: +The default format set by :func:`basicConfig` for messages is: + +.. code-block:: none severity:logger name:message @@ -522,7 +536,9 @@ indicator. .. method:: logging.Formatter.__init__(fmt=None, datefmt=None, style='%') If there is no message format string, the default is to use the -raw message. If there is no date format string, the default date format is:: +raw message. If there is no date format string, the default date format is: + +.. code-block:: none %Y-%m-%d %H:%M:%S @@ -628,7 +644,9 @@ the names of the objects:: logger.error('error message') logger.critical('critical message') -Here is the logging.conf file:: +Here is the logging.conf file: + +.. code-block:: ini [loggers] keys=root,simpleExample @@ -713,7 +731,9 @@ construct the dictionary in Python code, receive it in pickled form over a socket, or use whatever approach makes sense for your application. Here's an example of the same configuration as above, in YAML format for -the new dictionary-based approach:: +the new dictionary-based approach: + +.. code-block:: yaml version: 1 formatters: diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst index 87a6b1aba59f9f2..b09f748a9227330 100644 --- a/Doc/howto/regex.rst +++ b/Doc/howto/regex.rst @@ -289,6 +289,8 @@ Putting REs in strings keeps the Python language simpler, but has one disadvantage which is the topic of the next section. +.. _the-backslash-plague: + The Backslash Plague -------------------- @@ -327,6 +329,13 @@ backslashes are not handled in any special way in a string literal prefixed with while ``"\n"`` is a one-character string containing a newline. Regular expressions will often be written in Python code using this raw string notation. +In addition, special escape sequences that are valid in regular expressions, +but not valid as Python string literals, now result in a +:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, +which means the sequences will be invalid if raw string notation or escaping +the backslashes isn't used. + + +-------------------+------------------+ | Regular String | Raw string | +===================+==================+ @@ -457,10 +466,16 @@ In actual programs, the most common style is to store the Two pattern methods return all of the matches for a pattern. :meth:`~re.Pattern.findall` returns a list of matching strings:: - >>> p = re.compile('\d+') + >>> p = re.compile(r'\d+') >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') ['12', '11', '10'] +The ``r`` prefix, making the literal a raw string literal, is needed in this +example because escape sequences in a normal "cooked" string literal that are +not recognized by Python, as opposed to regular expressions, now result in a +:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See +:ref:`the-backslash-plague`. + :meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the result. The :meth:`~re.Pattern.finditer` method returns a sequence of :ref:`match object ` instances as an :term:`iterator`:: @@ -771,7 +786,9 @@ Frequently you need to obtain more information than just whether the RE matched or not. Regular expressions are often used to dissect strings by writing a RE divided into several subgroups which match different components of interest. For example, an RFC-822 header line is divided into a header name and a value, -separated by a ``':'``, like this:: +separated by a ``':'``, like this: + +.. code-block:: none From: author@example.com User-Agent: Thunderbird 1.5.0.9 (X11/20061227) @@ -1096,11 +1113,11 @@ following calls:: The module-level function :func:`re.split` adds the RE to be used as the first argument, but is otherwise the same. :: - >>> re.split('[\W]+', 'Words, words, words.') + >>> re.split(r'[\W]+', 'Words, words, words.') ['Words', 'words', 'words', ''] - >>> re.split('([\W]+)', 'Words, words, words.') + >>> re.split(r'([\W]+)', 'Words, words, words.') ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split('[\W]+', 'Words, words, words.', 1) + >>> re.split(r'[\W]+', 'Words, words, words.', 1) ['Words', 'words, words.'] diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index d4b8f8d2204ab75..be1fefb35a71f09 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -30,7 +30,9 @@ spellings such as 'coöperate'.) For a while people just wrote programs that didn't display accents. In the mid-1980s an Apple II BASIC program written by a French speaker -might have lines like these:: +might have lines like these: + +.. code-block:: basic PRINT "MISE A JOUR TERMINEE" PRINT "PARAMETRES ENREGISTRES" @@ -463,7 +465,7 @@ The string in this example has the number 57 written in both Thai and Arabic numerals:: import re - p = re.compile('\d+') + p = re.compile(r'\d+') s = "Over \u0e55\u0e57 57 flavours" m = p.search(s) diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst index c1fd5cf0680d96c..204a05a6100818f 100644 --- a/Doc/howto/urllib2.rst +++ b/Doc/howto/urllib2.rst @@ -457,7 +457,9 @@ error code) requesting authentication. This specifies the authentication scheme and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME realm="REALM"``. -e.g. :: +e.g. + +.. code-block:: none WWW-Authenticate: Basic realm="cPanel Users" diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c new file mode 100644 index 000000000000000..fb2c7b2a430e644 --- /dev/null +++ b/Doc/includes/custom.c @@ -0,0 +1,39 @@ +#include + +typedef struct { + PyObject_HEAD + /* Type-specific fields go here. */ +} CustomObject; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT, + .tp_new = PyType_GenericNew, +}; + +static PyModuleDef custommodule = { + PyModuleDef_HEAD_INIT, + .m_name = "custom", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_custom(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&custommodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + return m; +} diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c new file mode 100644 index 000000000000000..51ab4b80d680980 --- /dev/null +++ b/Doc/includes/custom2.c @@ -0,0 +1,132 @@ +#include +#include "structmember.h" + +typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; +} CustomObject; + +static void +Custom_dealloc(CustomObject *self) +{ + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } + return (PyObject *) self; +} + +static int +Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_XDECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_XDECREF(tmp); + } + return 0; +} + +static PyMemberDef Custom_members[] = { + {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, + "first name"}, + {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, + "last name"}, + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL} /* Sentinel */ +}; + +static PyObject * +Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +{ + if (self->first == NULL) { + PyErr_SetString(PyExc_AttributeError, "first"); + return NULL; + } + if (self->last == NULL) { + PyErr_SetString(PyExc_AttributeError, "last"); + return NULL; + } + return PyUnicode_FromFormat("%S %S", self->first, self->last); +} + +static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom2.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = Custom_new, + .tp_init = (initproc) Custom_init, + .tp_dealloc = (destructor) Custom_dealloc, + .tp_members = Custom_members, + .tp_methods = Custom_methods, +}; + +static PyModuleDef custommodule = { + PyModuleDef_HEAD_INIT, + .m_name = "custom2", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_custom2(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&custommodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + return m; +} diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c new file mode 100644 index 000000000000000..09e87355b91afa2 --- /dev/null +++ b/Doc/includes/custom3.c @@ -0,0 +1,183 @@ +#include +#include "structmember.h" + +typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; +} CustomObject; + +static void +Custom_dealloc(CustomObject *self) +{ + Py_XDECREF(self->first); + Py_XDECREF(self->last); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } + return (PyObject *) self; +} + +static int +Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_DECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_DECREF(tmp); + } + return 0; +} + +static PyMemberDef Custom_members[] = { + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL} /* Sentinel */ +}; + +static PyObject * +Custom_getfirst(CustomObject *self, void *closure) +{ + Py_INCREF(self->first); + return self->first; +} + +static int +Custom_setfirst(CustomObject *self, PyObject *value, void *closure) +{ + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The first attribute value must be a string"); + return -1; + } + tmp = self->first; + Py_INCREF(value); + self->first = value; + Py_DECREF(tmp); + return 0; +} + +static PyObject * +Custom_getlast(CustomObject *self, void *closure) +{ + Py_INCREF(self->last); + return self->last; +} + +static int +Custom_setlast(CustomObject *self, PyObject *value, void *closure) +{ + PyObject *tmp; + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The last attribute value must be a string"); + return -1; + } + tmp = self->last; + Py_INCREF(value); + self->last = value; + Py_DECREF(tmp); + return 0; +} + +static PyGetSetDef Custom_getsetters[] = { + {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + "first name", NULL}, + {"last", (getter) Custom_getlast, (setter) Custom_setlast, + "last name", NULL}, + {NULL} /* Sentinel */ +}; + +static PyObject * +Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromFormat("%S %S", self->first, self->last); +} + +static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom3.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_new = Custom_new, + .tp_init = (initproc) Custom_init, + .tp_dealloc = (destructor) Custom_dealloc, + .tp_members = Custom_members, + .tp_methods = Custom_methods, + .tp_getset = Custom_getsetters, +}; + +static PyModuleDef custommodule = { + PyModuleDef_HEAD_INIT, + .m_name = "custom3", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_custom3(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&custommodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + return m; +} diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c new file mode 100644 index 000000000000000..0994d8fda0e51fa --- /dev/null +++ b/Doc/includes/custom4.c @@ -0,0 +1,197 @@ +#include +#include "structmember.h" + +typedef struct { + PyObject_HEAD + PyObject *first; /* first name */ + PyObject *last; /* last name */ + int number; +} CustomObject; + +static int +Custom_traverse(CustomObject *self, visitproc visit, void *arg) +{ + Py_VISIT(self->first); + Py_VISIT(self->last); + return 0; +} + +static int +Custom_clear(CustomObject *self) +{ + Py_CLEAR(self->first); + Py_CLEAR(self->last); + return 0; +} + +static void +Custom_dealloc(CustomObject *self) +{ + PyObject_GC_UnTrack(self); + Custom_clear(self); + Py_TYPE(self)->tp_free((PyObject *) self); +} + +static PyObject * +Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + CustomObject *self; + self = (CustomObject *) type->tp_alloc(type, 0); + if (self != NULL) { + self->first = PyUnicode_FromString(""); + if (self->first == NULL) { + Py_DECREF(self); + return NULL; + } + self->last = PyUnicode_FromString(""); + if (self->last == NULL) { + Py_DECREF(self); + return NULL; + } + self->number = 0; + } + return (PyObject *) self; +} + +static int +Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"first", "last", "number", NULL}; + PyObject *first = NULL, *last = NULL, *tmp; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, + &first, &last, + &self->number)) + return -1; + + if (first) { + tmp = self->first; + Py_INCREF(first); + self->first = first; + Py_DECREF(tmp); + } + if (last) { + tmp = self->last; + Py_INCREF(last); + self->last = last; + Py_DECREF(tmp); + } + return 0; +} + +static PyMemberDef Custom_members[] = { + {"number", T_INT, offsetof(CustomObject, number), 0, + "custom number"}, + {NULL} /* Sentinel */ +}; + +static PyObject * +Custom_getfirst(CustomObject *self, void *closure) +{ + Py_INCREF(self->first); + return self->first; +} + +static int +Custom_setfirst(CustomObject *self, PyObject *value, void *closure) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The first attribute value must be a string"); + return -1; + } + Py_INCREF(value); + Py_CLEAR(self->first); + self->first = value; + return 0; +} + +static PyObject * +Custom_getlast(CustomObject *self, void *closure) +{ + Py_INCREF(self->last); + return self->last; +} + +static int +Custom_setlast(CustomObject *self, PyObject *value, void *closure) +{ + if (value == NULL) { + PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); + return -1; + } + if (!PyUnicode_Check(value)) { + PyErr_SetString(PyExc_TypeError, + "The last attribute value must be a string"); + return -1; + } + Py_INCREF(value); + Py_CLEAR(self->last); + self->last = value; + return 0; +} + +static PyGetSetDef Custom_getsetters[] = { + {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, + "first name", NULL}, + {"last", (getter) Custom_getlast, (setter) Custom_setlast, + "last name", NULL}, + {NULL} /* Sentinel */ +}; + +static PyObject * +Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) +{ + return PyUnicode_FromFormat("%S %S", self->first, self->last); +} + +static PyMethodDef Custom_methods[] = { + {"name", (PyCFunction) Custom_name, METH_NOARGS, + "Return the name, combining the first and last name" + }, + {NULL} /* Sentinel */ +}; + +static PyTypeObject CustomType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "custom4.Custom", + .tp_doc = "Custom objects", + .tp_basicsize = sizeof(CustomObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .tp_new = Custom_new, + .tp_init = (initproc) Custom_init, + .tp_dealloc = (destructor) Custom_dealloc, + .tp_traverse = (traverseproc) Custom_traverse, + .tp_clear = (inquiry) Custom_clear, + .tp_members = Custom_members, + .tp_methods = Custom_methods, + .tp_getset = Custom_getsetters, +}; + +static PyModuleDef custommodule = { + PyModuleDef_HEAD_INIT, + .m_name = "custom4", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_custom4(void) +{ + PyObject *m; + if (PyType_Ready(&CustomType) < 0) + return NULL; + + m = PyModule_Create(&custommodule); + if (m == NULL) + return NULL; + + Py_INCREF(&CustomType); + PyModule_AddObject(m, "Custom", (PyObject *) &CustomType); + return m; +} diff --git a/Doc/includes/noddy.c b/Doc/includes/noddy.c deleted file mode 100644 index 07b5d5a9b83ce08..000000000000000 --- a/Doc/includes/noddy.c +++ /dev/null @@ -1,72 +0,0 @@ -#include - -typedef struct { - PyObject_HEAD - /* Type-specific fields go here. */ -} noddy_NoddyObject; - -static PyTypeObject noddy_NoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "noddy.Noddy", /* tp_name */ - sizeof(noddy_NoddyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew, /* tp_new */ -}; - -static PyModuleDef noddymodule = { - PyModuleDef_HEAD_INIT, - "noddy", - "Example module that creates an extension type.", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC -PyInit_noddy(void) -{ - PyObject* m; - - if (PyType_Ready(&noddy_NoddyType) < 0) - return NULL; - - m = PyModule_Create(&noddymodule); - if (m == NULL) - return NULL; - - Py_INCREF(&noddy_NoddyType); - PyModule_AddObject(m, "Noddy", (PyObject *)&noddy_NoddyType); - return m; -} diff --git a/Doc/includes/noddy2.c b/Doc/includes/noddy2.c deleted file mode 100644 index 964155845fee83e..000000000000000 --- a/Doc/includes/noddy2.c +++ /dev/null @@ -1,172 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} Noddy; - -static void -Noddy_dealloc(Noddy* self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static PyObject * -Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - Noddy *self; - - self = (Noddy *)type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - - self->number = 0; - } - - return (PyObject *)self; -} - -static int -Noddy_init(Noddy *self, PyObject *args, PyObject *kwds) -{ - PyObject *first=NULL, *last=NULL, *tmp; - - static char *kwlist[] = {"first", "last", "number", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - - return 0; -} - - -static PyMemberDef Noddy_members[] = { - {"first", T_OBJECT_EX, offsetof(Noddy, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(Noddy, last), 0, - "last name"}, - {"number", T_INT, offsetof(Noddy, number), 0, - "noddy number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Noddy_name(Noddy* self) -{ - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Noddy_methods[] = { - {"name", (PyCFunction)Noddy_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject NoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "noddy.Noddy", /* tp_name */ - sizeof(Noddy), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Noddy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Noddy_methods, /* tp_methods */ - Noddy_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Noddy_init, /* tp_init */ - 0, /* tp_alloc */ - Noddy_new, /* tp_new */ -}; - -static PyModuleDef noddy2module = { - PyModuleDef_HEAD_INIT, - "noddy2", - "Example module that creates an extension type.", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC -PyInit_noddy2(void) -{ - PyObject* m; - - if (PyType_Ready(&NoddyType) < 0) - return NULL; - - m = PyModule_Create(&noddy2module); - if (m == NULL) - return NULL; - - Py_INCREF(&NoddyType); - PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType); - return m; -} diff --git a/Doc/includes/noddy3.c b/Doc/includes/noddy3.c deleted file mode 100644 index 8a5a753ca439eb9..000000000000000 --- a/Doc/includes/noddy3.c +++ /dev/null @@ -1,225 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; - PyObject *last; - int number; -} Noddy; - -static void -Noddy_dealloc(Noddy* self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static PyObject * -Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - Noddy *self; - - self = (Noddy *)type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - - self->number = 0; - } - - return (PyObject *)self; -} - -static int -Noddy_init(Noddy *self, PyObject *args, PyObject *kwds) -{ - PyObject *first=NULL, *last=NULL, *tmp; - - static char *kwlist[] = {"first", "last", "number", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|SSi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - - return 0; -} - -static PyMemberDef Noddy_members[] = { - {"number", T_INT, offsetof(Noddy, number), 0, - "noddy number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Noddy_getfirst(Noddy *self, void *closure) -{ - Py_INCREF(self->first); - return self->first; -} - -static int -Noddy_setfirst(Noddy *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - - if (! PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - - Py_DECREF(self->first); - Py_INCREF(value); - self->first = value; - - return 0; -} - -static PyObject * -Noddy_getlast(Noddy *self, void *closure) -{ - Py_INCREF(self->last); - return self->last; -} - -static int -Noddy_setlast(Noddy *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); - return -1; - } - - if (! PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The last attribute value must be a string"); - return -1; - } - - Py_DECREF(self->last); - Py_INCREF(value); - self->last = value; - - return 0; -} - -static PyGetSetDef Noddy_getseters[] = { - {"first", - (getter)Noddy_getfirst, (setter)Noddy_setfirst, - "first name", - NULL}, - {"last", - (getter)Noddy_getlast, (setter)Noddy_setlast, - "last name", - NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Noddy_name(Noddy* self) -{ - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Noddy_methods[] = { - {"name", (PyCFunction)Noddy_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject NoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "noddy.Noddy", /* tp_name */ - sizeof(Noddy), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Noddy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - "Noddy objects", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Noddy_methods, /* tp_methods */ - Noddy_members, /* tp_members */ - Noddy_getseters, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Noddy_init, /* tp_init */ - 0, /* tp_alloc */ - Noddy_new, /* tp_new */ -}; - -static PyModuleDef noddy3module = { - PyModuleDef_HEAD_INIT, - "noddy3", - "Example module that creates an extension type.", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC -PyInit_noddy3(void) -{ - PyObject* m; - - if (PyType_Ready(&NoddyType) < 0) - return NULL; - - m = PyModule_Create(&noddy3module); - if (m == NULL) - return NULL; - - Py_INCREF(&NoddyType); - PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType); - return m; -} diff --git a/Doc/includes/noddy4.c b/Doc/includes/noddy4.c deleted file mode 100644 index 08ba4c3d91a0305..000000000000000 --- a/Doc/includes/noddy4.c +++ /dev/null @@ -1,208 +0,0 @@ -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; - PyObject *last; - int number; -} Noddy; - -static int -Noddy_traverse(Noddy *self, visitproc visit, void *arg) -{ - int vret; - - if (self->first) { - vret = visit(self->first, arg); - if (vret != 0) - return vret; - } - if (self->last) { - vret = visit(self->last, arg); - if (vret != 0) - return vret; - } - - return 0; -} - -static int -Noddy_clear(Noddy *self) -{ - PyObject *tmp; - - tmp = self->first; - self->first = NULL; - Py_XDECREF(tmp); - - tmp = self->last; - self->last = NULL; - Py_XDECREF(tmp); - - return 0; -} - -static void -Noddy_dealloc(Noddy* self) -{ - PyObject_GC_UnTrack(self); - Noddy_clear(self); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static PyObject * -Noddy_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - Noddy *self; - - self = (Noddy *)type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - - self->number = 0; - } - - return (PyObject *)self; -} - -static int -Noddy_init(Noddy *self, PyObject *args, PyObject *kwds) -{ - PyObject *first=NULL, *last=NULL, *tmp; - - static char *kwlist[] = {"first", "last", "number", NULL}; - - if (! PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - - return 0; -} - - -static PyMemberDef Noddy_members[] = { - {"first", T_OBJECT_EX, offsetof(Noddy, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(Noddy, last), 0, - "last name"}, - {"number", T_INT, offsetof(Noddy, number), 0, - "noddy number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Noddy_name(Noddy* self) -{ - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Noddy_methods[] = { - {"name", (PyCFunction)Noddy_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject NoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "noddy.Noddy", /* tp_name */ - sizeof(Noddy), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)Noddy_dealloc, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE | - Py_TPFLAGS_HAVE_GC, /* tp_flags */ - "Noddy objects", /* tp_doc */ - (traverseproc)Noddy_traverse, /* tp_traverse */ - (inquiry)Noddy_clear, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Noddy_methods, /* tp_methods */ - Noddy_members, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Noddy_init, /* tp_init */ - 0, /* tp_alloc */ - Noddy_new, /* tp_new */ -}; - -static PyModuleDef noddy4module = { - PyModuleDef_HEAD_INIT, - "noddy4", - "Example module that creates an extension type.", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC -PyInit_noddy4(void) -{ - PyObject* m; - - if (PyType_Ready(&NoddyType) < 0) - return NULL; - - m = PyModule_Create(&noddy4module); - if (m == NULL) - return NULL; - - Py_INCREF(&NoddyType); - PyModule_AddObject(m, "Noddy", (PyObject *)&NoddyType); - return m; -} diff --git a/Doc/includes/shoddy.c b/Doc/includes/shoddy.c deleted file mode 100644 index 0ef4765327776fc..000000000000000 --- a/Doc/includes/shoddy.c +++ /dev/null @@ -1,99 +0,0 @@ -#include - -typedef struct { - PyListObject list; - int state; -} Shoddy; - - -static PyObject * -Shoddy_increment(Shoddy *self, PyObject *unused) -{ - self->state++; - return PyLong_FromLong(self->state); -} - - -static PyMethodDef Shoddy_methods[] = { - {"increment", (PyCFunction)Shoddy_increment, METH_NOARGS, - PyDoc_STR("increment state counter")}, - {NULL}, -}; - -static int -Shoddy_init(Shoddy *self, PyObject *args, PyObject *kwds) -{ - if (PyList_Type.tp_init((PyObject *)self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; -} - - -static PyTypeObject ShoddyType = { - PyVarObject_HEAD_INIT(NULL, 0) - "shoddy.Shoddy", /* tp_name */ - sizeof(Shoddy), /* tp_basicsize */ - 0, /* tp_itemsize */ - 0, /* tp_dealloc */ - 0, /* tp_print */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_reserved */ - 0, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | - Py_TPFLAGS_BASETYPE, /* tp_flags */ - 0, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - Shoddy_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)Shoddy_init, /* tp_init */ - 0, /* tp_alloc */ - 0, /* tp_new */ -}; - -static PyModuleDef shoddymodule = { - PyModuleDef_HEAD_INIT, - "shoddy", - "Shoddy module", - -1, - NULL, NULL, NULL, NULL, NULL -}; - -PyMODINIT_FUNC -PyInit_shoddy(void) -{ - PyObject *m; - - ShoddyType.tp_base = &PyList_Type; - if (PyType_Ready(&ShoddyType) < 0) - return NULL; - - m = PyModule_Create(&shoddymodule); - if (m == NULL) - return NULL; - - Py_INCREF(&ShoddyType); - PyModule_AddObject(m, "Shoddy", (PyObject *) &ShoddyType); - return m; -} diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c new file mode 100644 index 000000000000000..376dddfac09c685 --- /dev/null +++ b/Doc/includes/sublist.c @@ -0,0 +1,63 @@ +#include + +typedef struct { + PyListObject list; + int state; +} SubListObject; + +static PyObject * +SubList_increment(SubListObject *self, PyObject *unused) +{ + self->state++; + return PyLong_FromLong(self->state); +} + +static PyMethodDef SubList_methods[] = { + {"increment", (PyCFunction) SubList_increment, METH_NOARGS, + PyDoc_STR("increment state counter")}, + {NULL}, +}; + +static int +SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) +{ + if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) + return -1; + self->state = 0; + return 0; +} + +static PyTypeObject SubListType = { + PyVarObject_HEAD_INIT(NULL, 0) + .tp_name = "sublist.SubList", + .tp_doc = "SubList objects", + .tp_basicsize = sizeof(SubListObject), + .tp_itemsize = 0, + .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .tp_init = (initproc) SubList_init, + .tp_methods = SubList_methods, +}; + +static PyModuleDef sublistmodule = { + PyModuleDef_HEAD_INIT, + .m_name = "sublist", + .m_doc = "Example module that creates an extension type.", + .m_size = -1, +}; + +PyMODINIT_FUNC +PyInit_sublist(void) +{ + PyObject *m; + SubListType.tp_base = &PyList_Type; + if (PyType_Ready(&SubListType) < 0) + return NULL; + + m = PyModule_Create(&sublistmodule); + if (m == NULL) + return NULL; + + Py_INCREF(&SubListType); + PyModule_AddObject(m, "SubList", (PyObject *) &SubListType); + return m; +} diff --git a/Doc/includes/test.py b/Doc/includes/test.py index 9e9d4a67121cae1..09ebe3fec0bdbe3 100644 --- a/Doc/includes/test.py +++ b/Doc/includes/test.py @@ -1,181 +1,168 @@ -"""Test module for the noddy examples +"""Test module for the custom examples -Noddy 1: +Custom 1: ->>> import noddy ->>> n1 = noddy.Noddy() ->>> n2 = noddy.Noddy() ->>> del n1 ->>> del n2 +>>> import custom +>>> c1 = custom.Custom() +>>> c2 = custom.Custom() +>>> del c1 +>>> del c2 -Noddy 2 +Custom 2 ->>> import noddy2 ->>> n1 = noddy2.Noddy('jim', 'fulton', 42) ->>> n1.first +>>> import custom2 +>>> c1 = custom2.Custom('jim', 'fulton', 42) +>>> c1.first 'jim' ->>> n1.last +>>> c1.last 'fulton' ->>> n1.number +>>> c1.number 42 ->>> n1.name() +>>> c1.name() 'jim fulton' ->>> n1.first = 'will' ->>> n1.name() +>>> c1.first = 'will' +>>> c1.name() 'will fulton' ->>> n1.last = 'tell' ->>> n1.name() +>>> c1.last = 'tell' +>>> c1.name() 'will tell' ->>> del n1.first ->>> n1.name() +>>> del c1.first +>>> c1.name() Traceback (most recent call last): ... AttributeError: first ->>> n1.first +>>> c1.first Traceback (most recent call last): ... AttributeError: first ->>> n1.first = 'drew' ->>> n1.first +>>> c1.first = 'drew' +>>> c1.first 'drew' ->>> del n1.number +>>> del c1.number Traceback (most recent call last): ... TypeError: can't delete numeric/char attribute ->>> n1.number=2 ->>> n1.number +>>> c1.number=2 +>>> c1.number 2 ->>> n1.first = 42 ->>> n1.name() +>>> c1.first = 42 +>>> c1.name() '42 tell' ->>> n2 = noddy2.Noddy() ->>> n2.name() +>>> c2 = custom2.Custom() +>>> c2.name() ' ' ->>> n2.first +>>> c2.first '' ->>> n2.last +>>> c2.last '' ->>> del n2.first ->>> n2.first +>>> del c2.first +>>> c2.first Traceback (most recent call last): ... AttributeError: first ->>> n2.first +>>> c2.first Traceback (most recent call last): ... AttributeError: first ->>> n2.name() +>>> c2.name() Traceback (most recent call last): File "", line 1, in ? AttributeError: first ->>> n2.number +>>> c2.number 0 ->>> n3 = noddy2.Noddy('jim', 'fulton', 'waaa') +>>> n3 = custom2.Custom('jim', 'fulton', 'waaa') Traceback (most recent call last): File "", line 1, in ? -TypeError: an integer is required ->>> del n1 ->>> del n2 +TypeError: an integer is required (got type str) +>>> del c1 +>>> del c2 -Noddy 3 +Custom 3 ->>> import noddy3 ->>> n1 = noddy3.Noddy('jim', 'fulton', 42) ->>> n1 = noddy3.Noddy('jim', 'fulton', 42) ->>> n1.name() +>>> import custom3 +>>> c1 = custom3.Custom('jim', 'fulton', 42) +>>> c1 = custom3.Custom('jim', 'fulton', 42) +>>> c1.name() 'jim fulton' ->>> del n1.first +>>> del c1.first Traceback (most recent call last): File "", line 1, in ? TypeError: Cannot delete the first attribute ->>> n1.first = 42 +>>> c1.first = 42 Traceback (most recent call last): File "", line 1, in ? TypeError: The first attribute value must be a string ->>> n1.first = 'will' ->>> n1.name() +>>> c1.first = 'will' +>>> c1.name() 'will fulton' ->>> n2 = noddy3.Noddy() ->>> n2 = noddy3.Noddy() ->>> n2 = noddy3.Noddy() ->>> n3 = noddy3.Noddy('jim', 'fulton', 'waaa') +>>> c2 = custom3.Custom() +>>> c2 = custom3.Custom() +>>> c2 = custom3.Custom() +>>> n3 = custom3.Custom('jim', 'fulton', 'waaa') Traceback (most recent call last): File "", line 1, in ? -TypeError: an integer is required ->>> del n1 ->>> del n2 +TypeError: an integer is required (got type str) +>>> del c1 +>>> del c2 -Noddy 4 +Custom 4 ->>> import noddy4 ->>> n1 = noddy4.Noddy('jim', 'fulton', 42) ->>> n1.first +>>> import custom4 +>>> c1 = custom4.Custom('jim', 'fulton', 42) +>>> c1.first 'jim' ->>> n1.last +>>> c1.last 'fulton' ->>> n1.number +>>> c1.number 42 ->>> n1.name() +>>> c1.name() 'jim fulton' ->>> n1.first = 'will' ->>> n1.name() +>>> c1.first = 'will' +>>> c1.name() 'will fulton' ->>> n1.last = 'tell' ->>> n1.name() +>>> c1.last = 'tell' +>>> c1.name() 'will tell' ->>> del n1.first ->>> n1.name() +>>> del c1.first Traceback (most recent call last): ... -AttributeError: first ->>> n1.first -Traceback (most recent call last): -... -AttributeError: first ->>> n1.first = 'drew' ->>> n1.first +TypeError: Cannot delete the first attribute +>>> c1.name() +'will tell' +>>> c1.first = 'drew' +>>> c1.first 'drew' ->>> del n1.number +>>> del c1.number Traceback (most recent call last): ... TypeError: can't delete numeric/char attribute ->>> n1.number=2 ->>> n1.number +>>> c1.number=2 +>>> c1.number 2 ->>> n1.first = 42 ->>> n1.name() -'42 tell' ->>> n2 = noddy4.Noddy() ->>> n2 = noddy4.Noddy() ->>> n2 = noddy4.Noddy() ->>> n2 = noddy4.Noddy() ->>> n2.name() +>>> c1.first = 42 +Traceback (most recent call last): +... +TypeError: The first attribute value must be a string +>>> c1.name() +'drew tell' +>>> c2 = custom4.Custom() +>>> c2 = custom4.Custom() +>>> c2 = custom4.Custom() +>>> c2 = custom4.Custom() +>>> c2.name() ' ' ->>> n2.first +>>> c2.first '' ->>> n2.last +>>> c2.last '' ->>> del n2.first ->>> n2.first -Traceback (most recent call last): -... -AttributeError: first ->>> n2.first -Traceback (most recent call last): -... -AttributeError: first ->>> n2.name() -Traceback (most recent call last): - File "", line 1, in ? -AttributeError: first ->>> n2.number +>>> c2.number 0 ->>> n3 = noddy4.Noddy('jim', 'fulton', 'waaa') +>>> n3 = custom4.Custom('jim', 'fulton', 'waaa') Traceback (most recent call last): - File "", line 1, in ? -TypeError: an integer is required +... +TypeError: an integer is required (got type str) Test cyclic gc(?) @@ -183,15 +170,14 @@ >>> import gc >>> gc.disable() ->>> x = [] ->>> l = [x] ->>> n2.first = l ->>> n2.first -[[]] ->>> l.append(n2) ->>> del l ->>> del n1 ->>> del n2 +>>> class Subclass(custom4.Custom): pass +... +>>> s = Subclass() +>>> s.cycle = [s] +>>> s.cycle.append(s.cycle) +>>> x = object() +>>> s.x = x +>>> del s >>> sys.getrefcount(x) 3 >>> ignore = gc.collect() diff --git a/Doc/install/index.rst b/Doc/install/index.rst index 0545b8f33d0378b..92cdf2f11443fb2 100644 --- a/Doc/install/index.rst +++ b/Doc/install/index.rst @@ -283,7 +283,9 @@ Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> Python (command line)`. Once the interpreter is started, you type Python code at the prompt. For example, on my Linux system, I type the three Python statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`:: +:file:`{prefix}` and :file:`{exec-prefix}`: + +.. code-block:: pycon Python 2.4 (#26, Aug 7 2004, 17:19:02) Type "help", "copyright", "credits" or "license" for more information. @@ -622,7 +624,9 @@ parsing your configuration file(s). Obviously, specifying the entire installation scheme every time you install a new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`):: +into your Distutils config file (see section :ref:`inst-config-files`): + +.. code-block:: ini [install] install-base=$HOME @@ -631,7 +635,9 @@ into your Distutils config file (see section :ref:`inst-config-files`):: install-scripts=python/scripts install-data=python/data -or, equivalently, :: +or, equivalently, + +.. code-block:: ini [install] install-base=$HOME/python @@ -718,7 +724,9 @@ A slightly less convenient way is to edit the :file:`site.py` file in Python's standard library, and modify ``sys.path``. :file:`site.py` is automatically imported when the Python interpreter is executed, unless the :option:`-S` switch is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it:: +:file:`site.py` and add two lines to it: + +.. code-block:: python import sys sys.path.append('/www/python/') @@ -839,7 +847,9 @@ plus a ``global`` section for global options that affect every command. Each section consists of one option per line, specified as ``option=value``. For example, the following is a complete config file that just forces all -commands to run quietly by default:: +commands to run quietly by default: + +.. code-block:: ini [global] verbose=0 @@ -853,7 +863,9 @@ distribution. You could override the default "build base" directory and make the :command:`build\*` commands always forcibly rebuild all files with the -following:: +following: + +.. code-block:: ini [build] build-base=blib diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst index 9522dd62049138b..70710761a39555b 100644 --- a/Doc/library/abc.rst +++ b/Doc/library/abc.rst @@ -160,7 +160,7 @@ a helper class :class:`ABC` to alternatively define ABCs through inheritance: -The :mod:`abc` module also provides the following decorators: +The :mod:`abc` module also provides the following decorator: .. decorator:: abstractmethod @@ -236,8 +236,15 @@ The :mod:`abc` module also provides the following decorators: multiple-inheritance. +The :mod:`abc` module also supports the following legacy decorators: + .. decorator:: abstractclassmethod + .. versionadded:: 3.2 + .. deprecated:: 3.3 + It is now possible to use :class:`classmethod` with + :func:`abstractmethod`, making this decorator redundant. + A subclass of the built-in :func:`classmethod`, indicating an abstract classmethod. Otherwise it is similar to :func:`abstractmethod`. @@ -251,14 +258,14 @@ The :mod:`abc` module also provides the following decorators: def my_abstract_classmethod(cls, ...): ... + +.. decorator:: abstractstaticmethod + .. versionadded:: 3.2 .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with + It is now possible to use :class:`staticmethod` with :func:`abstractmethod`, making this decorator redundant. - -.. decorator:: abstractstaticmethod - A subclass of the built-in :func:`staticmethod`, indicating an abstract staticmethod. Otherwise it is similar to :func:`abstractmethod`. @@ -272,23 +279,17 @@ The :mod:`abc` module also provides the following decorators: def my_abstract_staticmethod(...): ... - .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. - .. decorator:: abstractproperty + .. deprecated:: 3.3 + It is now possible to use :class:`property`, :meth:`property.getter`, + :meth:`property.setter` and :meth:`property.deleter` with + :func:`abstractmethod`, making this decorator redundant. + A subclass of the built-in :func:`property`, indicating an abstract property. - Using this function requires that the class's metaclass is :class:`ABCMeta` - or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods - and properties are overridden. The abstract properties can be called using - any of the normal 'super' call mechanisms. - This special case is deprecated, as the :func:`property` decorator is now correctly identified as abstract when applied to an abstract method:: @@ -322,12 +323,6 @@ The :mod:`abc` module also provides the following decorators: ... - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. - - The :mod:`abc` module also provides the following functions: .. function:: get_cache_token() diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst index 53e670161dd5b36..3f327c0cfa77b7d 100644 --- a/Doc/library/argparse.rst +++ b/Doc/library/argparse.rst @@ -712,7 +712,7 @@ be positional:: Namespace(bar='BAR', foo='FOO') >>> parser.parse_args(['--foo', 'FOO']) usage: PROG [-h] [-f FOO] bar - PROG: error: too few arguments + PROG: error: the following arguments are required: bar action @@ -898,7 +898,7 @@ values are: Namespace(foo=['a', 'b']) >>> parser.parse_args([]) usage: PROG [-h] foo [foo ...] - PROG: error: too few arguments + PROG: error: the following arguments are required: foo .. _`argparse.REMAINDER`: @@ -981,7 +981,7 @@ is used when no command-line argument was present:: Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the -command-line argument was not present.:: +command-line argument was not present:: >>> parser = argparse.ArgumentParser() >>> parser.add_argument('--foo', default=argparse.SUPPRESS) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b7f610ba8b2a5fa..a75a6afbf2d7df6 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -113,6 +113,11 @@ and classes for traversing abstract syntax trees: Parse the source into an AST node. Equivalent to ``compile(source, filename, mode, ast.PyCF_ONLY_AST)``. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. function:: literal_eval(node_or_string) @@ -126,6 +131,11 @@ and classes for traversing abstract syntax trees: capable of evaluating arbitrarily complex expressions, for example involving operators or indexing. + .. warning:: + It is possible to crash the Python interpreter with a + sufficiently large/complex string due to stack depth limitations + in Python's AST compiler. + .. versionchanged:: 3.2 Now allows bytes and set literals. diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index 5f915c5c43921ad..ca8055bd162f695 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -171,7 +171,7 @@ a different clock than :func:`time.time`. Arrange for the *callback* to be called after the given *delay* seconds (either an int or float). - An instance of :class:`asyncio.Handle` is returned, which can be + An instance of :class:`asyncio.TimerHandle` is returned, which can be used to cancel the callback. *callback* will be called exactly once per call to :meth:`call_later`. @@ -193,7 +193,7 @@ a different clock than :func:`time.time`. This method's behavior is the same as :meth:`call_later`. - An instance of :class:`asyncio.Handle` is returned, which can be + An instance of :class:`asyncio.TimerHandle` is returned, which can be used to cancel the callback. :ref:`Use functools.partial to pass keywords to the callback @@ -1043,7 +1043,7 @@ Server async def main(host, port): srv = await asyncio.start_server( client_connected, host, port) - await loop.serve_forever() + await srv.serve_forever() asyncio.run(main('127.0.0.1', 0)) @@ -1076,8 +1076,7 @@ Handle .. class:: Handle A callback wrapper object returned by :func:`AbstractEventLoop.call_soon`, - :func:`AbstractEventLoop.call_soon_threadsafe`, :func:`AbstractEventLoop.call_later`, - and :func:`AbstractEventLoop.call_at`. + :func:`AbstractEventLoop.call_soon_threadsafe`. .. method:: cancel() @@ -1090,6 +1089,22 @@ Handle .. versionadded:: 3.7 +.. class:: TimerHandle + + A callback wrapper object returned by :func:`AbstractEventLoop.call_later`, + and :func:`AbstractEventLoop.call_at`. + + The class is inherited from :class:`Handle`. + + .. method:: when() + + Return a scheduled callback time as :class:`float` seconds. + + The time is an absolute timestamp, using the same time + reference as :meth:`AbstractEventLoop.time`. + + .. versionadded:: 3.7 + SendfileNotAvailableError ------------------------- @@ -1099,7 +1114,7 @@ SendfileNotAvailableError Sendfile syscall is not available, subclass of :exc:`RuntimeError`. - Raised if the OS does not support senfile syscall for + Raised if the OS does not support sendfile syscall for given socket or file type. diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index 004cac80d90c2cc..ef6441605cd72c2 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -339,7 +339,7 @@ Protocol classes control of the receive buffer. .. versionadded:: 3.7 - **Important:** this has been been added to asyncio in Python 3.7 + **Important:** this has been added to asyncio in Python 3.7 *on a provisional basis*! Treat it as an experimental API that might be changed or removed in Python 3.8. @@ -450,7 +450,7 @@ Streaming protocols with manual receive buffer control ------------------------------------------------------ .. versionadded:: 3.7 - **Important:** :class:`BufferedProtocol` has been been added to + **Important:** :class:`BufferedProtocol` has been added to asyncio in Python 3.7 *on a provisional basis*! Consider it as an experimental API that might be changed or removed in Python 3.8. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 71dbe06c899f184..364323d5ca8b65b 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -535,7 +535,7 @@ Task functions not provided, the default event loop is used. -.. function:: current_task(loop=None): +.. function:: current_task(loop=None) Return the current running :class:`Task` instance or ``None``, if no task is running. @@ -546,7 +546,7 @@ Task functions .. versionadded:: 3.7 -.. function:: all_tasks(loop=None): +.. function:: all_tasks(loop=None) Return a set of :class:`Task` objects created for the loop. @@ -682,7 +682,7 @@ Task functions This function is a :ref:`coroutine `. -.. function:: shield(arg, \*, loop=None) +.. coroutinefunction:: shield(arg, \*, loop=None) Wait for a future, shielding it from cancellation. diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst index ceecf17cba23e3c..ad9f5f58bee2aac 100644 --- a/Doc/library/base64.rst +++ b/Doc/library/base64.rst @@ -218,14 +218,6 @@ The modern interface provides: .. versionadded:: 3.4 -.. note:: - Both Base85 and Ascii85 have an expansion factor of 5 to 4 (5 Base85 or - Ascii85 characters can encode 4 binary bytes), while the better-known - Base64 has an expansion factor of 6 to 4. They are therefore more - efficient when space expensive. They differ by details such as the - character map used for encoding. - - The legacy interface: .. function:: decode(input, output) diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst index b60e1cc41e4be06..17386b831833421 100644 --- a/Doc/library/cgi.rst +++ b/Doc/library/cgi.rst @@ -289,11 +289,13 @@ algorithms implemented in this module in other circumstances. This function is deprecated in this module. Use :func:`urllib.parse.parse_qs` instead. It is maintained here only for backward compatibility. + .. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False) This function is deprecated in this module. Use :func:`urllib.parse.parse_qsl` instead. It is maintained here only for backward compatibility. + .. function:: parse_multipart(fp, pdict, encoding="utf-8") Parse input of type :mimetype:`multipart/form-data` (for file uploads). @@ -309,6 +311,10 @@ algorithms implemented in this module in other circumstances. uploaded --- in that case, use the :class:`FieldStorage` class instead which is much more flexible. + .. versionchanged:: 3.7 + Added the *encoding* parameter. For non-file fields, the value is now + a list of strings, not bytes. + .. function:: parse_header(string) diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst index 772ff60fe9839ac..2a83d30372770ba 100644 --- a/Doc/library/collections.rst +++ b/Doc/library/collections.rst @@ -114,7 +114,7 @@ The class can be used to simulate nested scopes and is useful in templating. for templating is a read-only chain of mappings. It also features pushing and popping of contexts similar to the :meth:`~collections.ChainMap.new_child` method and the - :meth:`~collections.ChainMap.parents` property. + :attr:`~collections.ChainMap.parents` property. * The `Nested Contexts recipe `_ has options to control @@ -270,7 +270,7 @@ For example:: Return a list of the *n* most common elements and their counts from the most common to the least. If *n* is omitted or ``None``, - :func:`most_common` returns *all* elements in the counter. + :meth:`most_common` returns *all* elements in the counter. Elements with equal counts are ordered arbitrarily: >>> Counter('abracadabra').most_common(3) # doctest: +SKIP @@ -357,12 +357,12 @@ or subtracting from an empty counter. restrictions on its keys and values. The values are intended to be numbers representing counts, but you *could* store anything in the value field. - * The :meth:`most_common` method requires only that the values be orderable. + * The :meth:`~Counter.most_common` method requires only that the values be orderable. * For in-place operations such as ``c[key] += 1``, the value type need only support addition and subtraction. So fractions, floats, and decimals would work and negative values are supported. The same is also true for - :meth:`update` and :meth:`subtract` which allow negative and zero values + :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values for both inputs and outputs. * The multiset methods are designed only for use cases with positive values. @@ -370,7 +370,7 @@ or subtracting from an empty counter. are created. There are no type restrictions, but the value type needs to support addition, subtraction, and comparison. - * The :meth:`elements` method requires integer counts. It ignores zero and + * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and negative counts. .. seealso:: @@ -388,9 +388,9 @@ or subtracting from an empty counter. Section 4.6.3, Exercise 19*. * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`: + elements, see :func:`itertools.combinations_with_replacement`:: - map(Counter, combinations_with_replacement('ABC', 2)) --> AA AB AC BB BC CC + map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC :class:`deque` objects @@ -509,11 +509,14 @@ or subtracting from an empty counter. .. versionadded:: 3.2 - .. method:: rotate(n) + .. method:: rotate(n=1) - Rotate the deque *n* steps to the right. If *n* is negative, rotate to - the left. Rotating one step to the right is equivalent to: - ``d.appendleft(d.pop())``. + Rotate the deque *n* steps to the right. If *n* is negative, rotate + to the left. + + When the deque is not empty, rotating one step to the right is equivalent + to ``d.appendleft(d.pop())``, and rotating one step to the left is + equivalent to ``d.append(d.popleft())``. Deque objects also provide one read-only attribute: @@ -637,9 +640,9 @@ the :meth:`~deque.rotate` method:: # Remove an exhausted iterator. iterators.popleft() -The :meth:`rotate` method provides a way to implement :class:`deque` slicing and +The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the :meth:`rotate` method to position elements to be popped:: +the ``rotate()`` method to position elements to be popped:: def delete_nth(d, n): d.rotate(-n) @@ -647,8 +650,8 @@ the :meth:`rotate` method to position elements to be popped:: d.rotate(n) To implement :class:`deque` slicing, use a similar approach applying -:meth:`rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`popleft`, add new entries with :meth:`extend`, and then +:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove +old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then reverse the rotation. With minor variations on that approach, it is easy to implement Forth style stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, @@ -709,7 +712,7 @@ stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, :class:`defaultdict` Examples ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Using :class:`list` as the :attr:`default_factory`, it is easy to group a +Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a sequence of key-value pairs into a dictionary of lists: >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] @@ -721,7 +724,7 @@ sequence of key-value pairs into a dictionary of lists: [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`default_factory` +mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` function which returns an empty :class:`list`. The :meth:`list.append` operation then attaches the value to the new list. When keys are encountered again, the look-up proceeds normally (returning the list for that key) and the @@ -735,7 +738,7 @@ simpler and faster than an equivalent technique using :meth:`dict.setdefault`: >>> sorted(d.items()) [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] -Setting the :attr:`default_factory` to :class:`int` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the :class:`defaultdict` useful for counting (like a bag or multiset in other languages): @@ -748,7 +751,7 @@ languages): [('i', 4), ('m', 1), ('p', 2), ('s', 4)] When a letter is first encountered, it is missing from the mapping, so the -:attr:`default_factory` function calls :func:`int` to supply a default count of +:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of zero. The increment operation then builds up the count for each letter. The function :func:`int` which always returns zero is just a special case of @@ -763,7 +766,7 @@ zero): >>> '%(name)s %(action)s to %(object)s' % d 'John ran to ' -Setting the :attr:`default_factory` to :class:`set` makes the +Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the :class:`defaultdict` useful for building a dictionary of sets: >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] @@ -970,7 +973,7 @@ The subclass shown above sets ``__slots__`` to an empty tuple. This helps keep memory requirements low by preventing the creation of instance dictionaries. Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`_fields` attribute: +create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) @@ -986,7 +989,7 @@ fields: .. versionchanged:: 3.5 Property docstrings became writeable. -Default values can be implemented by using :meth:`_replace` to +Default values can be implemented by using :meth:`~somenamedtuple._replace` to customize a prototype instance: >>> Account = namedtuple('Account', 'owner balance transaction_count') @@ -1197,15 +1200,22 @@ subclass directly from :class:`str`; however, this class can be easier to work with because the underlying string is accessible as an attribute. -.. class:: UserString([sequence]) +.. class:: UserString(seq) - Class that simulates a string or a Unicode string object. The instance's + Class that simulates a string object. The instance's content is kept in a regular string object, which is accessible via the :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *sequence*. The *sequence* can - be an instance of :class:`bytes`, :class:`str`, :class:`UserString` (or a - subclass) or an arbitrary sequence which can be converted into a string using - the built-in :func:`str` function. + contents are initially set to a copy of *seq*. The *seq* argument can + be any object which can be converted into a string using the built-in + :func:`str` function. + + In addition to supporting the methods and operations of strings, + :class:`UserString` instances provide the following attribute: + + .. attribute:: data + + A real :class:`str` object used to store the contents of the + :class:`UserString` class. .. versionchanged:: 3.5 New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst index 469a3eed606ff07..78f161963698108 100644 --- a/Doc/library/constants.rst +++ b/Doc/library/constants.rst @@ -86,10 +86,14 @@ should not be used in programs. specified exit code. .. data:: copyright - license credits - Objects that when printed, print a message like "Type license() to see the - full license text", and when called, display the corresponding text in a + Objects that when printed or called, print the text of copyright or + credits, respectively. + +.. data:: license + + Object that when printed, prints the message "Type license() to see the + full license text", and when called, displays the full license text in a pager-like fashion (one screen at a time). diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst new file mode 100644 index 000000000000000..abd0d5fa0fdf970 --- /dev/null +++ b/Doc/library/contextvars.rst @@ -0,0 +1,279 @@ +:mod:`contextvars` --- Context Variables +======================================== + +.. module:: contextvars + :synopsis: Context Variables + +.. sectionauthor:: Yury Selivanov + +-------------- + +This module provides APIs to manage, store, and access non-local +state. The :class:`~contextvars.ContextVar` class is used to declare +and work with *Context Variables*. The :func:`~contextvars.copy_context` +function and the :class:`~contextvars.Context` class should be used to +manage the current context in asynchronous frameworks. + +Context managers that have state should use Context Variables +instead of :func:`threading.local()` to prevent their state from +bleeding to other code unexpectedly, when used in concurrent code. + +See also :pep:`567` for additional details. + +.. versionadded:: 3.7 + + +Context Variables +----------------- + +.. class:: ContextVar(name, [\*, default]) + + This class is used to declare a new Context Variable, e.g.:: + + var: ContextVar[int] = ContextVar('var', default=42) + + The required *name* parameter is used for introspection and debug + purposes. + + The optional keyword-only *default* parameter is returned by + :meth:`ContextVar.get` when no value for the variable is found + in the current context. + + **Important:** Context Variables should be created at the top module + level and never in closures. :class:`Context` objects hold strong + references to context variables which prevents context variables + from being properly garbage collected. + + .. attribute:: ContextVar.name + + The name of the variable. This is a read-only property. + + .. method:: get([default]) + + Return a value for the context variable for the current context. + + If there is no value for the variable in the current context, + the method will: + + * return the value of the *default* argument of the method, + if provided; or + + * return the default value for the context variable, + if it was created with one; or + + * raise a :exc:`LookupError`. + + .. method:: set(value) + + Call to set a new value for the context variable in the current + context. + + The required *value* argument is the new value for the context + variable. + + Returns a :class:`~contextvars.Token` object that can be used + to restore the variable to its previous value via the + :meth:`ContextVar.reset` method. + + .. method:: reset(token) + + Reset the context variable to the value it had before the + :meth:`ContextVar.set` that created the *token* was used. + + For example:: + + var = ContextVar('var') + + token = var.set('new value') + # code that uses 'var'; var.get() returns 'new value'. + var.reset(token) + + # After the reset call the var has no value again, so + # var.get() would raise a LookupError. + + +.. class:: contextvars.Token + + *Token* objects are returned by the :meth:`ContextVar.set` method. + They can be passed to the :meth:`ContextVar.reset` method to revert + the value of the variable to what it was before the corresponding + *set*. + + .. attribute:: Token.var + + A read-only property. Points to the :class:`ContextVar` object + that created the token. + + .. attribute:: Token.old_value + + A read-only property. Set to the value the variable had before + the :meth:`ContextVar.set` method call that created the token. + It points to :attr:`Token.MISSING` is the variable was not set + before the call. + + .. attribute:: Token.MISSING + + A marker object used by :attr:`Token.old_value`. + + +Manual Context Management +------------------------- + +.. function:: copy_context() + + Returns a copy of the current :class:`~contextvars.Context` object. + + The following snippet gets a copy of the current context and prints + all variables and their values that are set in it:: + + ctx: Context = copy_context() + print(list(ctx.items())) + + The function has an O(1) complexity, i.e. works equally fast for + contexts with a few context variables and for contexts that have + a lot of them. + + +.. class:: Context() + + A mapping of :class:`ContextVars ` to their values. + + ``Context()`` creates an empty context with no values in it. + To get a copy of the current context use the + :func:`~contextvars.copy_context` function. + + Context implements the :class:`collections.abc.Mapping` interface. + + .. method:: run(callable, \*args, \*\*kwargs) + + Execute ``callable(*args, **kwargs)`` code in the context object + the *run* method is called on. Return the result of the execution + or propagate an exception if one occurred. + + Any changes to any context variables that *callable* makes will + be contained in the context object:: + + var = ContextVar('var') + var.set('spam') + + def main(): + # 'var' was set to 'spam' before + # calling 'copy_context()' and 'ctx.run(main)', so: + # var.get() == ctx[var] == 'spam' + + var.set('ham') + + # Now, after setting 'var' to 'ham': + # var.get() == ctx[var] == 'ham' + + ctx = copy_context() + + # Any changes that the 'main' function makes to 'var' + # will be contained in 'ctx'. + ctx.run(main) + + # The 'main()' function was run in the 'ctx' context, + # so changes to 'var' are contained in it: + # ctx[var] == 'ham' + + # However, outside of 'ctx', 'var' is still set to 'spam': + # var.get() == 'spam' + + The method raises a :exc:`RuntimeError` when called on the same + context object from more than one OS thread, or when called + recursively. + + .. method:: copy() + + Return a shallow copy of the context object. + + .. describe:: var in context + + Return ``True`` if the *context* has a value for *var* set; + return ``False`` otherwise. + + .. describe:: context[var] + + Return the value of the *var* :class:`ContextVar` variable. + If the variable is not set in the context object, a + :exc:`KeyError` is raised. + + .. method:: get(var, [default]) + + Return the value for *var* if *var* has the value in the context + object. Return *default* otherwise. If *default* is not given, + return ``None``. + + .. describe:: iter(context) + + Return an iterator over the variables stored in the context + object. + + .. describe:: len(proxy) + + Return the number of variables set in the context object. + + .. method:: keys() + + Return a list of all variables in the context object. + + .. method:: values() + + Return a list of all variables' values in the context object. + + + .. method:: items() + + Return a list of 2-tuples containing all variables and their + values in the context object. + + +asyncio support +--------------- + +Context variables are natively supported in :mod:`asyncio` and are +ready to be used without any extra configuration. For example, here +is a simple echo server, that uses a context variable to make the +address of a remote client available in the Task that handles that +client:: + + import asyncio + import contextvars + + client_addr_var = contextvars.ContextVar('client_addr') + + def render_goodbye(): + # The address of the currently handled client can be accessed + # without passing it explicitly to this function. + + client_addr = client_addr_var.get() + return f'Good bye, client @ {client_addr}\n'.encode() + + async def handle_request(reader, writer): + addr = writer.transport.get_extra_info('socket').getpeername() + client_addr_var.set(addr) + + # In any code that we call is now possible to get + # client's address by calling 'client_addr_var.get()'. + + while True: + line = await reader.readline() + print(line) + if not line.strip(): + break + writer.write(line) + + writer.write(render_goodbye()) + writer.close() + + async def main(): + srv = await asyncio.start_server( + handle_request, '127.0.0.1', 8081) + + async with srv: + await srv.serve_forever() + + asyncio.run(main()) + + # To test it you can use telnet: + # telnet 127.0.0.1 8081 diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst index 13717f7b35e6b73..2a2ee2be84af2ac 100644 --- a/Doc/library/curses.rst +++ b/Doc/library/curses.rst @@ -685,6 +685,12 @@ the following methods and attributes: character previously painter at that location. By default, the character position and attributes are the current settings for the window object. + .. note:: + + Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. + Attempting to write to the lower right corner of a window, subwindow, + or pad will cause an exception to be raised after the character is printed. + .. method:: window.addnstr(str, n[, attr]) window.addnstr(y, x, str, n[, attr]) @@ -700,6 +706,12 @@ the following methods and attributes: Paint the character string *str* at ``(y, x)`` with attributes *attr*, overwriting anything previously on the display. + .. note:: + + Writing outside the window, subwindow, or pad raises :exc:`curses.error`. + Attempting to write to the lower right corner of a window, subwindow, + or pad will cause an exception to be raised after the string is printed. + .. method:: window.attroff(attr) diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst index c1b164ebc1f23ab..8d91f4ef9346541 100644 --- a/Doc/library/datetime.rst +++ b/Doc/library/datetime.rst @@ -2209,8 +2209,8 @@ Notes: :meth:`utcoffset` is transformed into a string of the form ±HHMM[SS[.uuuuuu]], where HH is a 2-digit string giving the number of UTC offset hours, and MM is a 2-digit string giving the number of UTC offset - minutes, SS is a 2-digit string string giving the number of UTC offset - seconds and uuuuuu is a 2-digit string string giving the number of UTC + minutes, SS is a 2-digit string giving the number of UTC offset + seconds and uuuuuu is a 2-digit string giving the number of UTC offset microseconds. The uuuuuu part is omitted when the offset is a whole number of minutes and both the uuuuuu and the SS parts are omitted when the offset is a whole number of minutes. For example, if diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst index 32e80b2cf6ed222..0150f5d5c6e8c33 100644 --- a/Doc/library/dbm.rst +++ b/Doc/library/dbm.rst @@ -347,6 +347,11 @@ The module defines the following: database has to be created. It defaults to octal ``0o666`` (and will be modified by the prevailing umask). + .. warning:: + It is possible to crash the Python interpreter when loading a database + with a sufficiently large/complex entry due to stack depth limitations in + Python's AST compiler. + .. versionchanged:: 3.5 :func:`.open` always creates a new database when the flag has the value ``'n'``. diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 48c42d119d5dc9d..535b36efbf2ec8c 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -339,12 +339,16 @@ The Python compiler currently generates the following bytecode instructions. Duplicates the reference on top of the stack. + .. versionadded:: 3.2 + .. opcode:: DUP_TOP_TWO Duplicates the two references on top of the stack, leaving them in the same order. + .. versionadded:: 3.2 + **Unary operations** @@ -555,11 +559,14 @@ the original TOS1. the CO_ITERABLE_COROUTINE flag, or resolves ``o.__await__``. + .. versionadded:: 3.5 + .. opcode:: GET_AITER Implements ``TOS = TOS.__aiter__()``. + .. versionadded:: 3.5 .. versionchanged:: 3.7 Returning awaitable objects from ``__aiter__`` is no longer supported. @@ -570,17 +577,23 @@ the original TOS1. Implements ``PUSH(get_awaitable(TOS.__anext__()))``. See ``GET_AWAITABLE`` for details about ``get_awaitable`` + .. versionadded:: 3.5 + .. opcode:: BEFORE_ASYNC_WITH Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack. + .. versionadded:: 3.5 + .. opcode:: SETUP_ASYNC_WITH Creates a new frame object. + .. versionadded:: 3.5 + **Miscellaneous opcodes** @@ -618,6 +631,8 @@ the original TOS1. Calls ``dict.setitem(TOS1[-i], TOS, TOS1)``. Used to implement dict comprehensions. + .. versionadded:: 3.1 + For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` instructions, while the added value or key/value pair is popped off, the container object remains on the stack so that it is available for further @@ -640,6 +655,7 @@ iterations of the loop. .. versionadded:: 3.3 + .. opcode:: SETUP_ANNOTATIONS Checks whether ``__annotations__`` is defined in ``locals()``, if not it is @@ -649,6 +665,7 @@ iterations of the loop. .. versionadded:: 3.6 + .. opcode:: IMPORT_STAR Loads all symbols not starting with ``'_'`` directly from the module TOS to @@ -694,6 +711,8 @@ iterations of the loop. store it in (a) variable(s) (:opcode:`STORE_FAST`, :opcode:`STORE_NAME`, or :opcode:`UNPACK_SEQUENCE`). + .. versionadded:: 3.2 + .. opcode:: WITH_CLEANUP_START @@ -924,23 +943,31 @@ All of the following opcodes use their arguments. If TOS is true, sets the bytecode counter to *target*. TOS is popped. + .. versionadded:: 3.1 + .. opcode:: POP_JUMP_IF_FALSE (target) If TOS is false, sets the bytecode counter to *target*. TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_IF_TRUE_OR_POP (target) If TOS is true, sets the bytecode counter to *target* and leaves TOS on the stack. Otherwise (TOS is false), TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_IF_FALSE_OR_POP (target) If TOS is false, sets the bytecode counter to *target* and leaves TOS on the stack. Otherwise (TOS is true), TOS is popped. + .. versionadded:: 3.1 + .. opcode:: JUMP_ABSOLUTE (target) @@ -1013,6 +1040,8 @@ All of the following opcodes use their arguments. consulting the cell. This is used for loading free variables in class bodies. + .. versionadded:: 3.4 + .. opcode:: STORE_DEREF (i) @@ -1025,6 +1054,8 @@ All of the following opcodes use their arguments. Empties the cell contained in slot *i* of the cell and free variable storage. Used by the :keyword:`del` statement. + .. versionadded:: 3.2 + .. opcode:: RAISE_VARARGS (argc) diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst index fc65a3d078f19c2..787670c6ddad4ed 100644 --- a/Doc/library/enum.rst +++ b/Doc/library/enum.rst @@ -976,7 +976,7 @@ Enum Classes The :class:`EnumMeta` metaclass is responsible for providing the :meth:`__contains__`, :meth:`__dir__`, :meth:`__iter__` and other methods that allow one to do things with an :class:`Enum` class that fail on a typical -class, such as `list(Color)` or `some_var in Color`. :class:`EnumMeta` is +class, such as `list(Color)` or `some_enum_var in Color`. :class:`EnumMeta` is responsible for ensuring that various other methods on the final :class:`Enum` class are correct (such as :meth:`__new__`, :meth:`__getnewargs__`, :meth:`__str__` and :meth:`__repr__`). diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst index d0c4cd07c57187f..94ebd87639c5019 100644 --- a/Doc/library/faulthandler.rst +++ b/Doc/library/faulthandler.rst @@ -152,10 +152,10 @@ these functions again each time that the file is replaced. Example ------- -.. highlight:: sh - Example of a segmentation fault on Linux with and without enabling the fault -handler:: +handler: + +.. code-block:: shell-session $ python3 -c "import ctypes; ctypes.string_at(0)" Segmentation fault diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst index bfb813cf39060a2..c3b6385723382ba 100644 --- a/Doc/library/functions.rst +++ b/Doc/library/functions.rst @@ -274,6 +274,12 @@ are always available. They are listed here in alphabetical order. character. This is to facilitate detection of incomplete and complete statements in the :mod:`code` module. + .. warning:: + + It is possible to crash the Python interpreter with a + sufficiently large/complex string when compiling to an AST + object due to stack depth limitations in Python's AST compiler. + .. versionchanged:: 3.2 Allowed use of Windows and Mac newlines. Also input in ``'exec'`` mode does not have to end in a newline anymore. Added the *optimize* parameter. @@ -725,8 +731,11 @@ are always available. They are listed here in alphabetical order. Return an integer object constructed from a number or string *x*, or return ``0`` if no arguments are given. If *x* is a number, return - :meth:`x.__int__() `. For floating point numbers, this - truncates towards zero. + :meth:`x.__int__() `. If *x* defines + :meth:`x.__trunc__() ` but not + :meth:`x.__int__() `, then return + if :meth:`x.__trunc__() `. For floating point numbers, + this truncates towards zero. If *x* is not a number or if *base* is given, then *x* must be a string, :class:`bytes`, or :class:`bytearray` instance representing an :ref:`integer diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst index 92240c7606774a0..153d8fb70456238 100644 --- a/Doc/library/gc.rst +++ b/Doc/library/gc.rst @@ -33,34 +33,6 @@ The :mod:`gc` module provides the following functions: Disable automatic garbage collection. -.. class:: ensure_disabled() - - Return a context manager object that disables the garbage collector and reenables the previous - state upon completion of the block. This is basically equivalent to:: - - from gc import enable, disable, isenabled - - @contextmanager - def ensure_disabled(): - was_enabled_previously = isenabled() - gc.disable() - yield - if was_enabled_previously: - gc.enable() - - And lets you write code like this:: - - with ensure_disabled(): - run_some_timing() - - with ensure_disabled(): - # do_something_that_has_real_time_guarantees - # such as a pair trade, robotic braking, etc - - without needing to explicitly enable and disable the garbage collector yourself. - This context manager is implemented in C to assure atomicity, thread safety and speed. - - .. function:: isenabled() Returns true if automatic collection is enabled. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst index a8a5a500cbcfbf3..25bd4b7f9a874c6 100644 --- a/Doc/library/glob.rst +++ b/Doc/library/glob.rst @@ -48,7 +48,7 @@ For example, ``'[?]'`` matches the character ``'?'``. Support for recursive globs using "``**``". -.. function:: iglob(pathname, recursive=False) +.. function:: iglob(pathname, *, recursive=False) Return an :term:`iterator` which yields the same values as :func:`glob` without actually storing them all simultaneously. diff --git a/Doc/library/http.server.rst b/Doc/library/http.server.rst index c98843de02cba31..16cfa1798aef373 100644 --- a/Doc/library/http.server.rst +++ b/Doc/library/http.server.rst @@ -33,9 +33,19 @@ handler. Code to create and run the server looks like this:: :attr:`server_port`. The server is accessible by the handler, typically through the handler's :attr:`server` instance variable. +.. class:: ThreadedHTTPServer(server_address, RequestHandlerClass) -The :class:`HTTPServer` must be given a *RequestHandlerClass* on instantiation, -of which this module provides three different variants: + This class is identical to HTTPServer but uses threads to handle + requests by using the :class:`~socketserver.ThreadingMixIn`. This + is useful to handle web browsers pre-opening sockets, on which + :class:`HTTPServer` would wait indefinitely. + + .. versionadded:: 3.7 + + +The :class:`HTTPServer` and :class:`ThreadedHTTPServer` must be given +a *RequestHandlerClass* on instantiation, of which this module +provides three different variants: .. class:: BaseHTTPRequestHandler(request, client_address, server) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 5fa1d7d869dd9db..130aabf119a6f5f 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -369,9 +369,9 @@ ABC hierarchy:: An abstract base class for a :term:`loader`. See :pep:`302` for the exact definition for a loader. - For loaders that wish to support resource reading, they should - implement a ``get_resource_reader(fullname)`` method as specified - by :class:`importlib.abc.ResourceReader`. + Loaders that wish to support resource reading should implement a + ``get_resource_reader(fullname)`` method as specified by + :class:`importlib.abc.ResourceReader`. .. versionchanged:: 3.7 Introduced the optional ``get_resource_reader()`` method. @@ -813,8 +813,25 @@ Resources are roughly akin to files inside directories, though it's important to keep in mind that this is just a metaphor. Resources and packages **do not** have to exist as physical files and directories on the file system. -Loaders can support resources by implementing the :class:`ResourceReader` -abstract base class. +.. note:: + + This module provides functionality similar to `pkg_resources + `_ `Basic + Resource Access + `_ + without the performance overhead of that package. This makes reading + resources included in packages easier, with more stable and consistent + semantics. + + The standalone backport of this module provides more information + on `using importlib.resources + `_ and + `migrating from pkg_resources to importlib.resources + `_. + +Loaders that wish to support resource reading should implement a +``get_resource_reader(fullname)`` method as specified by +:class:`importlib.abc.ResourceReader`. The following types are defined. @@ -1013,7 +1030,7 @@ find and load modules. .. class:: WindowsRegistryFinder :term:`Finder` for modules declared in the Windows registry. This class - implements the :class:`importlib.abc.Finder` ABC. + implements the :class:`importlib.abc.MetaPathFinder` ABC. Only class methods are defined by this class to alleviate the need for instantiation. @@ -1064,7 +1081,12 @@ find and load modules. .. classmethod:: invalidate_caches() Calls :meth:`importlib.abc.PathEntryFinder.invalidate_caches` on all - finders stored in :attr:`sys.path_importer_cache`. + finders stored in :data:`sys.path_importer_cache` that define the method. + Otherwise entries in :data:`sys.path_importer_cache` set to ``None`` are + deleted. + + .. versionchanged:: 3.7 + Entries of ``None`` in :data:`sys.path_importer_cache` are deleted. .. versionchanged:: 3.4 Calls objects in :data:`sys.path_hooks` with the current working @@ -1291,7 +1313,7 @@ find and load modules. Name of the place from which the module is loaded, e.g. "builtin" for built-in modules and the filename for modules loaded from source. Normally "origin" should be set, but it may be ``None`` (the default) - which indicates it is unspecified. + which indicates it is unspecified (e.g. for namespace packages). .. attribute:: submodule_search_locations diff --git a/Doc/library/index.rst b/Doc/library/index.rst index a925e10ee4988fa..da6a460e2c31377 100644 --- a/Doc/library/index.rst +++ b/Doc/library/index.rst @@ -55,6 +55,7 @@ the `Python Package Index `_. crypto.rst allos.rst concurrency.rst + contextvars.rst ipc.rst netdata.rst markup.rst diff --git a/Doc/library/ipaddress.rst b/Doc/library/ipaddress.rst index 75c9107bd5eb311..b7b502aff15e0bf 100644 --- a/Doc/library/ipaddress.rst +++ b/Doc/library/ipaddress.rst @@ -91,7 +91,8 @@ Address objects The :class:`IPv4Address` and :class:`IPv6Address` objects share a lot of common attributes. Some attributes that are only meaningful for IPv6 addresses are also implemented by :class:`IPv4Address` objects, in order to make it easier to -write code that handles both IP versions correctly. +write code that handles both IP versions correctly. Address objects are +:term:`hashable`, so they can be used as keys in dictionaries. .. class:: IPv4Address(address) @@ -236,7 +237,7 @@ write code that handles both IP versions correctly. groups consisting entirely of zeroes included. - For the following attributes, see the corresponding documention of the + For the following attributes, see the corresponding documentation of the :class:`IPv4Address` class: .. attribute:: packed @@ -368,6 +369,8 @@ All attributes implemented by address objects are implemented by network objects as well. In addition, network objects implement additional attributes. All of these are common between :class:`IPv4Network` and :class:`IPv6Network`, so to avoid duplication they are only documented for :class:`IPv4Network`. +Network objects are :term:`hashable`, so they can be used as keys in +dictionaries. .. class:: IPv4Network(address, strict=True) @@ -377,8 +380,9 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. a slash (``/``). The IP address is the network address, and the mask can be either a single number, which means it's a *prefix*, or a string representation of an IPv4 address. If it's the latter, the mask is - interpreted as a *net mask* if it starts with a non-zero field, or as - a *host mask* if it starts with a zero field. If no mask is provided, + interpreted as a *net mask* if it starts with a non-zero field, or as a + *host mask* if it starts with a zero field, with the single exception of + an all-zero mask which is treated as a *net mask*. If no mask is provided, it's considered to be ``/32``. For example, the following *address* specifications are equivalent: @@ -408,7 +412,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Unless stated otherwise, all network methods accepting other network/address objects will raise :exc:`TypeError` if the argument's IP version is - incompatible to ``self`` + incompatible to ``self``. .. versionchanged:: 3.5 @@ -418,7 +422,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: max_prefixlen Refer to the corresponding attribute documentation in - :class:`IPv4Address` + :class:`IPv4Address`. .. attribute:: is_multicast .. attribute:: is_private @@ -428,7 +432,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: is_link_local These attributes are true for the network as a whole if they are true - for both the network address and the broadcast address + for both the network address and the broadcast address. .. attribute:: network_address @@ -442,7 +446,11 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: hostmask - The host mask, as a string. + The host mask, as an :class:`IPv4Address` object. + + .. attribute:: netmask + + The net mask, as an :class:`IPv4Address` object. .. attribute:: with_prefixlen .. attribute:: compressed @@ -477,12 +485,16 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Returns an iterator over the usable hosts in the network. The usable hosts are all the IP addresses that belong to the network, except the - network address itself and the network broadcast address. + network address itself and the network broadcast address. For networks + with a mask length of 31, the network address and network broadcast + address are also included in the result. >>> list(ip_network('192.0.2.0/29').hosts()) #doctest: +NORMALIZE_WHITESPACE [IPv4Address('192.0.2.1'), IPv4Address('192.0.2.2'), IPv4Address('192.0.2.3'), IPv4Address('192.0.2.4'), IPv4Address('192.0.2.5'), IPv4Address('192.0.2.6')] + >>> list(ip_network('192.0.2.0/31').hosts()) + [IPv4Address('192.0.2.0'), IPv4Address('192.0.2.1')] .. method:: overlaps(other) @@ -586,15 +598,14 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. Construct an IPv6 network definition. *address* can be one of the following: - 1. A string consisting of an IP address and an optional mask, separated by - a slash (``/``). The IP address is the network address, and the mask - can be either a single number, which means it's a *prefix*, or a string - representation of an IPv6 address. If it's the latter, the mask is - interpreted as a *net mask*. If no mask is provided, it's considered to - be ``/128``. + 1. A string consisting of an IP address and an optional prefix length, + separated by a slash (``/``). The IP address is the network address, + and the prefix length must be a single number, the *prefix*. If no + prefix length is provided, it's considered to be ``/128``. - For example, the following *address* specifications are equivalent: - ``2001:db00::0/24`` and ``2001:db00::0/ffff:ff00::``. + Note that currently expanded netmasks are not supported. That means + ``2001:db00::0/24`` is a valid argument while ``2001:db00::0/ffff:ff00::`` + not. 2. An integer that fits into 128 bits. This is equivalent to a single-address network, with the network address being *address* and @@ -631,6 +642,7 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: network_address .. attribute:: broadcast_address .. attribute:: hostmask + .. attribute:: netmask .. attribute:: with_prefixlen .. attribute:: compressed .. attribute:: exploded @@ -639,6 +651,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. attribute:: num_addresses .. attribute:: prefixlen .. method:: hosts() + + Returns an iterator over the usable hosts in the network. The usable + hosts are all the IP addresses that belong to the network, except the + Subnet-Router anycast address. For networks with a mask length of 127, + the Subnet-Router anycast address is also included in the result. + .. method:: overlaps(other) .. method:: address_exclude(network) .. method:: subnets(prefixlen_diff=1, new_prefix=None) @@ -648,12 +666,12 @@ so to avoid duplication they are only documented for :class:`IPv4Network`. .. method:: compare_networks(other) Refer to the corresponding attribute documentation in - :class:`IPv4Network` + :class:`IPv4Network`. .. attribute:: is_site_local These attribute is true for the network as a whole if it is true - for both the network address and the broadcast address + for both the network address and the broadcast address. Operators @@ -667,8 +685,8 @@ IPv6). Logical operators """"""""""""""""" -Network objects can be compared with the usual set of logical operators, -similarly to address objects. +Network objects can be compared with the usual set of logical operators. +Network objects are ordered first by network address, then by net mask. Iteration @@ -718,6 +736,9 @@ Network objects can act as containers of addresses. Some examples:: Interface objects ----------------- +Interface objects are :term:`hashable`, so they can be used as keys in +dictionaries. + .. class:: IPv4Interface(address) Construct an IPv4 interface. The meaning of *address* is as in the @@ -789,6 +810,30 @@ Interface objects :class:`IPv4Interface`. +Operators +^^^^^^^^^ + +Interface objects support some operators. Unless stated otherwise, operators +can only be applied between compatible objects (i.e. IPv4 with IPv4, IPv6 with +IPv6). + + +Logical operators +""""""""""""""""" + +Interface objects can be compared with the usual set of logical operators. + +For equality comparison (``==`` and ``!=``), both the IP address and network +must be the same for the objects to be equal. An interface will not compare +equal to any address or network object. + +For ordering (``<``, ``>``, etc) the rules are different. Interface and +address objects with the same IP version can be compared, and the address +objects will always sort before the interface objects. Two interface objects +are first compared by their networks and, if those are the same, then by their +IP addresses. + + Other Module Level Functions ---------------------------- @@ -854,7 +899,7 @@ The module also provides the following module level functions: doesn't make sense. There are some times however, where you may wish to have :mod:`ipaddress` sort these anyway. If you need to do this, you can use - this function as the ``key`` argument to :func:`sorted()`. + this function as the *key* argument to :func:`sorted()`. *obj* is either a network or address object. @@ -872,4 +917,4 @@ module defines the following exceptions: .. exception:: NetmaskValueError(ValueError) - Any value error related to the netmask. + Any value error related to the net mask. diff --git a/Doc/library/itertools.rst b/Doc/library/itertools.rst index 0b3829f19faf9ce..959424ff914390e 100644 --- a/Doc/library/itertools.rst +++ b/Doc/library/itertools.rst @@ -436,15 +436,24 @@ loops that truncate the stream. # islice('ABCDEFG', 2, None) --> C D E F G # islice('ABCDEFG', 0, None, 2) --> A C E G s = slice(*args) - it = iter(range(s.start or 0, s.stop or sys.maxsize, s.step or 1)) + start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 + it = iter(range(start, stop, step)) try: nexti = next(it) except StopIteration: + # Consume *iterable* up to the *start* position. + for i, element in zip(range(start), iterable): + pass return - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) + try: + for i, element in enumerate(iterable): + if i == nexti: + yield element + nexti = next(it) + except StopIteration: + # Consume to *stop*. + for i, element in zip(range(i + 1, stop), iterable): + pass If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, then the step defaults to one. @@ -679,6 +688,11 @@ which incur interpreter overhead. "Return first n items of the iterable as a list" return list(islice(iterable, n)) + def prepend(value, iterator): + "Prepend a single value in front of an iterator" + # prepend(1, [2, 3, 4]) -> 1 2 3 4 + return chain([value], iterator) + def tabulate(function, start=0): "Return function(0), function(1), ..." return map(function, count(start)) @@ -688,8 +702,8 @@ which incur interpreter overhead. # tail(3, 'ABCDEFG') --> E F G return iter(collections.deque(iterable, maxlen=n)) - def consume(iterator, n): - "Advance the iterator n-steps ahead. If n is none, consume entirely." + def consume(iterator, n=None): + "Advance the iterator n-steps ahead. If n is None, consume entirely." # Use functions that consume iterators at C speed. if n is None: # feed the entire iterator into a zero-length deque diff --git a/Doc/library/json.rst b/Doc/library/json.rst index 829218d558439b6..85798fa2cf723a7 100644 --- a/Doc/library/json.rst +++ b/Doc/library/json.rst @@ -100,9 +100,9 @@ Extending :class:`JSONEncoder`:: ['[2.0', ', 1.0', ']'] -.. highlight:: bash +Using :mod:`json.tool` from the shell to validate and pretty-print: -Using :mod:`json.tool` from the shell to validate and pretty-print:: +.. code-block:: shell-session $ echo '{"json":"obj"}' | python -m json.tool { @@ -113,8 +113,6 @@ Using :mod:`json.tool` from the shell to validate and pretty-print:: See :ref:`json-commandline` for detailed documentation. -.. highlight:: python3 - .. note:: JSON is a subset of `YAML `_ 1.2. The JSON produced by @@ -230,10 +228,8 @@ Basic Usage *object_pairs_hook* is an optional function that will be called with the result of any object literal decoded with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders that - rely on the order that the key and value pairs are decoded (for example, - :func:`collections.OrderedDict` will remember the order of insertion). If - *object_hook* is also defined, the *object_pairs_hook* takes priority. + :class:`dict`. This feature can be used to implement custom decoders. + If *object_hook* is also defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. @@ -325,10 +321,8 @@ Encoders and Decoders *object_pairs_hook*, if specified will be called with the result of every JSON object decoded with an ordered list of pairs. The return value of *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders that rely on the order - that the key and value pairs are decoded (for example, - :func:`collections.OrderedDict` will remember the order of insertion). If - *object_hook* is also defined, the *object_pairs_hook* takes priority. + feature can be used to implement custom decoders. If *object_hook* is also + defined, the *object_pairs_hook* takes priority. .. versionchanged:: 3.1 Added support for *object_pairs_hook*. @@ -651,8 +645,6 @@ when serializing Python :class:`int` values of extremely large magnitude, or when serializing instances of "exotic" numerical types such as :class:`decimal.Decimal`. -.. highlight:: bash - .. _json-commandline: Command Line Interface @@ -669,7 +661,9 @@ The :mod:`json.tool` module provides a simple command line interface to validate and pretty-print JSON objects. If the optional ``infile`` and ``outfile`` arguments are not -specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively:: +specified, :attr:`sys.stdin` and :attr:`sys.stdout` will be used respectively: + +.. code-block:: shell-session $ echo '{"json": "obj"}' | python -m json.tool { @@ -688,7 +682,9 @@ Command line options .. cmdoption:: infile - The JSON file to be validated or pretty-printed:: + The JSON file to be validated or pretty-printed: + + .. code-block:: shell-session $ python -m json.tool mp_films.json [ diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst index 06378379c331cff..1f9d7c7f220b3ce 100644 --- a/Doc/library/logging.config.rst +++ b/Doc/library/logging.config.rst @@ -538,7 +538,9 @@ target handler, and the system will resolve to the handler from the id. If, however, a user defines a ``my.package.MyHandler`` which has an ``alternate`` handler, the configuration system would not know that the ``alternate`` referred to a handler. To cater for this, a generic -resolution system allows the user to specify:: +resolution system allows the user to specify: + +.. code-block:: yaml handlers: file: @@ -552,7 +554,9 @@ The literal string ``'cfg://handlers.file'`` will be resolved in an analogous way to strings with the ``ext://`` prefix, but looking in the configuration itself rather than the import namespace. The mechanism allows access by dot or by index, in a similar way to -that provided by ``str.format``. Thus, given the following snippet:: +that provided by ``str.format``. Thus, given the following snippet: + +.. code-block:: yaml handlers: email: diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst index 0974286e55dc10b..f7262e60a31baa8 100644 --- a/Doc/library/logging.handlers.rst +++ b/Doc/library/logging.handlers.rst @@ -524,7 +524,7 @@ over UDP sockets. .. versionchanged:: 3.4 If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. + using the value in ``host`` - otherwise, a UDP socket is created. .. method:: emit() diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst index 1ed129c00d49072..f9eda173ad0dc91 100644 --- a/Doc/library/logging.rst +++ b/Doc/library/logging.rst @@ -179,7 +179,9 @@ is the module's name in the Python package namespace. You can specify *stack_info* independently of *exc_info*, e.g. to just show how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says:: + raised. The stack frames are printed following a header line which says: + + .. code-block:: none Stack (most recent call last): @@ -198,7 +200,9 @@ is the module's name in the Python package namespace. logger = logging.getLogger('tcpserver') logger.warning('Protocol problem: %s', 'connection reset', extra=d) - would print something like :: + would print something like + + .. code-block:: none 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset @@ -328,7 +332,7 @@ is the module's name in the Python package namespace. .. versionadded:: 3.2 .. versionchanged:: 3.7 - Loggers can now be picked and unpickled. + Loggers can now be pickled and unpickled. .. _levels: @@ -939,7 +943,9 @@ functions. You can specify *stack_info* independently of *exc_info*, e.g. to just show how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says:: + raised. The stack frames are printed following a header line which says: + + .. code-block:: none Stack (most recent call last): @@ -957,7 +963,9 @@ functions. d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} logging.warning('Protocol problem: %s', 'connection reset', extra=d) - would print something like:: + would print something like: + + .. code-block:: none 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset diff --git a/Doc/library/math.rst b/Doc/library/math.rst index 55eb41b86f5d162..33aec573a1f281b 100644 --- a/Doc/library/math.rst +++ b/Doc/library/math.rst @@ -203,7 +203,7 @@ Number-theoretic and representation functions Return the :class:`~numbers.Real` value *x* truncated to an :class:`~numbers.Integral` (usually an integer). Delegates to - ``x.__trunc__()``. + :meth:`x.__trunc__() `. Note that :func:`frexp` and :func:`modf` have a different call/return pattern diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst index 184119df5918832..ca09a6a3ca9955d 100644 --- a/Doc/library/mmap.rst +++ b/Doc/library/mmap.rst @@ -127,7 +127,7 @@ To map anonymous memory, -1 should be passed as the fileno along with the length :class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with` - statement.:: + statement:: import mmap diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst index 2ef187db2dcbfa5..337c7c2994169b0 100644 --- a/Doc/library/optparse.rst +++ b/Doc/library/optparse.rst @@ -567,7 +567,7 @@ An option group is obtained using the class :class:`OptionGroup`: where - * parser is the :class:`OptionParser` instance the group will be insterted in + * parser is the :class:`OptionParser` instance the group will be inserted in to * title is the group title * description, optional, is a long description of the group diff --git a/Doc/library/os.rst b/Doc/library/os.rst index bae432d33b0bd44..d269d0b0eaa6ce0 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -2812,7 +2812,7 @@ features: no effect on the behavior of the walk, because in bottom-up mode the directories in *dirnames* are generated before *dirpath* itself is generated. - By default, errors from the :func:`listdir` call are ignored. If optional + By default, errors from the :func:`scandir` call are ignored. If optional argument *onerror* is specified, it should be a function; it will be called with one argument, an :exc:`OSError` instance. It can report the error to continue with the walk, or raise the exception to abort the walk. Note that the filename @@ -3353,6 +3353,31 @@ written in Python, such as a mail server's external command delivery program. subprocesses. +.. function:: posix_spawn(path, argv, env, file_actions=None) + + Wraps the posix_spawn() C library API for use from Python. + + Most users should use :class:`subprocess.run` instead of posix_spawn. + + The *path*, *args*, and *env* arguments are similar to :func:`execve`. + + The *file_actions* argument may be a sequence of tuples describing actions + to take on specific file descriptors in the child process between the C + library implementation's fork and exec steps. The first item in each tuple + must be one of the three type indicator listed below describing the + remaining tuple elements: + + (os.POSIX_SPAWN_OPEN, fd, path, open flags, mode) + (os.POSIX_SPAWN_CLOSE, fd) + (os.POSIX_SPAWN_DUP2, fd, new_fd) + + These tuples correspond to the C library posix_spawn_file_actions_addopen, + posix_spawn_file_actions_addclose, and posix_spawn_file_actions_adddup2 API + calls used to prepare for the posix_spawn call itself. + + .. versionadded:: 3.7 + + .. function:: register_at_fork(*, before=None, after_in_parent=None, \ after_in_child=None) diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst index ec40c0b93abf7be..522bb7e092427bb 100644 --- a/Doc/library/ossaudiodev.rst +++ b/Doc/library/ossaudiodev.rst @@ -14,7 +14,7 @@ the standard audio interface for Linux and recent versions of FreeBSD. .. Things will get more complicated for future Linux versions, since ALSA is in the standard kernel as of 2.5.x. Presumably if you use ALSA, you'll have to make sure its OSS compatibility layer - is active to use ossaudiodev, but you're gonna need it for the vast + is active to use ossaudiodev, but you're going to need it for the vast majority of Linux audio apps anyway. Sounds like things are also complicated for other BSDs. In response @@ -447,4 +447,3 @@ The remaining methods are specific to audio mixing: microphone input:: mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC) - diff --git a/Doc/library/re.rst b/Doc/library/re.rst index 9b175f4e96756b6..d35aaf42f4ab92e 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -67,8 +67,8 @@ string *pq* will match AB. This holds unless *A* or *B* contain low precedence operations; boundary conditions between *A* and *B*; or have numbered group references. Thus, complex expressions can easily be constructed from simpler primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book referenced -above, or almost any textbook about compiler construction. +and implementation of regular expressions, consult the Friedl book [Frie09]_, +or almost any textbook about compiler construction. A brief explanation of the format of regular expressions follows. For further information and a gentler presentation, consult the :ref:`regex-howto`. @@ -345,7 +345,7 @@ The special characters are: This example looks for a word following a hyphen: - >>> m = re.search('(?<=-)\w+', 'spam-egg') + >>> m = re.search(r'(?<=-)\w+', 'spam-egg') >>> m.group(0) 'egg' @@ -489,14 +489,6 @@ three digits in length. Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. -.. seealso:: - - Mastering Regular Expressions - Book on regular expressions by Jeffrey Friedl, published by O'Reilly. The - second edition of the book no longer covers Python at all, but the first - edition covered writing good regular expression patterns in great detail. - - .. _contents-of-module-re: @@ -1582,3 +1574,9 @@ The tokenizer produces the following output:: Token(typ='END', value=';', line=4, column=27) Token(typ='ENDIF', value='ENDIF', line=5, column=4) Token(typ='END', value=';', line=5, column=9) + + +.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly + Media, 2009. The third edition of the book no longer covers Python at all, + but the first edition covered writing good regular expression patterns in + great detail. diff --git a/Doc/library/site.rst b/Doc/library/site.rst index 43daf790b77cb09..0003a7cd8f17316 100644 --- a/Doc/library/site.rst +++ b/Doc/library/site.rst @@ -97,12 +97,12 @@ not mentioned in either path configuration file. After these path manipulations, an attempt is made to import a module named :mod:`sitecustomize`, which can perform arbitrary site-specific customizations. It is typically created by a system administrator in the site-packages -directory. If this import fails with an :exc:`ImportError` exception, it is -silently ignored. If Python is started without output streams available, as +directory. If this import fails with an :exc:`ImportError` or its subclass +exception, and the exception's :attr:`name` attribute equals to ``'sitecustomize'``, +it is silently ignored. If Python is started without output streams available, as with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any exception other -than :exc:`ImportError` causes a silent and perhaps mysterious failure of the -process. +attempted output from :mod:`sitecustomize` is ignored. Any other exception +causes a silent and perhaps mysterious failure of the process. .. index:: module: usercustomize @@ -110,7 +110,9 @@ After this, an attempt is made to import a module named :mod:`usercustomize`, which can perform arbitrary user-specific customizations, if :data:`ENABLE_USER_SITE` is true. This file is intended to be created in the user site-packages directory (see below), which is part of ``sys.path`` unless -disabled by :option:`-s`. An :exc:`ImportError` will be silently ignored. +disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or +its subclass exception, and the exception's :attr:`name` attribute equals to +``'usercustomize'``, it is silently ignored. Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are empty, and the path manipulations are skipped; however the import of @@ -220,7 +222,7 @@ Module contents The :mod:`site` module also provides a way to get the user directories from the command line: -.. code-block:: sh +.. code-block:: shell-session $ python3 -m site --user-site /home/user/.local/lib/python3.3/site-packages diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst index 7f0d4ede7ccf4ca..4ce966981ebae1a 100644 --- a/Doc/library/socket.rst +++ b/Doc/library/socket.rst @@ -77,6 +77,11 @@ created. Socket addresses are represented as follows: backward compatibility. Note, however, omission of *scopeid* can cause problems in manipulating scoped IPv6 addresses. + .. versionchanged:: 3.7 + For multicast addresses (with *scopeid* meaningful) *address* may not contain + ``%scope`` (or ``zone id``) part. This information is superfluous and may + be safely omitted (recommended). + - :const:`AF_NETLINK` sockets are represented as pairs ``(pid, groups)``. - Linux-only support for TIPC is available using the :const:`AF_TIPC` @@ -315,9 +320,16 @@ Constants ``SO_DOMAIN``, ``SO_PROTOCOL``, ``SO_PEERSEC``, ``SO_PASSSEC``, ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. + .. versionchanged:: 3.6.5 + On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows + supports. + .. versionchanged:: 3.7 ``TCP_NOTSENT_LOWAT`` was added. + On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows + supports. + .. data:: AF_CAN PF_CAN SOL_CAN_* @@ -635,6 +647,10 @@ The :mod:`socket` module also offers various network-related services: .. versionchanged:: 3.2 parameters can now be passed using keyword arguments. + .. versionchanged:: 3.7 + for IPv6 multicast addresses, string representing an address will not + contain ``%scope`` part. + .. function:: getfqdn([name]) Return a fully qualified domain name for *name*. If *name* is omitted or empty, @@ -693,6 +709,8 @@ The :mod:`socket` module also offers various network-related services: or numeric address representation in *host*. Similarly, *port* can contain a string port name or a numeric port number. + For IPv6 addresses, ``%scope`` is appended to the host part if *sockaddr* + contains meaningful *scopeid*. Usually this happens for multicast addresses. .. function:: getprotobyname(protocolname) @@ -1193,6 +1211,10 @@ to sockets. an exception, the method now retries the system call instead of raising an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). + .. versionchanged:: 3.7 + For multicast IPv6 address, first item of *address* does not contain + ``%scope`` part anymore. In order to get full IPv6 address use + :func:`getnameinfo`. .. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst index e7676a9f3a50821..d7eaea638f82cbe 100644 --- a/Doc/library/sqlite3.rst +++ b/Doc/library/sqlite3.rst @@ -532,6 +532,56 @@ Connection Objects f.write('%s\n' % line) + .. method:: backup(target, *, pages=0, progress=None, name="main", sleep=0.250) + + This method makes a backup of a SQLite database even while it's being accessed + by other clients, or concurrently by the same connection. The copy will be + written into the mandatory argument *target*, that must be another + :class:`Connection` instance. + + By default, or when *pages* is either ``0`` or a negative integer, the entire + database is copied in a single step; otherwise the method performs a loop + copying up to *pages* pages at a time. + + If *progress* is specified, it must either be ``None`` or a callable object that + will be executed at each iteration with three integer arguments, respectively + the *status* of the last iteration, the *remaining* number of pages still to be + copied and the *total* number of pages. + + The *name* argument specifies the database name that will be copied: it must be + a string containing either ``"main"``, the default, to indicate the main + database, ``"temp"`` to indicate the temporary database or the name specified + after the ``AS`` keyword in an ``ATTACH DATABASE`` statement for an attached + database. + + The *sleep* argument specifies the number of seconds to sleep by between + successive attempts to backup remaining pages, can be specified either as an + integer or a floating point value. + + Example 1, copy an existing database into another:: + + import sqlite3 + + def progress(status, remaining, total): + print(f'Copied {total-remaining} of {total} pages...') + + con = sqlite3.connect('existing_db.db') + with sqlite3.connect('backup.db') as bck: + con.backup(bck, pages=1, progress=progress) + + Example 2, copy an existing database into a transient copy:: + + import sqlite3 + + source = sqlite3.connect('existing_db.db') + dest = sqlite3.connect(':memory:') + source.backup(dest) + + Availability: SQLite 3.6.11 or higher + + .. versionadded:: 3.7 + + .. _sqlite3-cursor-objects: Cursor Objects diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index aa1075d4b02b09a..8082a383d5ab438 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -59,6 +59,125 @@ by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. Functions, Constants, and Exceptions ------------------------------------ + +Socket creation +^^^^^^^^^^^^^^^ + +Since Python 3.2 and 2.7.9, it is recommended to use the +:meth:`SSLContext.wrap_socket` of an :class:`SSLContext` instance to wrap +sockets as :class:`SSLSocket` objects. The helper functions +:func:`create_default_context` returns a new context with secure default +settings. The old :func:`wrap_socket` function is deprecated since it is +both inefficient and has no support for server name indication (SNI) and +hostname matching. + +Client socket example with default context and IPv4/IPv6 dual stack:: + + import socket + import ssl + + hostname = 'www.python.org' + context = ssl.create_default_context() + + with socket.create_connection((hostname, 443)) as sock: + with context.wrap_socket(sock, server_hostname=hostname) as ssock: + print(ssock.version()) + + +Client socket example with custom context and IPv4:: + + hostname = 'www.python.org' + # PROTOCOL_TLS_CLIENT requires valid cert chain and hostname + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.load_verify_locations('path/to/cabundle.pem') + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + with context.wrap_socket(sock, server_hostname=hostname) as ssock: + print(ssock.version()) + + +Server socket example listening on localhost IPv4:: + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.load_cert_chain('/path/to/certchain.pem', '/path/to/private.key') + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: + sock.bind(('127.0.0.1', 8443)) + sock.listen(5) + with context.wrap_socket(sock, server_side=True) as ssock: + conn, addr = ssock.accept() + ... + + +Context creation +^^^^^^^^^^^^^^^^ + +A convenience function helps create :class:`SSLContext` objects for common +purposes. + +.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) + + Return a new :class:`SSLContext` object with default settings for + the given *purpose*. The settings are chosen by the :mod:`ssl` module, + and usually represent a higher security level than when calling the + :class:`SSLContext` constructor directly. + + *cafile*, *capath*, *cadata* represent optional CA certificates to + trust for certificate verification, as in + :meth:`SSLContext.load_verify_locations`. If all three are + :const:`None`, this function can choose to trust the system's default + CA certificates instead. + + The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and + :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and + without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` + as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` + and either loads CA certificates (when at least one of *cafile*, *capath* or + *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load + default CA certificates. + + .. note:: + The protocol, options, cipher and other settings may change to more + restrictive values anytime without prior deprecation. The values + represent a fair balance between compatibility and security. + + If your application needs specific settings, you should create a + :class:`SSLContext` and apply the settings yourself. + + .. note:: + If you find that when certain older clients or servers attempt to connect + with a :class:`SSLContext` created by this function that they get an error + stating "Protocol or cipher suite mismatch", it may be that they only + support SSL3.0 which this function excludes using the + :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken + `_. If you still wish to continue to + use this function but still allow SSL 3.0 connections you can re-enable + them using:: + + ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) + ctx.options &= ~ssl.OP_NO_SSLv3 + + .. versionadded:: 3.4 + + .. versionchanged:: 3.4.4 + + RC4 was dropped from the default cipher string. + + .. versionchanged:: 3.6 + + ChaCha20/Poly1305 was added to the default cipher string. + + 3DES was dropped from the default cipher string. + + .. versionchanged:: 3.7 + + TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, + and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string. + + +Exceptions +^^^^^^^^^^ + .. exception:: SSLError Raised to signal an error from the underlying SSL implementation @@ -152,173 +271,6 @@ Functions, Constants, and Exceptions The exception is now an alias for :exc:`SSLCertVerificationError`. -Socket creation -^^^^^^^^^^^^^^^ - -The following function allows for standalone socket creation. Starting from -Python 3.2, it can be more flexible to use :meth:`SSLContext.wrap_socket` -instead. - -.. function:: wrap_socket(sock, keyfile=None, certfile=None, server_side=False, cert_reqs=CERT_NONE, ssl_version={see docs}, ca_certs=None, do_handshake_on_connect=True, suppress_ragged_eofs=True, ciphers=None) - - Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance - of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps - the underlying socket in an SSL context. ``sock`` must be a - :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported. - - For client-side sockets, the context construction is lazy; if the - underlying socket isn't connected yet, the context construction will be - performed after :meth:`connect` is called on the socket. For - server-side sockets, if the socket has no remote peer, it is assumed - to be a listening socket, and the server-side SSL wrapping is - automatically performed on client connections accepted via the - :meth:`accept` method. :func:`wrap_socket` may raise :exc:`SSLError`. - - The ``keyfile`` and ``certfile`` parameters specify optional files which - contain a certificate to be used to identify the local side of the - connection. See the discussion of :ref:`ssl-certificates` for more - information on how the certificate is stored in the ``certfile``. - - The parameter ``server_side`` is a boolean which identifies whether - server-side or client-side behavior is desired from this socket. - - The parameter ``cert_reqs`` specifies whether a certificate is required from - the other side of the connection, and whether it will be validated if - provided. It must be one of the three values :const:`CERT_NONE` - (certificates ignored), :const:`CERT_OPTIONAL` (not required, but validated - if provided), or :const:`CERT_REQUIRED` (required and validated). If the - value of this parameter is not :const:`CERT_NONE`, then the ``ca_certs`` - parameter must point to a file of CA certificates. - - The ``ca_certs`` file contains a set of concatenated "certification - authority" certificates, which are used to validate certificates passed from - the other end of the connection. See the discussion of - :ref:`ssl-certificates` for more information about how to arrange the - certificates in this file. - - The parameter ``ssl_version`` specifies which version of the SSL protocol to - use. Typically, the server chooses a particular protocol version, and the - client must adapt to the server's choice. Most of the versions are not - interoperable with the other versions. If not specified, the default is - :data:`PROTOCOL_TLS`; it provides the most compatibility with other - versions. - - Here's a table showing which versions in a client (down the side) can connect - to which versions in a server (along the top): - - .. table:: - - ======================== ============ ============ ============= ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** - ------------------------ ------------ ------------ ------------- --------- ----------- ----------- - *SSLv2* yes no no [1]_ no no no - *SSLv3* no yes no [2]_ no no no - *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes - *TLSv1* no no yes yes no no - *TLSv1.1* no no yes no yes no - *TLSv1.2* no no yes no no yes - ======================== ============ ============ ============= ========= =========== =========== - - .. rubric:: Footnotes - .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. - .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default. - .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in - OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just - TLS 1.3. - - .. note:: - - Which connections succeed will vary depending on the version of - OpenSSL. For example, before OpenSSL 1.0.0, an SSLv23 client - would always attempt SSLv2 connections. - - The *ciphers* parameter sets the available ciphers for this SSL object. - It should be a string in the `OpenSSL cipher list format - `_. - - The parameter ``do_handshake_on_connect`` specifies whether to do the SSL - handshake automatically after doing a :meth:`socket.connect`, or whether the - application program will call it explicitly, by invoking the - :meth:`SSLSocket.do_handshake` method. Calling - :meth:`SSLSocket.do_handshake` explicitly gives the program control over the - blocking behavior of the socket I/O involved in the handshake. - - The parameter ``suppress_ragged_eofs`` specifies how the - :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end - of the connection. If specified as :const:`True` (the default), it returns a - normal EOF (an empty bytes object) in response to unexpected EOF errors - raised from the underlying socket; if :const:`False`, it will raise the - exceptions back to the caller. - - .. versionchanged:: 3.2 - New optional argument *ciphers*. - -Context creation -^^^^^^^^^^^^^^^^ - -A convenience function helps create :class:`SSLContext` objects for common -purposes. - -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) - - Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, - and usually represent a higher security level than when calling the - :class:`SSLContext` constructor directly. - - *cafile*, *capath*, *cadata* represent optional CA certificates to - trust for certificate verification, as in - :meth:`SSLContext.load_verify_locations`. If all three are - :const:`None`, this function can choose to trust the system's default - CA certificates instead. - - The settings are: :data:`PROTOCOL_TLS`, :data:`OP_NO_SSLv2`, and - :data:`OP_NO_SSLv3` with high encryption cipher suites without RC4 and - without unauthenticated cipher suites. Passing :data:`~Purpose.SERVER_AUTH` - as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` - and either loads CA certificates (when at least one of *cafile*, *capath* or - *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load - default CA certificates. - - .. note:: - The protocol, options, cipher and other settings may change to more - restrictive values anytime without prior deprecation. The values - represent a fair balance between compatibility and security. - - If your application needs specific settings, you should create a - :class:`SSLContext` and apply the settings yourself. - - .. note:: - If you find that when certain older clients or servers attempt to connect - with a :class:`SSLContext` created by this function that they get an error - stating "Protocol or cipher suite mismatch", it may be that they only - support SSL3.0 which this function excludes using the - :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken - `_. If you still wish to continue to - use this function but still allow SSL 3.0 connections you can re-enable - them using:: - - ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) - ctx.options &= ~ssl.OP_NO_SSLv3 - - .. versionadded:: 3.4 - - .. versionchanged:: 3.4.4 - - RC4 was dropped from the default cipher string. - - .. versionchanged:: 3.6 - - ChaCha20/Poly1305 was added to the default cipher string. - - 3DES was dropped from the default cipher string. - - .. versionchanged:: 3.7 - - TLS 1.3 cipher suites TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, - and TLS_CHACHA20_POLY1305_SHA256 were added to the default cipher string. - - Random generation ^^^^^^^^^^^^^^^^^ @@ -474,9 +426,10 @@ Certificate handling PEM-encoded string. If ``ssl_version`` is specified, uses that version of the SSL protocol to attempt to connect to the server. If ``ca_certs`` is specified, it should be a file containing a list of root certificates, the - same format as used for the same parameter in :func:`wrap_socket`. The call - will attempt to validate the server certificate against that set of root - certificates, and will fail if the validation attempt fails. + same format as used for the same parameter in + :meth:`SSLContext.wrap_socket`. The call will attempt to validate the + server certificate against that set of root certificates, and will fail + if the validation attempt fails. .. versionchanged:: 3.3 This function is now IPv6-compatible. @@ -552,6 +505,33 @@ Certificate handling .. versionadded:: 3.4 +.. function:: wrap_socket(sock, keyfile=None, certfile=None, \ + server_side=False, cert_reqs=CERT_NONE, ssl_version=PROTOCOL_TLS, \ + ca_certs=None, do_handshake_on_connect=True, \ + suppress_ragged_eofs=True, ciphers=None) + + Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance + of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps + the underlying socket in an SSL context. ``sock`` must be a + :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported. + + Internally, function creates a :class:`SSLContext` with protocol + *ssl_version* and :attr:`SSLContext.options` set to *cert_reqs*. If + parameters *keyfile*, *certfile*, *ca_certs* or *ciphers* are set, then + the values are passed to :meth:`SSLContext.load_cert_chain`, + :meth:`SSLContext.load_verify_locations`, and + :meth:`SSLContext.set_ciphers`. + + The arguments *server_side*, *do_handshake_on_connect*, and + *suppress_ragged_eofs* have the same meaning as + :meth:`SSLContext.wrap_socket`. + + .. deprecated:: 3.7 + + Since Python 3.2 and 2.7.9, it is recommended to use the + :meth:`SSLContext.wrap_socket` instead of :func:`wrap_socket`. The + top-level function is limited and creates an insecure client socket + without server name indication or hostname matching. Constants ^^^^^^^^^ @@ -782,6 +762,11 @@ Constants .. versionadded:: 3.2 + .. deprecated:: 3.7 + The option is deprecated since OpenSSL 1.1.0, use the new + :attr:`SSLContext.minimum_version` and + :attr:`SSLContext.maximum_version` instead. + .. data:: OP_NO_TLSv1_1 Prevents a TLSv1.1 connection. This option is only applicable in conjunction @@ -790,6 +775,9 @@ Constants .. versionadded:: 3.4 + .. deprecated:: 3.7 + The option is deprecated since OpenSSL 1.1.0. + .. data:: OP_NO_TLSv1_2 Prevents a TLSv1.2 connection. This option is only applicable in conjunction @@ -798,6 +786,9 @@ Constants .. versionadded:: 3.4 + .. deprecated:: 3.7 + The option is deprecated since OpenSSL 1.1.0. + .. data:: OP_NO_TLSv1_3 Prevents a TLSv1.3 connection. This option is only applicable in conjunction @@ -808,6 +799,10 @@ Constants .. versionadded:: 3.7 + .. deprecated:: 3.7 + The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15, + 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2. + .. data:: OP_CIPHER_SERVER_PREFERENCE Use the server's cipher ordering preference, rather than the client's. @@ -831,6 +826,15 @@ Constants .. versionadded:: 3.3 +.. data:: OP_ENABLE_MIDDLEBOX_COMPAT + + Send dummy Change Cipher Spec (CCS) messages in TLS 1.3 handshake to make + a TLS 1.3 connection look more like a TLS 1.2 connection. + + This option is only available with OpenSSL 1.1.1 and later. + + .. versionadded:: 3.8 + .. data:: OP_NO_COMPRESSION Disable compression on the SSL channel. This is useful if the application @@ -867,7 +871,7 @@ Constants .. data:: HAS_ECDH - Whether the OpenSSL library has built-in support for Elliptic Curve-based + Whether the OpenSSL library has built-in support for the Elliptic Curve-based Diffie-Hellman key exchange. This should be true unless the feature was explicitly disabled by the distributor. @@ -882,7 +886,7 @@ Constants .. data:: HAS_NPN - Whether the OpenSSL library has built-in support for *Next Protocol + Whether the OpenSSL library has built-in support for the *Next Protocol Negotiation* as described in the `Application Layer Protocol Negotiation `_. When true, you can use the :meth:`SSLContext.set_npn_protocols` method to advertise @@ -890,6 +894,36 @@ Constants .. versionadded:: 3.3 +.. data:: HAS_SSLv2 + + Whether the OpenSSL library has built-in support for the SSL 2.0 protocol. + + .. versionadded:: 3.7 + +.. data:: HAS_SSLv3 + + Whether the OpenSSL library has built-in support for the SSL 3.0 protocol. + + .. versionadded:: 3.7 + +.. data:: HAS_TLSv1 + + Whether the OpenSSL library has built-in support for the TLS 1.0 protocol. + + .. versionadded:: 3.7 + +.. data:: HAS_TLSv1_1 + + Whether the OpenSSL library has built-in support for the TLS 1.1 protocol. + + .. versionadded:: 3.7 + +.. data:: HAS_TLSv1_2 + + Whether the OpenSSL library has built-in support for the TLS 1.2 protocol. + + .. versionadded:: 3.7 + .. data:: HAS_TLSv1_3 Whether the OpenSSL library has built-in support for the TLS 1.3 protocol. @@ -976,6 +1010,27 @@ Constants .. versionadded:: 3.6 +.. class:: TLSVersion + + :class:`enum.IntEnum` collection of SSL and TLS versions for + :attr:`SSLContext.maximum_version` and :attr:`SSLContext.minimum_version`. + + .. versionadded:: 3.7 + +.. attribute:: TLSVersion.MINIMUM_SUPPORTED +.. attribute:: TLSVersion.MAXIMUM_SUPPORTED + + The minimum or maximum supported SSL or TLS version. These are magic + constants. Their values don't reflect the lowest and highest available + TLS/SSL versions. + +.. attribute:: TLSVersion.SSLv3 +.. attribute:: TLSVersion.TLSv1 +.. attribute:: TLSVersion.TLSv1_1 +.. attribute:: TLSVersion.TLSv1_2 +.. attribute:: TLSVersion.TLSv1_3 + + SSL 3.0 to TLS 1.3. SSL Sockets ----------- @@ -1009,7 +1064,7 @@ SSL Sockets the specification of normal, OS-level sockets. See especially the :ref:`notes on non-blocking sockets `. - Usually, :class:`SSLSocket` are not created directly, but using the + Instances of :class:`SSLSocket` must be created using the :meth:`SSLContext.wrap_socket` method. .. versionchanged:: 3.5 @@ -1024,6 +1079,11 @@ SSL Sockets It is deprecated to create a :class:`SSLSocket` instance directly, use :meth:`SSLContext.wrap_socket` to wrap a socket. + .. versionchanged:: 3.7 + :class:`SSLSocket` instances must to created with + :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible + to create instances directly. This was never documented or officially + supported. SSL sockets also have the following additional methods and attributes: @@ -1248,7 +1308,7 @@ SSL sockets also have the following additional methods and attributes: .. attribute:: SSLSocket.context The :class:`SSLContext` object this SSL socket is tied to. If the SSL - socket was created using the top-level :func:`wrap_socket` function + socket was created using the deprecated :func:`wrap_socket` function (rather than :meth:`SSLContext.wrap_socket`), this is a custom context object created for this SSL socket. @@ -1268,6 +1328,12 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 3.2 + .. versionchanged:: 3.7 + The attribute is now always ASCII text. When ``server_hostname`` is + an internationalized domain name (IDN), this attribute now stores the + A-label form (``"xn--pythn-mua.org"``), rather than the U-label form + (``"pythön.org"``). + .. attribute:: SSLSocket.session The :class:`SSLSession` for this SSL connection. The session is available @@ -1295,9 +1361,36 @@ to speed up repeated connections from the same clients. .. class:: SSLContext(protocol=PROTOCOL_TLS) Create a new SSL context. You may pass *protocol* which must be one - of the ``PROTOCOL_*`` constants defined in this module. - :data:`PROTOCOL_TLS` is currently recommended for maximum - interoperability and default value. + of the ``PROTOCOL_*`` constants defined in this module. The parameter + specifies which version of the SSL protocol to use. Typically, the + server chooses a particular protocol version, and the client must adapt + to the server's choice. Most of the versions are not interoperable + with the other versions. If not specified, the default is + :data:`PROTOCOL_TLS`; it provides the most compatibility with other + versions. + + Here's a table showing which versions in a client (down the side) can connect + to which versions in a server (along the top): + + .. table:: + + ======================== ============ ============ ============= ========= =========== =========== + *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** + ------------------------ ------------ ------------ ------------- --------- ----------- ----------- + *SSLv2* yes no no [1]_ no no no + *SSLv3* no yes no [2]_ no no no + *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes + *TLSv1* no no yes yes no no + *TLSv1.1* no no yes no yes no + *TLSv1.2* no no yes no no yes + ======================== ============ ============ ============= ========= =========== =========== + + .. rubric:: Footnotes + .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. + .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default. + .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in + OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just + TLS 1.3. .. seealso:: :func:`create_default_context` lets the :mod:`ssl` module choose @@ -1532,23 +1625,24 @@ to speed up repeated connections from the same clients. .. versionadded:: 3.3 -.. method:: SSLContext.set_servername_callback(server_name_callback) +.. attribute:: SSLContext.sni_callback Register a callback function that will be called after the TLS Client Hello handshake message has been received by the SSL/TLS server when the TLS client specifies a server name indication. The server name indication mechanism is specified in :rfc:`6066` section 3 - Server Name Indication. - Only one callback can be set per ``SSLContext``. If *server_name_callback* - is ``None`` then the callback is disabled. Calling this function a + Only one callback can be set per ``SSLContext``. If *sni_callback* + is set to ``None`` then the callback is disabled. Calling this function a subsequent time will disable the previously registered callback. - The callback function, *server_name_callback*, will be called with three + The callback function will be called with three arguments; the first being the :class:`ssl.SSLSocket`, the second is a string that represents the server name that the client is intending to communicate (or :const:`None` if the TLS Client Hello does not contain a server name) and the third argument is the original :class:`SSLContext`. The server name - argument is the IDNA decoded server name. + argument is text. For internationalized domain name, the server + name is an IDN A-label (``"xn--pythn-mua.org"``). A typical use of this callback is to change the :class:`ssl.SSLSocket`'s :attr:`SSLSocket.context` attribute to a new object of type @@ -1563,28 +1657,38 @@ to speed up repeated connections from the same clients. the TLS connection has progressed beyond the TLS Client Hello and therefore will not contain return meaningful values nor can they be called safely. - The *server_name_callback* function must return ``None`` to allow the + The *sni_callback* function must return ``None`` to allow the TLS negotiation to continue. If a TLS failure is required, a constant :const:`ALERT_DESCRIPTION_* ` can be returned. Other return values will result in a TLS fatal error with :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`. - If there is an IDNA decoding error on the server name, the TLS connection - will terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS - alert message to the client. - - If an exception is raised from the *server_name_callback* function the TLS + If an exception is raised from the *sni_callback* function the TLS connection will terminate with a fatal TLS alert message :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`. This method will raise :exc:`NotImplementedError` if the OpenSSL library had OPENSSL_NO_TLSEXT defined when it was built. + .. versionadded:: 3.7 + +.. attribute:: SSLContext.set_servername_callback(server_name_callback) + + This is a legacy API retained for backwards compatibility. When possible, + you should use :attr:`sni_callback` instead. The given *server_name_callback* + is similar to *sni_callback*, except that when the server hostname is an + IDN-encoded internationalized domain name, the *server_name_callback* + receives a decoded U-label (``"pythön.org"``). + + If there is an decoding error on the server name, the TLS connection will + terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS + alert message to the client. + .. versionadded:: 3.4 .. method:: SSLContext.load_dh_params(dhfile) - Load the key generation parameters for Diffie-Helman (DH) key exchange. + Load the key generation parameters for Diffie-Hellman (DH) key exchange. Using DH key exchange improves forward secrecy at the expense of computational resources (both on the server and on the client). The *dhfile* parameter should be the path to a file containing DH @@ -1619,14 +1723,21 @@ to speed up repeated connections from the same clients. server_hostname=None, session=None) Wrap an existing Python socket *sock* and return an instance of - :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`). - *sock* must be a :data:`~socket.SOCK_STREAM` socket; other socket - types are unsupported. + :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`). The + returned SSL socket is tied to the context, its settings and certificates. + *sock* must be a :data:`~socket.SOCK_STREAM` socket; other + socket types are unsupported. + + The parameter ``server_side`` is a boolean which identifies whether + server-side or client-side behavior is desired from this socket. - The returned SSL socket is tied to the context, its settings and - certificates. The parameters *server_side*, *do_handshake_on_connect* - and *suppress_ragged_eofs* have the same meaning as in the top-level - :func:`wrap_socket` function. + For client-side sockets, the context construction is lazy; if the + underlying socket isn't connected yet, the context construction will be + performed after :meth:`connect` is called on the socket. For + server-side sockets, if the socket has no remote peer, it is assumed + to be a listening socket, and the server-side SSL wrapping is + automatically performed on client connections accepted via the + :meth:`accept` method. The method may raise :exc:`SSLError`. On client connections, the optional parameter *server_hostname* specifies the hostname of the service which we are connecting to. This allows a @@ -1634,6 +1745,20 @@ to speed up repeated connections from the same clients. quite similarly to HTTP virtual hosts. Specifying *server_hostname* will raise a :exc:`ValueError` if *server_side* is true. + The parameter ``do_handshake_on_connect`` specifies whether to do the SSL + handshake automatically after doing a :meth:`socket.connect`, or whether the + application program will call it explicitly, by invoking the + :meth:`SSLSocket.do_handshake` method. Calling + :meth:`SSLSocket.do_handshake` explicitly gives the program control over the + blocking behavior of the socket I/O involved in the handshake. + + The parameter ``suppress_ragged_eofs`` specifies how the + :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end + of the connection. If specified as :const:`True` (the default), it returns a + normal EOF (an empty bytes object) in response to unexpected EOF errors + raised from the underlying socket; if :const:`False`, it will raise the + exceptions back to the caller. + *session*, see :attr:`~SSLSocket.session`. .. versionchanged:: 3.5 @@ -1707,7 +1832,7 @@ to speed up repeated connections from the same clients. import socket, ssl - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) + context = ssl.SSLContext() context.verify_mode = ssl.CERT_REQUIRED context.check_hostname = True context.load_default_certs() @@ -1729,6 +1854,37 @@ to speed up repeated connections from the same clients. This features requires OpenSSL 0.9.8f or newer. +.. attribute:: SSLContext.maximum_version + + A :class:`TLSVersion` enum member representing the highest supported + TLS version. The value defaults to :attr:`TLSVersion.MAXIMUM_SUPPORTED`. + The attribute is read-only for protocols other than :attr:`PROTOCOL_TLS`, + :attr:`PROTOCOL_TLS_CLIENT`, and :attr:`PROTOCOL_TLS_SERVER`. + + The attributes :attr:`~SSLContext.maximum_version`, + :attr:`~SSLContext.minimum_version` and + :attr:`SSLContext.options` all affect the supported SSL + and TLS versions of the context. The implementation does not prevent + invalid combination. For example a context with + :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and + :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` + will not be able to establish a TLS 1.2 connection. + + .. note:: + + This attribute is not available unless the ssl module is compiled + with OpenSSL 1.1.0g or newer. + +.. attribute:: SSLContext.minimum_version + + Like :attr:`SSLContext.maximum_version` except it is the lowest + supported version or :attr:`TLSVersion.MINIMUM_SUPPORTED`. + + .. note:: + + This attribute is not available unless the ssl module is compiled + with OpenSSL 1.1.0g or newer. + .. attribute:: SSLContext.options An integer representing the set of SSL options enabled on this context. @@ -1952,7 +2108,7 @@ If you prefer to tune security settings yourself, you might create a context from scratch (but beware that you might not get the settings right):: - >>> context = ssl.SSLContext(ssl.PROTOCOL_TLS) + >>> context = ssl.SSLContext() >>> context.verify_mode = ssl.CERT_REQUIRED >>> context.check_hostname = True >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt") @@ -2195,11 +2351,12 @@ provided. but does not provide any network IO itself. IO needs to be performed through separate "BIO" objects which are OpenSSL's IO abstraction layer. - An :class:`SSLObject` instance can be created using the - :meth:`~SSLContext.wrap_bio` method. This method will create the - :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming* - BIO is used to pass data from Python to the SSL protocol instance, while the - *outgoing* BIO is used to pass data the other way around. + This class has no public constructor. An :class:`SSLObject` instance + must be created using the :meth:`~SSLContext.wrap_bio` method. This + method will create the :class:`SSLObject` instance and bind it to a + pair of BIOs. The *incoming* BIO is used to pass data from Python to the + SSL protocol instance, while the *outgoing* BIO is used to pass data the + other way around. The following methods are available: @@ -2251,6 +2408,12 @@ provided. :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created via an :class:`SSLContext`. + .. versionchanged:: 3.7 + :class:`SSLObject` instances must to created with + :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to + create instances directly. This was never documented or officially + supported. + An SSLObject communicates with the outside world using memory buffers. The class :class:`MemoryBIO` provides a memory buffer that can be used for this purpose. It wraps an OpenSSL memory BIO (Basic IO) object: @@ -2417,13 +2580,30 @@ successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or :func:`~ssl.RAND_pseudo_bytes` is sufficient. +.. ssl-libressl: + +LibreSSL support +---------------- + +LibreSSL is a fork of OpenSSL 1.0.1. The ssl module has limited support for +LibreSSL. Some features are not available when the ssl module is compiled +with LibreSSL. + +* LibreSSL >= 2.6.1 no longer supports NPN. The methods + :meth:`SSLContext.set_npn_protocols` and + :meth:`SSLSocket.selected_npn_protocol` are not available. +* :meth:`SSLContext.set_default_verify_paths` ignores the env vars + :envvar:`SSL_CERT_FILE` and :envvar:`SSL_CERT_PATH` although + :func:`get_default_verify_paths` still reports them. + + .. seealso:: Class :class:`socket.socket` Documentation of underlying :mod:`socket` class `SSL/TLS Strong Encryption: An Introduction `_ - Intro from the Apache webserver documentation + Intro from the Apache HTTP Server documentation `RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management `_ Steve Kent diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ad7f578e0860694..a213189a3407499 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -4256,17 +4256,17 @@ support membership tests: Return an iterator over the keys, values or items (represented as tuples of ``(key, value)``) in the dictionary. - Keys and values are iterated over in an arbitrary order which is non-random, - varies across Python implementations, and depends on the dictionary's history - of insertions and deletions. If keys, values and items views are iterated - over with no intervening modifications to the dictionary, the order of items - will directly correspond. This allows the creation of ``(value, key)`` pairs + Keys and values are iterated over in insertion order. + This allows the creation of ``(value, key)`` pairs using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``. Iterating views while adding or deleting entries in the dictionary may raise a :exc:`RuntimeError` or fail to iterate over all entries. + .. versionchanged:: 3.7 + Dict order is guaranteed to be insertion order. + .. describe:: x in dictview Return ``True`` if *x* is in the underlying dictionary's keys, values or @@ -4293,9 +4293,9 @@ An example of dictionary view usage:: >>> print(n) 504 - >>> # keys and values are iterated over in the same order + >>> # keys and values are iterated over in the same order (insertion order) >>> list(keys) - ['eggs', 'bacon', 'sausage', 'spam'] + ['eggs', 'sausage', 'bacon', 'spam'] >>> list(values) [2, 1, 1, 500] @@ -4303,7 +4303,7 @@ An example of dictionary view usage:: >>> del dishes['eggs'] >>> del dishes['sausage'] >>> list(keys) - ['spam', 'bacon'] + ['bacon', 'spam'] >>> # set operations >>> keys & {'eggs', 'bacon', 'salad'} diff --git a/Doc/library/string.rst b/Doc/library/string.rst index fc3d94bfbfca000..ee8ea857da4b96e 100644 --- a/Doc/library/string.rst +++ b/Doc/library/string.rst @@ -202,9 +202,9 @@ The grammar for a replacement field is as follows: .. productionlist:: sf replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `integer`] + arg_name: [`identifier` | `digit`+] attribute_name: `identifier` - element_index: `integer` | `index_string` + element_index: `digit`+ | `index_string` index_string: + conversion: "r" | "s" | "a" format_spec: @@ -304,9 +304,9 @@ The general form of a *standard format specifier* is: fill: align: "<" | ">" | "=" | "^" sign: "+" | "-" | " " - width: `integer` + width: `digit`+ grouping_option: "_" | "," - precision: `integer` + precision: `digit`+ type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" If a valid *align* value is specified, it can be preceded by a *fill* diff --git a/Doc/library/subprocess.rst b/Doc/library/subprocess.rst index 27d4288f67b33dc..fbf2c3d9fac9a26 100644 --- a/Doc/library/subprocess.rst +++ b/Doc/library/subprocess.rst @@ -39,7 +39,7 @@ compatibility with older versions, see the :ref:`call-function-trio` section. .. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ shell=False, cwd=None, timeout=None, check=False, \ - encoding=None, errors=None) + encoding=None, errors=None, text=None) Run the command described by *args*. Wait for command to complete, then return a :class:`CompletedProcess` instance. @@ -267,7 +267,8 @@ default values. The arguments that are most commonly needed are: .. index:: single: universal newlines; subprocess module - If *encoding* or *errors* are specified, or *universal_newlines* is true, + If *encoding* or *errors* are specified, or *text* (also known as + *universal_newlines*) is true, the file objects *stdin*, *stdout* and *stderr* will be opened in text mode using the *encoding* and *errors* specified in the call or the defaults for :class:`io.TextIOWrapper`. @@ -284,6 +285,9 @@ default values. The arguments that are most commonly needed are: .. versionadded:: 3.6 Added *encoding* and *errors* parameters. + .. versionadded:: 3.7 + Added the *text* parameter as an alias for *universal_newlines*. + .. note:: The newlines attribute of the file objects :attr:`Popen.stdin`, @@ -328,19 +332,19 @@ functions. cwd=None, env=None, universal_newlines=False, \ startupinfo=None, creationflags=0, restore_signals=True, \ start_new_session=False, pass_fds=(), *, \ - encoding=None, errors=None) + encoding=None, errors=None, text=None) Execute a child program in a new process. On POSIX, the class uses :meth:`os.execvp`-like behavior to execute the child program. On Windows, the class uses the Windows ``CreateProcess()`` function. The arguments to :class:`Popen` are as follows. - *args* should be a sequence of program arguments or else a single string or - :term:`path-like object`. By default, the program to execute is the first - item in *args* if *args* is a sequence. If *args* is a string, the - interpretation is platform-dependent and described below. See the *shell* - and *executable* arguments for additional differences from the default - behavior. Unless otherwise stated, it is recommended to pass *args* as a sequence. + *args* should be a sequence of program arguments or else a single string. + By default, the program to execute is the first item in *args* if *args* is + a sequence. If *args* is a string, the interpretation is + platform-dependent and described below. See the *shell* and *executable* + arguments for additional differences from the default behavior. Unless + otherwise stated, it is recommended to pass *args* as a sequence. On POSIX, if *args* is a string, the string is interpreted as the name or path of the program to execute. However, this can only be done if not @@ -455,7 +459,10 @@ functions. common use of *preexec_fn* to call os.setsid() in the child. If *close_fds* is true, all file descriptors except :const:`0`, :const:`1` and - :const:`2` will be closed before the child process is executed. + :const:`2` will be closed before the child process is executed. Otherwise + when *close_fds* is false, file descriptors obey their inheritable flag + as described in :ref:`fd_inheritance`. + On Windows, if *close_fds* is true then no handles will be inherited by the child process unless explicitly passed in the ``handle_list`` element of :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. @@ -511,15 +518,18 @@ functions. .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly - If *encoding* or *errors* are specified, the file objects *stdin*, *stdout* - and *stderr* are opened in text mode with the specified encoding and - *errors*, as described above in :ref:`frequently-used-arguments`. If - *universal_newlines* is ``True``, they are opened in text mode with default - encoding. Otherwise, they are opened as binary streams. + If *encoding* or *errors* are specified, or *text* is true, the file objects + *stdin*, *stdout* and *stderr* are opened in text mode with the specified + encoding and *errors*, as described above in :ref:`frequently-used-arguments`. + The *universal_newlines* argument is equivalent to *text* and is provided + for backwards compatibility. By default, file objects are opened in binary mode. .. versionadded:: 3.6 *encoding* and *errors* were added. + .. versionadded:: 3.7 + *text* was added as a more readable alias for *universal_newlines*. + If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is passed to the underlying ``CreateProcess`` function. *creationflags*, if given, can be one or more of the following flags: @@ -551,10 +561,6 @@ functions. Popen destructor now emits a :exc:`ResourceWarning` warning if the child process is still running. - .. versionchanged:: 3.7 - *args*, or the first element of *args* if *args* is a sequence, can now - be a :term:`path-like object`. - Exceptions ^^^^^^^^^^ @@ -1087,6 +1093,9 @@ calls these functions. .. versionchanged:: 3.4 Support for the *input* keyword argument was added. + .. versionchanged:: 3.6 + *encoding* and *errors* were added. See :func:`run` for details. + .. _subprocess-replacements: Replacing Older Functions with the :mod:`subprocess` Module diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst index 2450716a1d91205..9cd07158e7f6286 100644 --- a/Doc/library/tarfile.rst +++ b/Doc/library/tarfile.rst @@ -451,7 +451,8 @@ be finalized; only the internally used file object will be closed. See the (directory, fifo, symbolic link, etc.). If given, *arcname* specifies an alternative name for the file in the archive. Directories are added recursively by default. This can be avoided by setting *recursive* to - :const:`False`. If *filter* is given, it + :const:`False`. Recursion adds entries in sorted order. + If *filter* is given, it should be a function that takes a :class:`TarInfo` object argument and returns the changed :class:`TarInfo` object. If it instead returns :const:`None` the :class:`TarInfo` object will be excluded from the @@ -460,6 +461,9 @@ be finalized; only the internally used file object will be closed. See the .. versionchanged:: 3.2 Added the *filter* parameter. + .. versionchanged:: 3.7 + Recursion adds entries in sorted order. + .. method:: TarFile.addfile(tarinfo, fileobj=None) diff --git a/Doc/library/test.rst b/Doc/library/test.rst index 01ba1ec7062c98a..7b0971a83bcb6ea 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -237,12 +237,127 @@ The :mod:`test.support` module defines the following constants: ``True`` if the running interpreter is Jython. +.. data:: is_android + + ``True`` if the system is Android. + + +.. data:: unix_shell + + Path for shell if not on Windows; otherwise ``None``. + + +.. data:: FS_NONASCII + + A non-ASCII character encodable by :func:`os.fsencode`. + + .. data:: TESTFN Set to a name that is safe to use as the name of a temporary file. Any temporary file that is created should be closed and unlinked (removed). +.. data:: TESTFN_UNICODE + + Set to a non-ASCII name for a temporary file. + + +.. data:: TESTFN_ENCODING + + Set to :func:`sys.getfilesystemencoding`. + + +.. data:: TESTFN_UNENCODABLE + + Set to a filename (str type) that should not be able to be encoded by file + system encoding in strict mode. It may be ``None`` if it's not possible to + generate such a filename. + + +.. data:: TESTFN_UNDECODABLE + + Set to a filename (bytes type) that should not be able to be decoded by + file system encoding in strict mode. It may be ``None`` if it's not + possible to generate such a filename. + + +.. data:: TESTFN_NONASCII + + Set to a filename containing the :data:`FS_NONASCII` character. + + +.. data:: IPV6_ENABLED + + Set to ``True`` if IPV6 is enabled on this host, ``False`` otherwise. + + +.. data:: SAVEDCWD + + Set to :func:`os.getcwd`. + + +.. data:: PGO + + Set when tests can be skipped when they are not useful for PGO. + + +.. data:: PIPE_MAX_SIZE + + A constant that is likely larger than the underlying OS pipe buffer size, + to make writes blocking. + + +.. data:: SOCK_MAX_SIZE + + A constant that is likely larger than the underlying OS socket buffer size, + to make writes blocking. + + +.. data:: TEST_SUPPORT_DIR + + Set to the top level directory that contains :mod:`test.support`. + + +.. data:: TEST_HOME_DIR + + Set to the top level directory for the test package. + + +.. data:: TEST_DATA_DIR + + Set to the ``data`` directory within the test package. + + +.. data:: MAX_Py_ssize_t + + Set to :data:`sys.maxsize` for big memory tests. + + +.. data:: max_memuse + + Set by :func:`set_memlimit` as the memory limit for big memory tests. + Limited by :data:`MAX_Py_ssize_t`. + + +.. data:: real_max_memuse + + Set by :func:`set_memlimit` as the memory limit for big memory tests. Not + limited by :data:`MAX_Py_ssize_t`. + + +.. data:: MISSING_C_DOCSTRINGS + + Return ``True`` if running on CPython, not on Windows, and configuration + not set with ``WITH_DOC_STRINGS``. + + +.. data:: HAVE_DOCSTRINGS + + Check for presence of docstrings. + + + The :mod:`test.support` module defines the following functions: .. function:: forget(module_name) @@ -251,6 +366,38 @@ The :mod:`test.support` module defines the following functions: byte-compiled files of the module. +.. function:: unload(name) + + Delete *name* from ``sys.modules``. + + +.. function:: unlink(filename) + + Call :func:`os.unlink` on *filename*. On Windows platforms, this is + wrapped with a wait loop that checks for the existence fo the file. + + +.. function:: rmdir(filename) + + Call :func:`os.rmdir` on *filename*. On Windows platforms, this is + wrapped with a wait loop that checks for the existence of the file. + + +.. function:: rmtree(path) + + Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and + :func:`os.rmdir` to remove a path and its contents. On Windows platforms, + this is wrapped with a wait loop that checks for the existence of the files. + + +.. function:: make_legacy_pyc(source) + + Move a PEP 3147/488 pyc file to its legacy pyc location and return the file + system path to the legacy pyc file. The *source* value is the file system + path to the source file. It does not need to exist, however the PEP + 3147/488 pyc file must exist. + + .. function:: is_resource_enabled(resource) Return ``True`` if *resource* is enabled and available. The list of @@ -258,6 +405,16 @@ The :mod:`test.support` module defines the following functions: tests. +.. function:: python_is_optimized() + + Return ``True`` if Python was not built with ``-O0`` or ``-Og``. + + +.. function:: with_pymalloc() + + Return :data:`_testcapi.WITH_PYMALLOC`. + + .. function:: requires(resource, msg=None) Raise :exc:`ResourceDenied` if *resource* is not available. *msg* is the @@ -266,14 +423,44 @@ The :mod:`test.support` module defines the following functions: Used when tests are executed by :mod:`test.regrtest`. +.. function:: system_must_validate_cert(f) + + Raise :exc:`unittest.SkipTest` on TLS certification validation failures. + + +.. function:: sortdict(dict) + + Return a repr of *dict* with keys sorted. + + .. function:: findfile(filename, subdir=None) Return the path to the file named *filename*. If no match is found *filename* is returned. This does not equal a failure since it could be the path to the file. - Setting *subdir* indicates a relative path to use to find the file - rather than looking directly in the path directories. + Setting *subdir* indicates a relative path to use to find the file + rather than looking directly in the path directories. + + +.. function:: create_empty_file(filename) + + Create an empty file with *filename*. If it already exists, truncate it. + + +.. function:: fd_count() + + Count the number of open file descriptors. + + +.. function:: match_test(test) + + Match *test* to patterns set in :func:`set_match_tests`. + + +.. function:: set_match_tests(patterns) + + Define match test with regular expression *patterns*. .. function:: run_unittest(\*classes) @@ -293,14 +480,32 @@ The :mod:`test.support` module defines the following functions: This will run all tests defined in the named module. -.. function:: run_doctest(module, verbosity=None) +.. function:: run_doctest(module, verbosity=None, optionflags=0) Run :func:`doctest.testmod` on the given *module*. Return ``(failure_count, test_count)``. If *verbosity* is ``None``, :func:`doctest.testmod` is run with verbosity set to :data:`verbose`. Otherwise, it is run with verbosity set to - ``None``. + ``None``. *optionflags* is passed as ``optionflags`` to + :func:`doctest.testmod`. + + +.. function:: setswitchinterval(interval) + + Set the :func:`sys.setswitchinterval` to the given *interval*. Defines + a minimum interval for Android systems to prevent the system from hanging. + + +.. function:: check_impl_detail(**guards) + + Use this check to guard CPython's implementation-specific tests or to + run them only on the implementations guarded by the arguments:: + + check_impl_detail() # Only on CPython (default). + check_impl_detail(jython=True) # Only on Jython. + check_impl_detail(cpython=False) # Everywhere except CPython. + .. function:: check_warnings(\*filters, quiet=True) @@ -368,6 +573,50 @@ The :mod:`test.support` module defines the following functions: New optional arguments *filters* and *quiet*. +.. function:: check_no_resource_warning(testcase) + + Context manager to check that no :exc:`ResourceWarning` was raised. You + must remove the object which may emit :exc:`ResourceWarning` before the + end of the context manager. + + +.. function:: set_memlimit(limit) + + Set the values for :data:`max_memuse` and :data:`real_max_memuse` for big + memory tests. + + +.. function:: record_original_stdout(stdout) + + Store the value from *stdout*. It is meant to hold the stdout at the + time the regrtest began. + + +.. function:: get_original_stdout + + Return the original stdout set by :func:`record_original_stdout` or + ``sys.stdout`` if it's not set. + + +.. function:: strip_python_strerr(stderr) + + Strip the *stderr* of a Python process from potential debug output + emitted by the interpreter. This will typically be run on the result of + :meth:`subprocess.Popen.communicate`. + + +.. function:: args_from_interpreter_flags() + + Return a list of command line arguments reproducing the current settings + in ``sys.flags`` and ``sys.warnoptions``. + + +.. function:: optim_args_from_interpreter_flags() + + Return a list of command line arguments reproducing the current + optimization settings in ``sys.flags``. + + .. function:: captured_stdin() captured_stdout() captured_stderr() @@ -434,17 +683,125 @@ The :mod:`test.support` module defines the following functions: A context manager that temporarily sets the process umask. +.. function:: transient_internet(resource_name, *, timeout=30.0, errnos=()) + + A context manager that raises :exc:`ResourceDenied` when various issues + with the internet connection manifest themselves as exceptions. + + +.. function:: disable_faulthandler() + + A context manager that replaces ``sys.stderr`` with ``sys.__stderr__``. + + +.. function:: gc_collect() + + Force as many objects as possible to be collected. This is needed because + timely deallocation is not guaranteed by the garbage collector. This means + that ``__del__`` methods may be called later than expected and weakrefs + may remain alive for longer than expected. + + +.. function:: disable_gc() + + A context manager that disables the garbage collector upon entry and + reenables it upon exit. + + +.. function:: swap_attr(obj, attr, new_val) + + Context manager to swap out an attribute with a new object. + + Usage:: + + with swap_attr(obj, "attr", 5): + ... + + This will set ``obj.attr`` to 5 for the duration of the ``with`` block, + restoring the old value at the end of the block. If ``attr`` doesn't + exist on ``obj``, it will be created and then deleted at the end of the + block. + + The old value (or ``None`` if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. + + +.. function:: swap_item(obj, attr, new_val) + + Context manager to swap out an item with a new object. + + Usage:: + + with swap_item(obj, "item", 5): + ... + + This will set ``obj["item"]`` to 5 for the duration of the ``with`` block, + restoring the old value at the end of the block. If ``item`` doesn't + exist on ``obj``, it will be created and then deleted at the end of the + block. + + The old value (or ``None`` if it doesn't exist) will be assigned to the + target of the "as" clause, if there is one. + + +.. function:: wait_threads_exit(timeout=60.0) + + Context manager to wait until all threads created in the ``with`` statment + exit. + + +.. function:: start_threads(threads, unlock=None) + + Context manager to start *threads*. It attempts to join the threads upon + exit. + + +.. function:: calcobjsize(fmt) + + Return :func:`struct.calcsize` for ``nP{fmt}0n`` or, if ``gettotalrefcount`` + exists, ``2PnP{fmt}0P``. + + +.. function:: calcvobjsize(fmt) + + Return :func:`struct.calcsize` for ``nPn{fmt}0n`` or, if ``gettotalrefcount`` + exists, ``2PnPn{fmt}0P``. + + +.. function:: checksizeof(test, o, size) + + For testcase *test*, assert that the ``sys.getsizeof`` for *o* plus the GC + header size equals *size*. + + .. function:: can_symlink() Return ``True`` if the OS supports symbolic links, ``False`` otherwise. +.. function:: can_xattr() + + Return ``True`` if the OS supports xattr, ``False`` + otherwise. + + .. decorator:: skip_unless_symlink A decorator for running tests that require support for symbolic links. +.. decorator:: skip_unless_xattr + + A decorator for running tests that require support for xattr. + + +.. decorator:: skip_unless_bind_unix_socket + + A decorator for running tests that require a functional bind() for Unix + sockets. + + .. decorator:: anticipate_failure(condition) A decorator to conditionally mark tests with @@ -460,20 +817,145 @@ The :mod:`test.support` module defines the following functions: sequentially, and the first valid locale will be used. +.. decorator:: run_with_tz(tz) + + A decorator for running a function in a specific timezone, correctly + resetting it after it has finished. + + +.. decorator:: requires_freebsd_version(*min_version) + + Decorator for the minimum version when running test on FreeBSD. If the + FreeBSD version is less than the minimum, raise :exc:`unittest.SkipTest`. + + +.. decorator:: requires_linux_version(*min_version) + + Decorator for the minimum version when running test on Linux. If the + Linux version is less than the minimum, raise :exc:`unittest.SkipTest`. + + +.. decorator:: requires_mac_version(*min_version) + + Decorator for the minimum version when running test on Mac OS X. If the + MAC OS X version is less than the minimum, raise :exc:`unittest.SkipTest`. + + +.. decorator:: requires_IEEE_754 + + Decorator for skipping tests on non-IEEE 754 platforms. + + +.. decorator:: requires_zlib + + Decorator for skipping tests if :mod:`zlib` doesn't exist. + + +.. decorator:: requires_gzip + + Decorator for skipping tests if :mod:`gzip` doesn't exist. + + +.. decorator:: requires_bz2 + + Decorator for skipping tests if :mod:`bz2` doesn't exist. + + +.. decorator:: requires_lzma + + Decorator for skipping tests if :mod:`lzma` doesn't exist. + + +.. decorator:: requires_resource(resource) + + Decorator for skipping tests if *resource* is not available. + + +.. decorator:: requires_docstrings + + Decorator for only running the test if :data:`HAVE_DOCSTRINGS`. + + +.. decorator:: cpython_only(test) + + Decorator for tests only applicable to CPython. + + +.. decorator:: impl_detail(msg=None, **guards) + + Decorator for invoking :func:`check_impl_detail` on *guards*. If that + returns ``False``, then uses *msg* as the reason for skipping the test. + + +.. decorator:: no_tracing(func) + + Decorator to temporarily turn off tracing for the duration of the test. + + +.. decorator:: refcount_test(test) + + Decorator for tests which involve reference counting. The decorator does + not run the test if it is not run by CPython. Any trace function is unset + for the duration of the test to prevent unexpected refcounts caused by + the trace function. + + +.. decorator:: reap_threads(func) + + Decorator to ensure the threads are cleaned up even if the test fails. + + +.. decorator:: bigmemtest(size, memuse, dry_run=True) + + Decorator for bigmem tests. + + *size* is a requested size for the test (in arbitrary, test-interpreted + units.) *memuse* is the number of bytes per unit for the test, or a good + estimate of it. For example, a test that needs two byte buffers, of 4 GiB + each, could be decorated with ``@bigmemtest(size=_4G, memuse=2)``. + + The *size* argument is normally passed to the decorated test method as an + extra argument. If *dry_run* is ``True``, the value passed to the test + method may be less than the requested value. If *dry_run* is ``False``, it + means the test doesn't support dummy runs when ``-M`` is not specified. + + +.. decorator:: bigaddrspacetest(f) + + Decorator for tests that fill the address space. *f* is the function to + wrap. + + .. function:: make_bad_fd() Create an invalid file descriptor by opening and closing a temporary file, and returning its descriptor. -.. function:: import_module(name, deprecated=False) +.. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) + + Test for syntax errors in *statement* by attempting to compile *statement*. + *testcase* is the :mod:`unittest` instance for the test. *errtext* is the + text of the error raised by :exc:`SyntaxError`. If *lineno* is not None, + compares to the line of the :exc:`SyntaxError`. If *offset* is not None, + compares to the offset of the :exc:`SyntaxError`. + + +.. function:: open_urlresource(url, *args, **kw) + + Open *url*. If open fails, raises :exc:`TestFailed`. + + +.. function:: import_module(name, deprecated=False, *, required_on()) This function imports and returns the named module. Unlike a normal import, this function raises :exc:`unittest.SkipTest` if the module cannot be imported. Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. + if *deprecated* is ``True``. If a module is required on a platform but + optional for others, set *required_on* to an iterable of platform prefixes + which will be compared against :data:`sys.platform`. .. versionadded:: 3.1 @@ -514,6 +996,47 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.1 +.. function:: modules_setup() + + Return a copy of :data:`sys.modules`. + + +.. function:: modules_cleanup(oldmodules) + + Remove modules except for *oldmodules* and ``encodings`` in order to + preserve internal cache. + + +.. function:: threading_setup() + + Return current thread count and copy of dangling threads. + + +.. function:: threading_cleanup(*original_values) + + Cleanup up threads not specified in *original_values*. Designed to emit + a warning if a test leaves running threads in the background. + + +.. function:: join_thread(thread, timeout=30.0) + + Join a *thread* within *timeout*. Raise an :exc:`AssertionError` if thread + is still alive after *timeout* seconds. + + +.. function:: reap_children() + + Use this at the end of ``test_main`` whenever sub-processes are started. + This will help ensure that no extra children (zombies) stick around to + hog resources and create problems when looking for refleaks. + + +.. function:: get_attribute(obj, name) + + Get an attribute, raising :exc:`unittest.SkipTest` if :exc:`AttributeError` + is raised. + + .. function:: bind_port(sock, host=HOST) Bind the socket to a free port and return the port number. Relies on @@ -533,6 +1056,12 @@ The :mod:`test.support` module defines the following functions: test. +.. function:: bind_unix_socket(sock, addr) + + Bind a unix socket, raising :exc:`unittest.SkipTest` if + :exc:`PermissionError` is raised. + + .. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) Returns an unused port that should be suitable for binding. This is @@ -571,6 +1100,11 @@ The :mod:`test.support` module defines the following functions: return load_package_tests(os.path.dirname(__file__), *args) +.. function:: fs_is_case_insensitive(directory) + + Return ``True`` if the file system for *directory* is case-insensitive. + + .. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) Returns the set of attributes, functions or methods of *ref_api* not @@ -583,6 +1117,33 @@ The :mod:`test.support` module defines the following functions: .. versionadded:: 3.5 +.. function:: patch(test_instance, object_to_patch, attr_name, new_value) + + Override *object_to_patch.attr_name* with *new_value*. Also add + cleanup procedure to *test_instance* to restore *object_to_patch* for + *attr_name*. The *attr_name* should be a valid attribute for + *object_to_patch*. + + +.. function:: run_in_subinterp(code) + + Run *code* in subinterpreter. Raise :exc:`unittest.SkipTest` if + :mod:`tracemalloc` is enabled. + + +.. function:: check_free_after_iterating(test, iter, cls, args=()) + + Assert that *iter* is deallocated after iterating. + + +.. function:: missing_compiler_executable(cmd_names=[]) + + Check for the existence of the compiler executables whose names are listed + in *cmd_names* or all the compiler executables when *cmd_names* is empty + and return the first missing executable or ``None`` when none is found + missing. + + .. function:: check__all__(test_case, module, name_of_module=None, extra=(), blacklist=()) Assert that the ``__all__`` variable of *module* contains all public names. @@ -592,7 +1153,7 @@ The :mod:`test.support` module defines the following functions: *module*. The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in in order to be detected as a public + module(s) an API could be defined in order to be detected as a public API. One case for this is when *module* imports part of its public API from other modules, possibly a C backend (like ``csv`` and its ``_csv``). @@ -673,7 +1234,169 @@ The :mod:`test.support` module defines the following classes: On both platforms, the old value is restored by :meth:`__exit__`. +.. class:: CleanImport(*module_names) + + A context manager to force import to return a new module reference. This + is useful for testing module-level behaviors, such as the emission of a + DeprecationWarning on import. Example usage:: + + with CleanImport('foo'): + importlib.import_module('foo') # New reference. + + +.. class:: DirsOnSysPath(*paths) + + A context manager to temporarily add directories to sys.path. + + This makes a copy of :data:`sys.path`, appends any directories given + as positional arguments, then reverts :data:`sys.path` to the copied + settings when the context ends. + + Note that *all* :data:`sys.path` modifications in the body of the + context manager, including replacement of the object, + will be reverted at the end of the block. + + +.. class:: SaveSignals() + + Class to save and restore signal handlers registered by the Python signal + handler. + + +.. class:: Matcher() + + .. method:: matches(self, d, **kwargs) + + Try to match a single dict with the supplied arguments. + + + .. method:: match_value(self, k, dv, v) + + Try to match a single stored value (*dv*) with a supplied value (*v*). + + .. class:: WarningsRecorder() Class used to record warnings for unit tests. See documentation of :func:`check_warnings` above for more details. + + +.. class:: BasicTestRunner() + + .. method:: run(test) + + Run *test* and return the result. + + +.. class:: TestHandler(logging.handlers.BufferingHandler) + + Class for logging support. + + +.. class:: FakePath(path) + + Simple :term:`path-like object`. It implements the :meth:`__fspath__` + method which just returns the *path* argument. If *path* is an exception, + it will be raised in :meth:`!__fspath__`. + + +:mod:`test.support.script_helper` --- Utilities for the Python execution tests +============================================================================== + +.. module:: test.support.script_helper + :synopsis: Support for Python's script execution tests. + + +The :mod:`test.support.script_helper` module provides support for Python's +script execution tests. + +.. function:: interpreter_requires_environment() + + Return ``True`` if ``sys.executable interpreter`` requires environment + variables in order to be able to run at all. + + This is designed to be used with ``@unittest.skipIf()`` to annotate tests + that need to use an ``assert_python*()`` function to launch an isolated + mode (``-I``) or no environment mode (``-E``) sub-interpreter process. + + A normal build & test does not run into this situation but it can happen + when trying to run the standard library test suite from an interpreter that + doesn't have an obvious home with Python's current home finding logic. + + Setting :envvar:`PYTHONHOME` is one way to get most of the testsuite to run + in that situation. :envvar:`PYTHONPATH` or :envvar:`PYTHONUSERSITE` are + other common environment variables that might impact whether or not the + interpreter can start. + + +.. function:: run_python_until_end(*args, **env_vars) + + Set up the environment based on *env_vars* for running the interpreter + in a subprocess. The values can include ``__isolated``, ``__cleanenv``, + ``__cwd``, and ``TERM``. + + +.. function:: assert_python_ok(*args, **env_vars) + + Assert that running the interpreter with *args* and optional environment + variables *env_vars* succeeds (``rc == 0``) and return a ``(return code, + stdout, stderr)`` tuple. + + If the ``__cleanenv`` keyword is set, *env_vars* is used as a fresh + environment. + + Python is started in isolated mode (command line option ``-I``), + except if the ``__isolated`` keyword is set to ``False``. + + +.. function:: assert_python_failure(*args, **env_vars) + + Assert that running the interpreter with *args* and optional environment + variables *env_vars* fails (``rc != 0``) and return a ``(return code, + stdout, stderr)`` tuple. + + See :func:`assert_python_ok` for more options. + + +.. function:: spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw) + + Run a Python subprocess with the given arguments. + + *kw* is extra keyword args to pass to :func:`subprocess.Popen`. Returns a + :class:`subprocess.Popen` object. + + +.. function:: kill_python(p) + + Run the given :class:`subprocess.Popen` process until completion and return + stdout. + + +.. function:: make_script(script_dir, script_basename, source, omit_suffix=False) + + Create script containing *source* in path *script_dir* and *script_basename*. + If *omit_suffix* is ``False``, append ``.py`` to the name. Return the full + script path. + + +.. function:: make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None) + + Create zip file at *zip_dir* and *zip_basename* with extension ``zip`` which + contains the files in *script_name*. *name_in_zip* is the archive name. + Return a tuple containing ``(full path, full path of archive name)``. + + +.. function:: make_pkg(pkg_dir, init_source='') + + Create a directory named *pkg_dir* containing an ``__init__`` file with + *init_source* as its contents. + + +.. function:: make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, \ + source, depth=1, compiled=False) + + Create a zip package directory with a path of *zip_dir* and *zip_basename* + containing an empty ``__init__`` file and a file *script_basename* + containing the *source*. If *compiled* is ``True``, both source files will + be compiled and added to the zip package. Return a tuple of the full zip + path and the archive name for the zip file. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst index 8bbadf11608b347..d922fd6f32e26b6 100644 --- a/Doc/library/timeit.rst +++ b/Doc/library/timeit.rst @@ -25,7 +25,7 @@ Basic Examples The following example shows how the :ref:`timeit-command-line-interface` can be used to compare three different expressions: -.. code-block:: sh +.. code-block:: shell-session $ python3 -m timeit '"-".join(str(n) for n in range(100))' 10000 loops, best of 5: 30.2 usec per loop @@ -264,7 +264,7 @@ Examples It is possible to provide a setup statement that is executed only once at the beginning: -.. code-block:: sh +.. code-block:: shell-session $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' 5000000 loops, best of 5: 0.0877 usec per loop @@ -293,7 +293,7 @@ The following examples show how to time expressions that contain multiple lines. Here we compare the cost of using :func:`hasattr` vs. :keyword:`try`/:keyword:`except` to test for missing and present object attributes: -.. code-block:: sh +.. code-block:: shell-session $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' 20000 loops, best of 5: 15.7 usec per loop diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst index debbc81ce1a3903..76ecfcce497775e 100644 --- a/Doc/library/tkinter.ttk.rst +++ b/Doc/library/tkinter.ttk.rst @@ -66,13 +66,13 @@ for improved styling effects. Ttk Widgets ----------- -Ttk comes with 17 widgets, eleven of which already existed in tkinter: +Ttk comes with 18 widgets, twelve of which already existed in tkinter: :class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`. The other six are -new: :class:`Combobox`, :class:`Notebook`, :class:`Progressbar`, -:class:`Separator`, :class:`Sizegrip` and :class:`Treeview`. And all them are -subclasses of :class:`Widget`. +:class:`Radiobutton`, :class:`Scale`, :class:`Scrollbar`, and :class:`Spinbox`. +The other six are new: :class:`Combobox`, :class:`Notebook`, +:class:`Progressbar`, :class:`Separator`, :class:`Sizegrip` and +:class:`Treeview`. And all them are subclasses of :class:`Widget`. Using the Ttk widgets gives the application an improved look and feel. As discussed above, there are differences in how the styling is coded. @@ -381,6 +381,87 @@ ttk.Combobox Sets the value of the combobox to *value*. +Spinbox +------- +The :class:`ttk.Spinbox` widget is a :class:`ttk.Entry` enhanced with increment +and decrement arrows. It can be used for numbers or lists of string values. +This widget is a subclass of :class:`Entry`. + +Besides the methods inherited from :class:`Widget`: :meth:`Widget.cget`, +:meth:`Widget.configure`, :meth:`Widget.identify`, :meth:`Widget.instate` +and :meth:`Widget.state`, and the following inherited from :class:`Entry`: +:meth:`Entry.bbox`, :meth:`Entry.delete`, :meth:`Entry.icursor`, +:meth:`Entry.index`, :meth:`Entry.insert`, :meth:`Entry.xview`, +it has some other methods, described at :class:`ttk.Spinbox`. + +Options +^^^^^^^ + +This widget accepts the following specific options: + + .. tabularcolumns:: |l|L| + ++----------------------+------------------------------------------------------+ +| Option | Description | ++======================+======================================================+ +| from | Float value. If set, this is the minimum value to | +| | which the decrement button will decrement. Must be | +| | spelled as ``from_`` when used as an argument, since | +| | ``from`` is a Python keyword. | ++----------------------+------------------------------------------------------+ +| to | Float value. If set, this is the maximum value to | +| | which the increment button will increment. | ++----------------------+------------------------------------------------------+ +| increment | Float value. Specifies the amount which the | +| | increment/decrement buttons change the | +| | value. Defaults to 1.0. | ++----------------------+------------------------------------------------------+ +| values | Sequence of string or float values. If specified, | +| | the increment/decrement buttons will cycle through | +| | the items in this sequence rather than incrementing | +| | or decrementing numbers. | +| | | ++----------------------+------------------------------------------------------+ +| wrap | Boolean value. If ``True``, increment and decrement | +| | buttons will cycle from the ``to`` value to the | +| | ``from`` value or the ``from`` value to the ``to`` | +| | value, respectively. | ++----------------------+------------------------------------------------------+ +| format | String value. This specifies the format of numbers | +| | set by the increment/decrement buttons. It must be | +| | in the form "%W.Pf", where W is the padded width of | +| | the value, P is the precision, and '%' and 'f' are | +| | literal. | ++----------------------+------------------------------------------------------+ +| command | Python callable. Will be called with no arguments | +| | whenever either of the increment or decrement buttons| +| | are pressed. | +| | | ++----------------------+------------------------------------------------------+ + + +Virtual events +^^^^^^^^^^^^^^ + +The spinbox widget generates an **<>** virtual event when the +user presses , and a **<>** virtual event when the user +presses . + +ttk.Spinbox +^^^^^^^^^^^^ + +.. class:: Spinbox + + .. method:: get() + + Returns the current value of the spinbox. + + + .. method:: set(value) + + Sets the value of the spinbox to *value*. + + Notebook -------- diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst index 02a0428f21bc769..4c0a0ceef7dc4e7 100644 --- a/Doc/library/tokenize.rst +++ b/Doc/library/tokenize.rst @@ -218,7 +218,7 @@ will be tokenized to the following output where the first column is the range of the line/column coordinates where the token is found, the second column is the name of the token, and the final column is the value of the token (if any) -.. code-block:: sh +.. code-block:: shell-session $ python -m tokenize hello.py 0,0-0,0: ENCODING 'utf-8' @@ -244,7 +244,7 @@ the name of the token, and the final column is the value of the token (if any) The exact token type names can be displayed using the :option:`-e` option: -.. code-block:: sh +.. code-block:: shell-session $ python -m tokenize -e hello.py 0,0-0,0: ENCODING 'utf-8' diff --git a/Doc/library/types.rst b/Doc/library/types.rst index bbc1d1301d2c2a2..67cd4d702ad4a61 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -198,16 +198,23 @@ Standard names are defined for the following types: Defaults to ``None``. Previously the attribute was optional. -.. data:: TracebackType +.. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) The type of traceback objects such as found in ``sys.exc_info()[2]``. + See :ref:`the language reference ` for details of the + available attributes and operations, and guidance on creating tracebacks + dynamically. + .. data:: FrameType The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a traceback object. + See :ref:`the language reference ` for details of the + available attributes and operations. + .. data:: GetSetDescriptorType diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst index 93ccd0fd61139c3..af71be40b89ff27 100644 --- a/Doc/library/unittest.rst +++ b/Doc/library/unittest.rst @@ -2316,7 +2316,7 @@ handling functionality within test frameworks. When called without arguments this function removes the control-c handler if it has been installed. This function can also be used as a test decorator - to temporarily remove the handler whilst the test is being executed:: + to temporarily remove the handler while the test is being executed:: @unittest.removeHandler def test_signal_handling(self): diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index 61808596a555323..6298f1396ae294d 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -297,7 +297,7 @@ If the XML input has `namespaces with prefixes in the form ``prefix:sometag`` get expanded to ``{uri}sometag`` where the *prefix* is replaced by the full *URI*. Also, if there is a `default namespace -`__, +`__, that full URI gets prepended to all of the non-prefixed tags. Here is an XML example that incorporates two namespaces, one with the diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst index 9076593c4b1c220..26b0f19fdeb648f 100644 --- a/Doc/library/zipapp.rst +++ b/Doc/library/zipapp.rst @@ -27,7 +27,7 @@ can be used to create an executable archive from a directory containing Python code. When run, the archive will execute the ``main`` function from the module ``myapp`` in the archive. -.. code-block:: sh +.. code-block:: shell-session $ python -m zipapp myapp -m "myapp:main" $ python myapp.pyz @@ -41,7 +41,7 @@ Command-Line Interface When called as a program from the command line, the following form is used: -.. code-block:: sh +.. code-block:: shell-session $ python -m zipapp source [options] @@ -189,7 +189,7 @@ Examples Pack up a directory into an archive, and run it. -.. code-block:: sh +.. code-block:: shell-session $ python -m zipapp myapp $ python myapp.pyz @@ -203,7 +203,7 @@ The same can be done using the :func:`create_archive` functon:: To make the application directly executable on POSIX, specify an interpreter to use. -.. code-block:: sh +.. code-block:: shell-session $ python -m zipapp myapp -p "/usr/bin/env python" $ ./myapp.pyz @@ -229,6 +229,12 @@ fits in memory:: >>> with open('myapp.pyz', 'wb') as f: >>> f.write(temp.getvalue()) + +.. _zipapp-specifying-the-interpreter: + +Specifying the Interpreter +-------------------------- + Note that if you specify an interpreter and then distribute your application archive, you need to ensure that the interpreter used is portable. The Python launcher for Windows supports most common forms of POSIX ``#!`` line, but there @@ -245,6 +251,169 @@ are other issues to consider: exact version like "/usr/bin/env python3.4" as you will need to change your shebang line for users of Python 3.5, for example. +Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", +depending on whether your code is written for Python 2 or 3. + + +Creating Standalone Applications with zipapp +-------------------------------------------- + +Using the :mod:`zipapp` module, it is possible to create self-contained Python +programs, which can be distributed to end users who only need to have a +suitable version of Python installed on their system. The key to doing this +is to bundle all of the application's dependencies into the archive, along +with the application code. + +The steps to create a standalone archive are as follows: + +1. Create your application in a directory as normal, so you have a ``myapp`` + directory containing a ``__main__.py`` file, and any supporting application + code. + +2. Install all of your application's dependencies into the ``myapp`` directory, + using pip: + + .. code-block:: shell-session + + $ python -m pip install -r requirements.txt --target myapp + + (this assumes you have your project requirements in a ``requirements.txt`` + file - if not, you can just list the dependencies manually on the pip command + line). + +3. Optionally, delete the ``.dist-info`` directories created by pip in the + ``myapp`` directory. These hold metadata for pip to manage the packages, and + as you won't be making any further use of pip they aren't required - + although it won't do any harm if you leave them. + +4. Package the application using: + + .. code-block:: shell-session + + $ python -m zipapp -p "interpreter" myapp + +This will produce a standalone executable, which can be run on any machine with +the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` +for details. It can be shipped to users as a single file. + +On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the +file to remove the ``.pyz`` extension if you prefer a "plain" command name. On +Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that +the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions +when installed. + + +Making a Windows executable +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +On Windows, registration of the ``.pyz`` extension is optional, and +furthermore, there are certain places that don't recognise registered +extensions "transparently" (the simplest example is that +``subprocess.run(['myapp'])`` won't find your application - you need to +explicitly specify the extension). + +On Windows, therefore, it is often preferable to create an executable from the +zipapp. This is relatively easy, although it does require a C compiler. The +basic approach relies on the fact that zipfiles can have arbitrary data +prepended, and Windows exe files can have arbitrary data appended. So by +creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, +you end up with a single-file executable that runs your application. + +A suitable launcher can be as simple as the following:: + + #define Py_LIMITED_API 1 + #include "Python.h" + + #define WIN32_LEAN_AND_MEAN + #include + + #ifdef WINDOWS + int WINAPI wWinMain( + HINSTANCE hInstance, /* handle to current instance */ + HINSTANCE hPrevInstance, /* handle to previous instance */ + LPWSTR lpCmdLine, /* pointer to command line */ + int nCmdShow /* show state of window */ + ) + #else + int wmain() + #endif + { + wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); + myargv[0] = __wargv[0]; + memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); + return Py_Main(__argc+1, myargv); + } + +If you define the ``WINDOWS`` preprocessor symbol, this will generate a +GUI executable, and without it, a console executable. + +To compile the executable, you can either just use the standard MSVC +command line tools, or you can take advantage of the fact that distutils +knows how to compile Python source:: + + >>> from distutils.ccompiler import new_compiler + >>> import distutils.sysconfig + >>> import sys + >>> import os + >>> from pathlib import Path + + >>> def compile(src): + >>> src = Path(src) + >>> cc = new_compiler() + >>> exe = src.stem + >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) + >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) + >>> # First the CLI executable + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe) + >>> # Now the GUI executable + >>> cc.define_macro('WINDOWS') + >>> objs = cc.compile([str(src)]) + >>> cc.link_executable(objs, exe + 'w') + + >>> if __name__ == "__main__": + >>> compile("zastub.c") + +The resulting launcher uses the "Limited ABI", so it will run unchanged with +any version of Python 3.x. All it needs is for Python (``python3.dll``) to be +on the user's ``PATH``. + +For a fully standalone distribution, you can distribute the launcher with your +application appended, bundled with the Python "embedded" distribution. This +will run on any PC with the appropriate architecture (32 bit or 64 bit). + + +Caveats +~~~~~~~ + +There are some limitations to the process of bundling your application into +a single file. In most, if not all, cases they can be addressed without +needing major changes to your application. + +1. If your application depends on a package that includes a C extension, that + package cannot be run from a zip file (this is an OS limitation, as executable + code must be present in the filesystem for the OS loader to load it). In this + case, you can exclude that dependency from the zipfile, and either require + your users to have it installed, or ship it alongside your zipfile and add code + to your ``__main__.py`` to include the directory containing the unzipped + module in ``sys.path``. In this case, you will need to make sure to ship + appropriate binaries for your target architecture(s) (and potentially pick the + correct version to add to ``sys.path`` at runtime, based on the user's machine). + +2. If you are shipping a Windows executable as described above, you either need to + ensure that your users have ``python3.dll`` on their PATH (which is not the + default behaviour of the installer) or you should bundle your application with + the embedded distribution. + +3. The suggested launcher above uses the Python embedding API. This means that in + your application, ``sys.executable`` will be your application, and *not* a + conventional Python interpreter. Your code and its dependencies need to be + prepared for this possibility. For example, if your application uses the + :mod:`multiprocessing` module, it will need to call + :func:`multiprocessing.set_executable` to let the module know where to find the + standard Python interpreter. + + The Python Zip Application Archive Format ----------------------------------------- diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst index 7c9a8c80225491a..c0f2a89a3a17fa6 100644 --- a/Doc/library/zipfile.rst +++ b/Doc/library/zipfile.rst @@ -491,7 +491,7 @@ The :class:`PyZipFile` constructor takes the same parameters as the :file:`\*.pyc` are added at the top level. If the directory is a package directory, then all :file:`\*.pyc` are added under the package name as a file path, and if any subdirectories are package directories, - all of these are added recursively. + all of these are added recursively in sorted order. *basename* is intended for internal use only. @@ -524,6 +524,9 @@ The :class:`PyZipFile` constructor takes the same parameters as the .. versionchanged:: 3.6.2 The *pathname* parameter accepts a :term:`path-like object`. + .. versionchanged:: 3.7 + Recursion sorts directory entries. + .. _zipinfo-objects: diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 3d742ab35b9cc93..8a531c92b8ff154 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -51,9 +51,9 @@ The available exception and functions in this module are: Compresses the bytes in *data*, returning a bytes object containing compressed data. *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` is fastest and produces the least compression, ``9`` is slowest and - produces the most. ``0`` is no compression. The default value is ``-1`` - (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default + ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) + is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. + The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression (currently equivalent to level 6). Raises the :exc:`error` exception if any error occurs. @@ -61,23 +61,25 @@ The available exception and functions in this module are: *level* can now be used as a keyword parameter. -.. function:: compressobj(level=-1, method=DEFLATED, wbits=15, memLevel=8, strategy=Z_DEFAULT_STRATEGY[, zdict]) +.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) Returns a compression object, to be used for compressing data streams that won't fit into memory at once. *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` is fastest and produces the least compression, while a value of - ``9`` is slowest and produces the most. ``0`` is no compression. The default - value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). + A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, + while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. + ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). + Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression + (currently equivalent to level 6). *method* is the compression algorithm. Currently, the only supported value is - ``DEFLATED``. + :const:`DEFLATED`. The *wbits* argument controls the size of the history buffer (or the "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values: + trailer is included in the output. It can take several ranges of values, + defaulting to ``15`` (MAX_WBITS): * +9 to +15: The base-two logarithm of the window size, which therefore ranges between 512 and 32768. Larger values produce @@ -97,7 +99,8 @@ The available exception and functions in this module are: Higher values use more memory, but are faster and produce smaller output. *strategy* is used to tune the compression algorithm. Possible values are - ``Z_DEFAULT_STRATEGY``, ``Z_FILTERED``, and ``Z_HUFFMAN_ONLY``. + :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, + :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). *zdict* is a predefined compression dictionary. This is a sequence of bytes (such as a :class:`bytes` object) containing subsequences that are expected @@ -175,7 +178,7 @@ The available exception and functions in this module are: .. versionchanged:: 3.6 *wbits* and *bufsize* can be used as keyword arguments. -.. function:: decompressobj(wbits=15[, zdict]) +.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) Returns a decompression object, to be used for decompressing data streams that won't fit into memory at once. @@ -213,13 +216,13 @@ Compression objects support the following methods: All pending input is processed, and a bytes object containing the remaining compressed output is returned. *mode* can be selected from the constants - :const:`Z_SYNC_FLUSH`, :const:`Z_FULL_FLUSH`, or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. :const:`Z_SYNC_FLUSH` and - :const:`Z_FULL_FLUSH` allow compressing further bytestrings of data, while - :const:`Z_FINISH` finishes the compressed stream and prevents compressing any - more data. After calling :meth:`flush` with *mode* set to :const:`Z_FINISH`, - the :meth:`compress` method cannot be called again; the only realistic action is - to delete the object. + :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, + :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, + defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants + allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the + compressed stream and prevents compressing any more data. After calling :meth:`flush` + with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; + the only realistic action is to delete the object. .. method:: Compress.copy() diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 25b95c115438b3b..1e93ef4594a9344 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -950,7 +950,7 @@ Internal types .. index:: object: frame Frame objects represent execution frames. They may occur in traceback objects - (see below). + (see below), and are also passed to registered trace functions. .. index:: single: f_back (frame attribute) @@ -1003,6 +1003,8 @@ Internal types .. versionadded:: 3.4 + .. _traceback-objects: + Traceback objects .. index:: object: traceback @@ -1015,31 +1017,51 @@ Internal types single: sys.last_traceback Traceback objects represent a stack trace of an exception. A traceback object - is created when an exception occurs. When the search for an exception handler + is implicitly created when an exception occurs, and may also be explicitly + created by calling :class:`types.TracebackType`. + + For implicitly created tracebacks, when the search for an exception handler unwinds the execution stack, at each unwound level a traceback object is inserted in front of the current traceback. When an exception handler is entered, the stack trace is made available to the program. (See section :ref:`try`.) It is accessible as the third item of the - tuple returned by ``sys.exc_info()``. When the program contains no suitable + tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute + of the caught exception. + + When the program contains no suitable handler, the stack trace is written (nicely formatted) to the standard error stream; if the interpreter is interactive, it is also made available to the user as ``sys.last_traceback``. + For explicitly created tracebacks, it is up to the creator of the traceback + to determine how the ``tb_next`` attributes should be linked to form a + full stack trace. + .. index:: - single: tb_next (traceback attribute) single: tb_frame (traceback attribute) single: tb_lineno (traceback attribute) single: tb_lasti (traceback attribute) statement: try - Special read-only attributes: :attr:`tb_next` is the next level in the stack - trace (towards the frame where the exception occurred), or ``None`` if there is - no next level; :attr:`tb_frame` points to the execution frame of the current - level; :attr:`tb_lineno` gives the line number where the exception occurred; - :attr:`tb_lasti` indicates the precise instruction. The line number and last - instruction in the traceback may differ from the line number of its frame object - if the exception occurred in a :keyword:`try` statement with no matching except - clause or with a finally clause. + Special read-only attributes: + :attr:`tb_frame` points to the execution frame of the current level; + :attr:`tb_lineno` gives the line number where the exception occurred; + :attr:`tb_lasti` indicates the precise instruction. + The line number and last instruction in the traceback may differ from the + line number of its frame object if the exception occurred in a + :keyword:`try` statement with no matching except clause or with a + finally clause. + + .. index:: + single: tb_next (traceback attribute) + + Special writable attribute: :attr:`tb_next` is the next level in the stack + trace (towards the frame where the exception occurred), or ``None`` if + there is no next level. + + .. versionchanged:: 3.7 + Traceback objects can now be explicitly instantiated from Python code, + and the ``tb_next`` attribute of existing instances can be updated. Slice objects .. index:: builtin: slice @@ -1463,10 +1485,12 @@ access (use of, assignment to, or deletion of ``x.name``) for class instances. .. method:: object.__getattr__(self, name) - Called when an attribute lookup has not found the attribute in the usual places - (i.e. it is not an instance attribute nor is it found in the class tree for - ``self``). ``name`` is the attribute name. This method should return the - (computed) attribute value or raise an :exc:`AttributeError` exception. + Called when the default attribute access fails with an :exc:`AttributeError` + (either :meth:`__getattribute__` raises an :exc:`AttributeError` because + *name* is not an instance attribute or an attribute in the class tree + for ``self``; or :meth:`__get__` of a *name* property raises + :exc:`AttributeError`). This method should either return the (computed) + attribute value or raise an :exc:`AttributeError` exception. Note that if the attribute is found through the normal mechanism, :meth:`__getattr__` is not called. (This is an intentional asymmetry between @@ -2340,16 +2364,14 @@ left undefined. .. method:: object.__complex__(self) object.__int__(self) object.__float__(self) - object.__round__(self, [,n]) .. index:: builtin: complex builtin: int builtin: float - builtin: round Called to implement the built-in functions :func:`complex`, - :func:`int`, :func:`float` and :func:`round`. Should return a value + :func:`int` and :func:`float`. Should return a value of the appropriate type. @@ -2368,6 +2390,23 @@ left undefined. the same value. +.. method:: object.__round__(self, [,ndigits]) + object.__trunc__(self) + object.__floor__(self) + object.__ceil__(self) + + .. index:: builtin: round + + Called to implement the built-in function :func:`round` and :mod:`math` + functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. + Unless *ndigits* is passed to :meth:`!__round__` all these methods should + return the value of the object truncated to an :class:`~numbers.Integral` + (typically an :class:`int`). + + If :meth:`__int__` is not defined then the built-in function :func:`int` + falls back to :meth:`__trunc__`. + + .. _context-managers: With Statement Context Managers diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index fb92ad0f07c21ed..7fe989b90524119 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -172,7 +172,7 @@ Common syntax elements for comprehensions are: .. productionlist:: comprehension: `expression` `comp_for` - comp_for: [ASYNC] "for" `target_list` "in" `or_test` [`comp_iter`] + comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` comp_if: "if" `expression_nocond` [`comp_iter`] diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst index 95b0f53a83e6042..c6d96ca612b8c38 100644 --- a/Doc/reference/lexical_analysis.rst +++ b/Doc/reference/lexical_analysis.rst @@ -676,7 +676,7 @@ Some examples of formatted string literals:: >>> f"result: {value:{width}.{precision}}" # nested fields 'result: 12.35' >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%b %d, %Y}" # using date format specifier + >>> f"{today:%B %d, %Y}" # using date format specifier 'January 27, 2017' >>> number = 1024 >>> f"{number:#0x}" # using integer format specifier diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py index 00acd4f55b8b957..bb69366ecd4991c 100644 --- a/Doc/tools/extensions/pyspecific.py +++ b/Doc/tools/extensions/pyspecific.py @@ -35,7 +35,7 @@ ISSUE_URI = 'https://bugs.python.org/issue%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/master/%s' +SOURCE_URI = 'https://github.com/python/cpython/tree/3.7/%s' # monkey-patch reST parser to disable alphabetic and roman enumerated lists from docutils.parsers.rst.states import Body @@ -196,7 +196,7 @@ class DeprecatedRemoved(Directive): final_argument_whitespace = True option_spec = {} - _label = 'Deprecated since version %s, will be removed in version %s' + _label = 'Deprecated since version {deprecated}, will be removed in version {removed}' def run(self): node = addnodes.versionmodified() @@ -204,11 +204,12 @@ def run(self): node['type'] = 'deprecated-removed' version = (self.arguments[0], self.arguments[1]) node['version'] = version - text = self._label % version + label = translators['sphinx'].gettext(self._label) + text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) if len(self.arguments) == 3: inodes, messages = self.state.inline_text(self.arguments[2], self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes) + para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) node.append(para) else: messages = [] @@ -220,13 +221,14 @@ def run(self): content.source = node[0].source content.line = node[0].line content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content)) + node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) node[0].insert(0, nodes.inline('', '%s: ' % text, classes=['versionmodified'])) else: para = nodes.paragraph('', '', nodes.inline('', '%s.' % text, - classes=['versionmodified'])) + classes=['versionmodified']), + translatable=False) node.append(para) env = self.state.document.settings.env env.note_versionchange('deprecated', version[0], node, self.lineno) diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js index c450f5eafffc891..8e0c5ea0092a9fe 100644 --- a/Doc/tools/static/switchers.js +++ b/Doc/tools/static/switchers.js @@ -10,7 +10,8 @@ '(?:release/\\d.\\d[\\x\\d\\.]*)']; var all_versions = { - '3.7': 'dev (3.7)', + '3.8': 'dev (3.8)', + '3.7': 'pre (3.7)', '3.6': '3.6', '3.5': '3.5', '2.7': '2.7', diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html index fca44e9163cac74..21af621efce95b1 100644 --- a/Doc/tools/templates/customsourcelink.html +++ b/Doc/tools/templates/customsourcelink.html @@ -4,7 +4,7 @@

{{ _('This Page') }}

  • {% trans %}Report a Bug{% endtrans %}
  • - {{ _('Show Source') }}
    • diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 6e43be23230b544..8d94137b01b5192 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -4,3 +4,4 @@ In extensions/pyspecific.py: {% trans %}CPython implementation detail:{% endtrans %} +{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 9fa814f923b6e2d..ea50ba9028fdc8e 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,6 +2,7 @@

    {% trans %}Download{% endtrans %}

    {% trans %}Download these documents{% endtrans %}

    {% trans %}Docs for other versions{% endtrans %}

      +
    • {% trans %}Python 3.8 (in development){% endtrans %}
    • {% trans %}Python 3.6 (stable){% endtrans %}
    • {% trans %}Python 3.5 (stable){% endtrans %}
    • {% trans %}Python 2.7 (stable){% endtrans %}
      • diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst index ffd16aa97a15fae..241a812037469e2 100644 --- a/Doc/tutorial/appendix.rst +++ b/Doc/tutorial/appendix.rst @@ -52,7 +52,7 @@ comment in Python. The script can be given an executable mode, or permission, using the :program:`chmod` command. -.. code-block:: bash +.. code-block:: shell-session $ chmod +x myscript.py diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst index 36b093950e10d82..30ef159f8b369ae 100644 --- a/Doc/tutorial/controlflow.rst +++ b/Doc/tutorial/controlflow.rst @@ -105,7 +105,7 @@ is possible to let the range start at another number, or to specify a different increment (even negative; sometimes this is called the 'step'):: range(5, 10) - 5 through 9 + 5, 6, 7, 8, 9 range(0, 10, 3) 0, 3, 6, 9 diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst index b95aca885252070..7855ef2d2882874 100644 --- a/Doc/tutorial/datastructures.rst +++ b/Doc/tutorial/datastructures.rst @@ -497,7 +497,7 @@ You can't use lists as keys, since lists can be modified in place using index assignments, slice assignments, or methods like :meth:`append` and :meth:`extend`. -It is best to think of a dictionary as an unordered set of *key: value* pairs, +It is best to think of a dictionary as a set of *key: value* pairs, with the requirement that the keys are unique (within one dictionary). A pair of braces creates an empty dictionary: ``{}``. Placing a comma-separated list of key:value pairs within the braces adds initial key:value pairs to the @@ -509,9 +509,9 @@ pair with ``del``. If you store using a key that is already in use, the old value associated with that key is forgotten. It is an error to extract a value using a non-existent key. -Performing ``list(d.keys())`` on a dictionary returns a list of all the keys -used in the dictionary, in arbitrary order (if you want it sorted, just use -``sorted(d.keys())`` instead). [2]_ To check whether a single key is in the +Performing ``list(d)`` on a dictionary returns a list of all the keys +used in the dictionary, in insertion order (if you want it sorted, just use +``sorted(d)`` instead). To check whether a single key is in the dictionary, use the :keyword:`in` keyword. Here is a small example using a dictionary:: @@ -519,16 +519,16 @@ Here is a small example using a dictionary:: >>> tel = {'jack': 4098, 'sape': 4139} >>> tel['guido'] = 4127 >>> tel - {'sape': 4139, 'guido': 4127, 'jack': 4098} + {'jack': 4098, 'sape': 4139, 'guido': 4127} >>> tel['jack'] 4098 >>> del tel['sape'] >>> tel['irv'] = 4127 >>> tel - {'guido': 4127, 'irv': 4127, 'jack': 4098} - >>> list(tel.keys()) - ['irv', 'guido', 'jack'] - >>> sorted(tel.keys()) + {'jack': 4098, 'guido': 4127, 'irv': 4127} + >>> list(tel) + ['jack', 'guido', 'irv'] + >>> sorted(tel) ['guido', 'irv', 'jack'] >>> 'guido' in tel True @@ -539,7 +539,7 @@ The :func:`dict` constructor builds dictionaries directly from sequences of key-value pairs:: >>> dict([('sape', 4139), ('guido', 4127), ('jack', 4098)]) - {'sape': 4139, 'jack': 4098, 'guido': 4127} + {'sape': 4139, 'guido': 4127, 'jack': 4098} In addition, dict comprehensions can be used to create dictionaries from arbitrary key and value expressions:: @@ -551,7 +551,7 @@ When the keys are simple strings, it is sometimes easier to specify pairs using keyword arguments:: >>> dict(sape=4139, guido=4127, jack=4098) - {'sape': 4139, 'jack': 4098, 'guido': 4127} + {'sape': 4139, 'guido': 4127, 'jack': 4098} .. _tut-loopidioms: @@ -710,7 +710,3 @@ interpreter will raise a :exc:`TypeError` exception. .. [1] Other languages may return the mutated object, which allows method chaining, such as ``d->insert("a")->remove("b")->sort();``. - -.. [2] Calling ``d.keys()`` will return a :dfn:`dictionary view` object. It - supports operations like membership test and iteration, but its contents - are not independent of the original dictionary -- it is only a *view*. diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst index 74d7bad42a12be2..d5531029d064c2d 100644 --- a/Doc/tutorial/inputoutput.rst +++ b/Doc/tutorial/inputoutput.rst @@ -100,7 +100,7 @@ Here are two ways to write a table of squares and cubes:: 10 100 1000 (Note that in the first example, one space between each column was added by the -way :func:`print` works: it always adds spaces between its arguments.) +way :func:`print` works: by default it adds spaces between its arguments.) This example demonstrates the :meth:`str.rjust` method of string objects, which right-justifies a string in a field of a given width by padding diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst index 2be03ac6ae30765..584d4fd72ea82bc 100644 --- a/Doc/tutorial/modules.rst +++ b/Doc/tutorial/modules.rst @@ -112,6 +112,25 @@ Note that in general the practice of importing ``*`` from a module or package is frowned upon, since it often causes poorly readable code. However, it is okay to use it to save typing in interactive sessions. +If the module name is followed by :keyword:`as`, then the name +following :keyword:`as` is bound directly to the imported module. + +:: + + >>> import fibo as fib + >>> fib.fib(500) + 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 + +This is effectively importing the module in the same way that ``import fibo`` +will do, with the only difference of it being available as ``fib``. + +It can also be used when utilising :keyword:`from` with similar effects:: + + >>> from fibo import fib as fibonacci + >>> fibonacci(500) + 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 + + .. note:: For efficiency reasons, each module is only imported once per interpreter diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 1e9ed6e645a2e86..e72dea907580270 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -178,11 +178,15 @@ Generic options .. cmdoption:: -V --version - Print the Python version number and exit. Example output could be:: + Print the Python version number and exit. Example output could be: + + .. code-block:: none Python 3.6.0b2+ - When given twice, print more information about the build, like:: + When given twice, print more information about the build, like: + + .. code-block:: none Python 3.6.0b2+ (3.6:84a3c5003510+, Oct 26 2016, 02:33:55) [GCC 6.2.0 20161005] @@ -260,12 +264,23 @@ Miscellaneous options .. cmdoption:: -O - Turn on basic optimizations. See also :envvar:`PYTHONOPTIMIZE`. + Remove assert statements and any code conditional on the value of + :const:`__debug__`. Augment the filename for compiled + (:term:`bytecode`) files by adding ``.opt-1`` before the ``.pyc`` + extension (see :pep:`488`). See also :envvar:`PYTHONOPTIMIZE`. + + .. versionchanged:: 3.5 + Modify ``.pyc`` filenames according to :pep:`488`. .. cmdoption:: -OO - Discard docstrings in addition to the :option:`-O` optimizations. + Do :option:`-O` and also discard docstrings. Augment the filename + for compiled (:term:`bytecode`) files by adding ``.opt-2`` before the + ``.pyc`` extension (see :pep:`488`). + + .. versionchanged:: 3.5 + Modify ``.pyc`` filenames according to :pep:`488`. .. cmdoption:: -q @@ -344,7 +359,9 @@ Miscellaneous options Warning control. Python's warning machinery by default prints warning messages to :data:`sys.stderr`. A typical warning message has the following - form:: + form: + + .. code-block:: none file:line: category: message diff --git a/Doc/using/unix.rst b/Doc/using/unix.rst index ccdf84dcfa5edaa..8b392f8a56c4748 100644 --- a/Doc/using/unix.rst +++ b/Doc/using/unix.rst @@ -41,9 +41,11 @@ On FreeBSD and OpenBSD * FreeBSD users, to add the package use:: - pkg_add -r python + pkg install python3 + +* OpenBSD users, to add the package use:: -* OpenBSD users use:: + pkg_add -r python pkg_add ftp://ftp.openbsd.org/pub/OpenBSD/4.2/packages//python-.tgz @@ -116,7 +118,9 @@ Miscellaneous ============= To easily use Python scripts on Unix, you need to make them executable, -e.g. with :: +e.g. with + +.. code-block:: shell-session $ chmod +x script diff --git a/Doc/using/venv-create.inc b/Doc/using/venv-create.inc index 4292592ba7a25b8..4c7795ad801964f 100644 --- a/Doc/using/venv-create.inc +++ b/Doc/using/venv-create.inc @@ -9,7 +9,8 @@ with a ``home`` key pointing to the Python installation from which the command was run. It also creates a ``bin`` (or ``Scripts`` on Windows) subdirectory containing a copy of the ``python`` binary (or binaries, in the case of Windows). It also creates an (initially empty) ``lib/pythonX.Y/site-packages`` -subdirectory (on Windows, this is ``Lib\site-packages``). +subdirectory (on Windows, this is ``Lib\site-packages``). If an existing +directory is specified, it will be re-used. .. deprecated:: 3.6 ``pyvenv`` was the recommended tool for creating virtual environments for @@ -71,9 +72,7 @@ The command, if run with ``-h``, will show the available options:: .. versionchanged:: 3.4 In earlier versions, if the target directory already existed, an error was - raised, unless the ``--clear`` or ``--upgrade`` option was provided. Now, - if an existing directory is specified, its contents are removed and - the directory is processed as if it had been newly created. + raised, unless the ``--clear`` or ``--upgrade`` option was provided. The created ``pyvenv.cfg`` file also includes the ``include-system-site-packages`` key, set to ``true`` if ``venv`` is diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 47d423f42d6e175..3bab6fe503ca5a0 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -210,7 +210,9 @@ The options listed above can also be provided in a file named ``unattend.xml`` alongside the executable. This file specifies a list of options and values. When a value is provided as an attribute, it will be converted to a number if possible. Values provided as element text are always left as strings. This -example file sets the same options and the previous example:: +example file sets the same options and the previous example: + +.. code-block:: xml