diff --git a/Lib/http/client.py b/Lib/http/client.py index 6fb7d254ea9..7ef99e7201c 100644 --- a/Lib/http/client.py +++ b/Lib/http/client.py @@ -111,6 +111,13 @@ _MAXLINE = 65536 _MAXHEADERS = 100 +# maximal number of interim (1xx) responses tolerated before the final +# response. Real servers send at most a few; without a bound, a server +# streaming "100 Continue" responses would hang getresponse() forever. +# A socket timeout cannot detect that as data keeps arriving within every +# timeout window. +_MAXINTERIMRESPONSES = 100 + # Data larger than this will be read in chunks, to prevent extreme # overallocation. _MIN_READ_BUF_SIZE = 1 << 20 @@ -186,11 +193,10 @@ def _strip_ipv6_iface(enc_name: bytes) -> bytes: return enc_name class HTTPMessage(email.message.Message): - # XXX The only usage of this method is in - # http.server.CGIHTTPRequestHandler. Maybe move the code there so - # that it doesn't need to be part of the public API. The API has - # never been defined so this could cause backwards compatibility - # issues. + + # The getallmatchingheaders() method was only used by the CGI handler + # that was removed in Python 3.15. However, since the public API was not + # properly defined, it will be kept for backwards compatibility reasons. def getallmatchingheaders(self, name): """Find all header lines matching a given header name. @@ -215,22 +221,24 @@ def getallmatchingheaders(self, name): lst.append(line) return lst -def _read_headers(fp): +def _read_headers(fp, max_headers): """Reads potential header lines into a list from a file pointer. Length of line is limited by _MAXLINE, and number of - headers is limited by _MAXHEADERS. + headers is limited by max_headers. """ headers = [] + if max_headers is None: + max_headers = _MAXHEADERS while True: line = fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: raise LineTooLong("header line") - headers.append(line) - if len(headers) > _MAXHEADERS: - raise HTTPException("got more than %d headers" % _MAXHEADERS) if line in (b'\r\n', b'\n', b''): break + headers.append(line) + if len(headers) > max_headers: + raise HTTPException(f"got more than {max_headers} headers") return headers def _parse_header_lines(header_lines, _class=HTTPMessage): @@ -247,10 +255,10 @@ def _parse_header_lines(header_lines, _class=HTTPMessage): hstring = b''.join(header_lines).decode('iso-8859-1') return email.parser.Parser(_class=_class).parsestr(hstring) -def parse_headers(fp, _class=HTTPMessage): +def parse_headers(fp, _class=HTTPMessage, *, _max_headers=None): """Parses only RFC 5322 headers from a file pointer.""" - headers = _read_headers(fp) + headers = _read_headers(fp, _max_headers) return _parse_header_lines(headers, _class) @@ -292,6 +300,7 @@ def __init__(self, sock, debuglevel=0, method=None, url=None): self.chunk_left = _UNKNOWN # bytes left to read in current chunk self.length = _UNKNOWN # number of bytes left in response self.will_close = _UNKNOWN # conn will close at end of response + self._max_headers = None # configured header count limit def _read_status(self): line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1") @@ -326,21 +335,29 @@ def _read_status(self): raise BadStatusLine(line) return version, status, reason - def begin(self): + def begin(self, *, _max_headers=None): if self.headers is not None: # we've already started reading the response return + # Trailers of a chunked response are read by read() long after + # begin() returns, so remember the configured header count limit + # for _read_and_discard_trailer() to enforce. + self._max_headers = _max_headers + # read until we get a non-100 response - while True: + for _ in range(_MAXINTERIMRESPONSES): version, status, reason = self._read_status() if status != CONTINUE: break # skip the header from the 100 response - skipped_headers = _read_headers(self.fp) + skipped_headers = _read_headers(self.fp, _max_headers) if self.debuglevel > 0: print("headers:", skipped_headers) del skipped_headers + else: + raise HTTPException( + f"got more than {_MAXINTERIMRESPONSES} interim responses") self.code = self.status = status self.reason = reason.strip() @@ -352,7 +369,9 @@ def begin(self): else: raise UnknownProtocol(version) - self.headers = self.msg = parse_headers(self.fp) + self.headers = self.msg = parse_headers( + self.fp, _max_headers=_max_headers + ) if self.debuglevel > 0: for hdr, val in self.headers.items(): @@ -558,6 +577,10 @@ def _read_next_chunk_size(self): def _read_and_discard_trailer(self): # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! + max_trailers = self._max_headers + if max_trailers is None: + max_trailers = _MAXHEADERS + trailers_read = 0 while True: line = self.fp.readline(_MAXLINE + 1) if len(line) > _MAXLINE: @@ -568,6 +591,13 @@ def _read_and_discard_trailer(self): break if line in (b'\r\n', b'\n', b''): break + # Bound the trailer count just as response headers are bounded. + # A server streaming trailer lines forever would otherwise hang + # the client; a socket timeout cannot detect that as data keeps + # arriving within every timeout window. + trailers_read += 1 + if trailers_read > max_trailers: + raise HTTPException(f"got more than {max_trailers} trailers") def _get_chunk_left(self): # return self.chunk_left, reading a new chunk if necessary. @@ -885,7 +915,7 @@ def _get_content_length(body, method): return None def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, blocksize=8192): + source_address=None, blocksize=8192, *, max_response_headers=None): self.timeout = timeout self.source_address = source_address self.blocksize = blocksize @@ -898,6 +928,7 @@ def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, self._tunnel_port = None self._tunnel_headers = {} self._raw_proxy_headers = None + self.max_response_headers = max_response_headers (self.host, self.port) = self._get_hostport(host, port) @@ -999,7 +1030,7 @@ def _tunnel(self): try: (version, code, message) = response._read_status() - self._raw_proxy_headers = _read_headers(response.fp) + self._raw_proxy_headers = _read_headers(response.fp, self.max_response_headers) if self.debuglevel > 0: for header in self._raw_proxy_headers: @@ -1416,8 +1447,7 @@ def getresponse(self): """Get the response from the server. If the HTTPConnection is in the correct state, returns an - instance of HTTPResponse or of whatever object is returned by - the response_class variable. + instance of HTTPResponse. If a request has not been sent or if a previous response has not be handled, ResponseNotReady is raised. If the HTTP @@ -1456,7 +1486,10 @@ def getresponse(self): try: try: - response.begin() + if self.max_response_headers is None: + response.begin() + else: + response.begin(_max_headers=self.max_response_headers) except ConnectionError: self.close() raise @@ -1487,10 +1520,12 @@ class HTTPSConnection(HTTPConnection): def __init__(self, host, port=None, *, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, context=None, blocksize=8192): + source_address=None, context=None, blocksize=8192, + max_response_headers=None): super(HTTPSConnection, self).__init__(host, port, timeout, source_address, - blocksize=blocksize) + blocksize=blocksize, + max_response_headers=max_response_headers) if context is None: context = _create_https_context(self._http_vsn) self._context = context diff --git a/Lib/http/cookiejar.py b/Lib/http/cookiejar.py index 68cf16c93cc..13e5b104a81 100644 --- a/Lib/http/cookiejar.py +++ b/Lib/http/cookiejar.py @@ -1032,10 +1032,13 @@ def set_ok_domain(self, cookie, request): if j == 0: # domain like .foo.bar tld = domain[i+1:] sld = domain[j+1:i] - if sld.lower() in ("co", "ac", "com", "edu", "org", "net", - "gov", "mil", "int", "aero", "biz", "cat", "coop", - "info", "jobs", "mobi", "museum", "name", "pro", - "travel", "eu") and len(tld) == 2: + known_slds = ( + "co", "ac", "com", "edu", "org", "net", + "gov", "mil", "int", "aero", "biz", "cat", "coop", + "info", "jobs", "mobi", "museum", "name", "pro", + "travel", "eu", "tv", "or", "nom", "sch", "web", + ) + if sld.lower() in known_slds and len(tld) == 2: # domain like .co.uk _debug(" country-code second level domain %s", domain) return False diff --git a/Lib/http/cookies.py b/Lib/http/cookies.py index abebb4b69fd..9a6f01dfb5e 100644 --- a/Lib/http/cookies.py +++ b/Lib/http/cookies.py @@ -132,6 +132,7 @@ import re import string import types +lazy import warnings __all__ = ["CookieError", "BaseCookie", "SimpleCookie"] @@ -390,7 +391,8 @@ def output(self, attrs=None, header="Set-Cookie:"): def __repr__(self): return '<%s: %s>' % (self.__class__.__name__, self.OutputString()) - def js_output(self, attrs=None): + def _js_output(self, attrs=None): + """Internal implementation without deprecation warning.""" import urllib.parse # Print javascript output_string = self.OutputString(attrs) @@ -407,6 +409,14 @@ def js_output(self, attrs=None): """ % (output_encoded,) + def js_output(self, attrs=None): + warnings._deprecated( + "http.cookies.Morsel.js_output", + message=warnings._DEPRECATED_MSG + "; use output() instead", + remove=(3, 19), + ) + return self._js_output(attrs) + def OutputString(self, attrs=None): # Build up our result # @@ -541,10 +551,15 @@ def __repr__(self): def js_output(self, attrs=None): """Return a string suitable for JavaScript.""" + warnings._deprecated( + "http.cookies.BaseCookie.js_output", + message=warnings._DEPRECATED_MSG + "; use output() instead", + remove=(3, 19), + ) result = [] items = sorted(self.items()) for key, value in items: - result.append(value.js_output(attrs)) + result.append(value._js_output(attrs)) return _nulljoin(result) def load(self, rawdata): diff --git a/Lib/http/server.py b/Lib/http/server.py index ac1f57c29f0..095b5744bd1 100644 --- a/Lib/http/server.py +++ b/Lib/http/server.py @@ -1,29 +1,10 @@ """HTTP server classes. Note: BaseHTTPRequestHandler doesn't implement any HTTP request; see -SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST, -and (deprecated) CGIHTTPRequestHandler for CGI scripts. +SimpleHTTPRequestHandler for simple implementations of GET, HEAD and POST. It does, however, optionally implement HTTP/1.1 persistent connections. -Notes on CGIHTTPRequestHandler ------------------------------- - -This class is deprecated. It implements GET and POST requests to cgi-bin scripts. - -If the os.fork() function is not present (Windows), subprocess.Popen() is used, -with slightly altered but never documented semantics. Use from a threaded -process is likely to trigger a warning at os.fork() time. - -In all cases, the implementation is intentionally naive -- all -requests are executed synchronously. - -SECURITY WARNING: DON'T USE THIS CODE UNLESS YOU ARE INSIDE A FIREWALL --- it may execute arbitrary Python code or external programs. - -Note that status code 200 is sent prior to execution of a CGI script, so -scripts cannot send other status codes such as 302 (redirect). - XXX To do: - log requests even later (to capture byte count) @@ -80,16 +61,12 @@ # (Actually, the latter is only true if you know the server configuration # at the time the request was made!) -__version__ = "0.6" - __all__ = [ "HTTPServer", "ThreadingHTTPServer", "HTTPSServer", "ThreadingHTTPSServer", "BaseHTTPRequestHandler", "SimpleHTTPRequestHandler", - "CGIHTTPRequestHandler", ] -import copy import datetime import email.utils import html @@ -99,7 +76,6 @@ import mimetypes import os import posixpath -import select import shutil import socket import socketserver @@ -109,6 +85,8 @@ from http import HTTPStatus +lazy import _colorize + # Default error message template DEFAULT_ERROR_MESSAGE = """\ @@ -134,10 +112,6 @@ DEFAULT_ERROR_CONTENT_TYPE = "text/html;charset=utf-8" -# Data larger than this will be read in chunks, to prevent extreme -# overallocation. -_MIN_READ_BUF_SIZE = 1 << 20 - class HTTPServer(socketserver.TCPServer): allow_reuse_address = True # Seems to make sense in testing environment @@ -306,7 +280,7 @@ class BaseHTTPRequestHandler(socketserver.StreamRequestHandler): # The server software version. You may want to override this. # The format is multiple whitespace-separated strings, # where each string is of the form name[/version]. - server_version = "BaseHTTP/" + __version__ + server_version = "BaseHTTP" error_message_format = DEFAULT_ERROR_MESSAGE error_content_type = DEFAULT_ERROR_CONTENT_TYPE @@ -360,10 +334,13 @@ def parse_request(self): raise ValueError("unreasonable length http version") version_number = int(version_number[0]), int(version_number[1]) except (ValueError, IndexError): + # Send the error response with a status line and headers. + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad request version (%r)" % version) return False + self.request_version = version if version_number >= (1, 1) and self.protocol_version >= "HTTP/1.1": self.close_connection = False if version_number >= (2, 0): @@ -371,9 +348,9 @@ def parse_request(self): HTTPStatus.HTTP_VERSION_NOT_SUPPORTED, "Invalid HTTP version (%s)" % base_version_number) return False - self.request_version = version if not 2 <= len(words) <= 3: + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad request syntax (%r)" % requestline) @@ -382,6 +359,7 @@ def parse_request(self): if len(words) == 2: self.close_connection = True if command != 'GET': + self.request_version = '' self.send_error( HTTPStatus.BAD_REQUEST, "Bad HTTP/0.9 request type (%r)" % command) @@ -577,13 +555,17 @@ def send_response_only(self, code, message=None): (self.protocol_version, code, message)).encode( 'latin-1', 'strict')) - def send_header(self, keyword, value): + def send_header(self, keyword, value, *, _is_extra=False): """Send a MIME header to the headers buffer.""" if self.request_version != 'HTTP/0.9': if not hasattr(self, '_headers_buffer'): self._headers_buffer = [] self._headers_buffer.append( ("%s: %s\r\n" % (keyword, value)).encode('latin-1', 'strict')) + if not hasattr(self, '_default_response_headers'): + self._default_response_headers = [] + if not _is_extra: + self._default_response_headers.append((keyword, value)) if keyword.lower() == 'connection': if value.lower() == 'close': @@ -601,6 +583,33 @@ def flush_headers(self): if hasattr(self, '_headers_buffer'): self.wfile.write(b"".join(self._headers_buffer)) self._headers_buffer = [] + if hasattr(self, '_default_response_headers'): + self._default_response_headers = [] + + def _colorize_request(self, code, size, t): + try: + code_int = int(code) + except (TypeError, ValueError): + code_color = "" + else: + if code_int >= 500: + code_color = t.status_server_error + elif code_int >= 400: + code_color = t.status_client_error + elif code_int >= 300: + code_color = t.status_redirect + elif code_int >= 200: + code_color = t.status_ok + else: + code_color = t.status_informational + + request_line = self.requestline.translate(self._control_char_table) + parts = request_line.split(None, 2) + if len(parts) == 3: + method, path, version = parts + request_line = f"{method} {t.path}{path}{t.reset} {version}" + + return f'"{request_line}" {code_color}{code} {t.size}{size}{t.reset}' def log_request(self, code='-', size='-'): """Log an accepted request. @@ -610,6 +619,7 @@ def log_request(self, code='-', size='-'): """ if isinstance(code, HTTPStatus): code = code.value + self._log_request_info = (code, size) self.log_message('"%s" %s %s', self.requestline, str(code), str(size)) @@ -624,7 +634,7 @@ def log_error(self, format, *args): XXX This should go to the separate error log. """ - + self._log_is_error = True self.log_message(format, *args) # https://en.wikipedia.org/wiki/List_of_Unicode_characters#Control_codes @@ -651,12 +661,22 @@ def log_message(self, format, *args): before writing the output to stderr. """ - - message = format % args - sys.stderr.write("%s - - [%s] %s\n" % - (self.address_string(), - self.log_date_time_string(), - message.translate(self._control_char_table))) + message = (format % args).translate(self._control_char_table) + t = _colorize.get_theme(tty_file=sys.stderr).http_server + + info = getattr(self, "_log_request_info", None) + if info is not None: + self._log_request_info = None + message = self._colorize_request(*info, t) + elif getattr(self, "_log_is_error", False): + self._log_is_error = False + message = f"{t.error}{message}{t.reset}" + + sys.stderr.write( + f"{t.timestamp}{self.address_string()} - - " + f"[{self.log_date_time_string()}]{t.reset} " + f"{message}\n" + ) def version_string(self): """Return the server software version string.""" @@ -716,7 +736,8 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): """ - server_version = "SimpleHTTP/" + __version__ + server_version = "SimpleHTTP" + default_content_type = "application/octet-stream" index_pages = ("index.html", "index.htm") extensions_map = _encodings_map_default = { '.gz': 'application/gzip', @@ -725,10 +746,11 @@ class SimpleHTTPRequestHandler(BaseHTTPRequestHandler): '.xz': 'application/x-xz', } - def __init__(self, *args, directory=None, **kwargs): + def __init__(self, *args, directory=None, extra_response_headers=None, **kwargs): if directory is None: directory = os.getcwd() self.directory = os.fspath(directory) + self.extra_response_headers = extra_response_headers super().__init__(*args, **kwargs) def do_GET(self): @@ -746,6 +768,16 @@ def do_HEAD(self): if f: f.close() + def _send_extra_response_headers(self): + """Send the headers stored in self.extra_response_headers.""" + if self.extra_response_headers is not None: + default_headers = {h.lower() for h, _ in self._default_response_headers} + for header, value in self.extra_response_headers: + # Don't send the header if it's already sent + # as part of the default response headers + if header.lower() not in default_headers: + self.send_header(header, value, _is_extra=True) + def send_head(self): """Common code for GET and HEAD commands. @@ -828,6 +860,7 @@ def send_head(self): self.send_header("Content-Length", str(fs[6])) self.send_header("Last-Modified", self.date_time_string(fs.st_mtime)) + self._send_extra_response_headers() self.end_headers() return f except: @@ -892,6 +925,7 @@ def list_directory(self, path): self.send_response(HTTPStatus.OK) self.send_header("Content-type", "text/html; charset=%s" % enc) self.send_header("Content-Length", str(len(encoded))) + self._send_extra_response_headers() self.end_headers() return f @@ -964,57 +998,7 @@ def guess_type(self, path): guess, _ = mimetypes.guess_file_type(path) if guess: return guess - return 'application/octet-stream' - - -# Utilities for CGIHTTPRequestHandler - -def _url_collapse_path(path): - """ - Given a URL path, remove extra '/'s and '.' path elements and collapse - any '..' references and returns a collapsed path. - - Implements something akin to RFC-2396 5.2 step 6 to parse relative paths. - The utility of this function is limited to is_cgi method and helps - preventing some security attacks. - - Returns: The reconstituted URL, which will always start with a '/'. - - Raises: IndexError if too many '..' occur within the path. - - """ - # Query component should not be involved. - path, _, query = path.partition('?') - path = urllib.parse.unquote(path) - - # Similar to os.path.split(os.path.normpath(path)) but specific to URL - # path semantics rather than local operating system semantics. - path_parts = path.split('/') - head_parts = [] - for part in path_parts[:-1]: - if part == '..': - head_parts.pop() # IndexError if more '..' than prior parts - elif part and part != '.': - head_parts.append( part ) - if path_parts: - tail_part = path_parts.pop() - if tail_part: - if tail_part == '..': - head_parts.pop() - tail_part = '' - elif tail_part == '.': - tail_part = '' - else: - tail_part = '' - - if query: - tail_part = '?'.join((tail_part, query)) - - splitpath = ('/' + '/'.join(head_parts), tail_part) - collapsed_path = "/".join(splitpath) - - return collapsed_path - + return self.default_content_type nobody = None @@ -1040,285 +1024,6 @@ def executable(path): return os.access(path, os.X_OK) -class CGIHTTPRequestHandler(SimpleHTTPRequestHandler): - - """Complete HTTP server with GET, HEAD and POST commands. - - GET and HEAD also support running CGI scripts. - - The POST command is *only* implemented for CGI scripts. - - """ - - def __init__(self, *args, **kwargs): - import warnings - warnings._deprecated("http.server.CGIHTTPRequestHandler", - remove=(3, 15)) - super().__init__(*args, **kwargs) - - # Determine platform specifics - have_fork = hasattr(os, 'fork') - - # Make rfile unbuffered -- we need to read one line and then pass - # the rest to a subprocess, so we can't use buffered input. - rbufsize = 0 - - def do_POST(self): - """Serve a POST request. - - This is only implemented for CGI scripts. - - """ - - if self.is_cgi(): - self.run_cgi() - else: - self.send_error( - HTTPStatus.NOT_IMPLEMENTED, - "Can only POST to CGI scripts") - - def send_head(self): - """Version of send_head that support CGI scripts""" - if self.is_cgi(): - return self.run_cgi() - else: - return SimpleHTTPRequestHandler.send_head(self) - - def is_cgi(self): - """Test whether self.path corresponds to a CGI script. - - Returns True and updates the cgi_info attribute to the tuple - (dir, rest) if self.path requires running a CGI script. - Returns False otherwise. - - If any exception is raised, the caller should assume that - self.path was rejected as invalid and act accordingly. - - The default implementation tests whether the normalized url - path begins with one of the strings in self.cgi_directories - (and the next character is a '/' or the end of the string). - - """ - collapsed_path = _url_collapse_path(self.path) - dir_sep = collapsed_path.find('/', 1) - while dir_sep > 0 and not collapsed_path[:dir_sep] in self.cgi_directories: - dir_sep = collapsed_path.find('/', dir_sep+1) - if dir_sep > 0: - head, tail = collapsed_path[:dir_sep], collapsed_path[dir_sep+1:] - self.cgi_info = head, tail - return True - return False - - - cgi_directories = ['/cgi-bin', '/htbin'] - - def is_executable(self, path): - """Test whether argument path is an executable file.""" - return executable(path) - - def is_python(self, path): - """Test whether argument path is a Python script.""" - head, tail = os.path.splitext(path) - return tail.lower() in (".py", ".pyw") - - def run_cgi(self): - """Execute a CGI script.""" - dir, rest = self.cgi_info - path = dir + '/' + rest - i = path.find('/', len(dir)+1) - while i >= 0: - nextdir = path[:i] - nextrest = path[i+1:] - - scriptdir = self.translate_path(nextdir) - if os.path.isdir(scriptdir): - dir, rest = nextdir, nextrest - i = path.find('/', len(dir)+1) - else: - break - - # find an explicit query string, if present. - rest, _, query = rest.partition('?') - - # dissect the part after the directory name into a script name & - # a possible additional path, to be stored in PATH_INFO. - i = rest.find('/') - if i >= 0: - script, rest = rest[:i], rest[i:] - else: - script, rest = rest, '' - - scriptname = dir + '/' + script - scriptfile = self.translate_path(scriptname) - if not os.path.exists(scriptfile): - self.send_error( - HTTPStatus.NOT_FOUND, - "No such CGI script (%r)" % scriptname) - return - if not os.path.isfile(scriptfile): - self.send_error( - HTTPStatus.FORBIDDEN, - "CGI script is not a plain file (%r)" % scriptname) - return - ispy = self.is_python(scriptname) - if self.have_fork or not ispy: - if not self.is_executable(scriptfile): - self.send_error( - HTTPStatus.FORBIDDEN, - "CGI script is not executable (%r)" % scriptname) - return - - # Reference: https://www6.uniovi.es/~antonio/ncsa_httpd/cgi/env.html - # XXX Much of the following could be prepared ahead of time! - env = copy.deepcopy(os.environ) - env['SERVER_SOFTWARE'] = self.version_string() - env['SERVER_NAME'] = self.server.server_name - env['GATEWAY_INTERFACE'] = 'CGI/1.1' - env['SERVER_PROTOCOL'] = self.protocol_version - env['SERVER_PORT'] = str(self.server.server_port) - env['REQUEST_METHOD'] = self.command - uqrest = urllib.parse.unquote(rest) - env['PATH_INFO'] = uqrest - env['PATH_TRANSLATED'] = self.translate_path(uqrest) - env['SCRIPT_NAME'] = scriptname - env['QUERY_STRING'] = query - env['REMOTE_ADDR'] = self.client_address[0] - authorization = self.headers.get("authorization") - if authorization: - authorization = authorization.split() - if len(authorization) == 2: - import base64, binascii - env['AUTH_TYPE'] = authorization[0] - if authorization[0].lower() == "basic": - try: - authorization = authorization[1].encode('ascii') - authorization = base64.decodebytes(authorization).\ - decode('ascii') - except (binascii.Error, UnicodeError): - pass - else: - authorization = authorization.split(':') - if len(authorization) == 2: - env['REMOTE_USER'] = authorization[0] - # XXX REMOTE_IDENT - if self.headers.get('content-type') is None: - env['CONTENT_TYPE'] = self.headers.get_content_type() - else: - env['CONTENT_TYPE'] = self.headers['content-type'] - length = self.headers.get('content-length') - if length: - env['CONTENT_LENGTH'] = length - referer = self.headers.get('referer') - if referer: - env['HTTP_REFERER'] = referer - accept = self.headers.get_all('accept', ()) - env['HTTP_ACCEPT'] = ','.join(accept) - ua = self.headers.get('user-agent') - if ua: - env['HTTP_USER_AGENT'] = ua - co = filter(None, self.headers.get_all('cookie', [])) - cookie_str = ', '.join(co) - if cookie_str: - env['HTTP_COOKIE'] = cookie_str - # XXX Other HTTP_* headers - # Since we're setting the env in the parent, provide empty - # values to override previously set values - for k in ('QUERY_STRING', 'REMOTE_HOST', 'CONTENT_LENGTH', - 'HTTP_USER_AGENT', 'HTTP_COOKIE', 'HTTP_REFERER'): - env.setdefault(k, "") - - self.send_response(HTTPStatus.OK, "Script output follows") - self.flush_headers() - - decoded_query = query.replace('+', ' ') - - if self.have_fork: - # Unix -- fork as we should - args = [script] - if '=' not in decoded_query: - args.append(decoded_query) - nobody = nobody_uid() - self.wfile.flush() # Always flush before forking - pid = os.fork() - if pid != 0: - # Parent - pid, sts = os.waitpid(pid, 0) - # throw away additional data [see bug #427345] - while select.select([self.rfile], [], [], 0)[0]: - if not self.rfile.read(1): - break - exitcode = os.waitstatus_to_exitcode(sts) - if exitcode: - self.log_error(f"CGI script exit code {exitcode}") - return - # Child - try: - try: - os.setuid(nobody) - except OSError: - pass - os.dup2(self.rfile.fileno(), 0) - os.dup2(self.wfile.fileno(), 1) - os.execve(scriptfile, args, env) - except: - self.server.handle_error(self.request, self.client_address) - os._exit(127) - - else: - # Non-Unix -- use subprocess - import subprocess - cmdline = [scriptfile] - if self.is_python(scriptfile): - interp = sys.executable - if interp.lower().endswith("w.exe"): - # On Windows, use python.exe, not pythonw.exe - interp = interp[:-5] + interp[-4:] - cmdline = [interp, '-u'] + cmdline - if '=' not in query: - cmdline.append(query) - self.log_message("command: %s", subprocess.list2cmdline(cmdline)) - try: - nbytes = int(length) - except (TypeError, ValueError): - nbytes = 0 - p = subprocess.Popen(cmdline, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env = env - ) - if self.command.lower() == "post" and nbytes > 0: - cursize = 0 - data = self.rfile.read(min(nbytes, _MIN_READ_BUF_SIZE)) - while len(data) < nbytes and len(data) != cursize: - cursize = len(data) - # This is a geometric increase in read size (never more - # than doubling out the current length of data per loop - # iteration). - delta = min(cursize, nbytes - cursize) - try: - data += self.rfile.read(delta) - except TimeoutError: - break - else: - data = None - # throw away additional data [see bug #427345] - while select.select([self.rfile._sock], [], [], 0)[0]: - if not self.rfile._sock.recv(1): - break - stdout, stderr = p.communicate(data) - self.wfile.write(stdout) - if stderr: - self.log_error('%s', stderr) - p.stderr.close() - p.stdout.close() - status = p.returncode - if status: - self.log_error("CGI script exit status %#x", status) - else: - self.log_message("CGI script exited OK") - - def _get_best_family(*address): infos = socket.getaddrinfo( *address, @@ -1329,31 +1034,45 @@ def _get_best_family(*address): return family, sockaddr -def test(HandlerClass=BaseHTTPRequestHandler, +def _make_server(HandlerClass=BaseHTTPRequestHandler, + ServerClass=ThreadingHTTPServer, + protocol="HTTP/1.0", port=8000, bind=None, + tls_cert=None, tls_key=None, tls_password=None, + default_content_type=SimpleHTTPRequestHandler.default_content_type): + ServerClass.address_family, addr = _get_best_family(bind, port) + HandlerClass.protocol_version = protocol + HandlerClass.default_content_type = default_content_type + + if tls_cert: + return ServerClass(addr, HandlerClass, certfile=tls_cert, + keyfile=tls_key, password=tls_password) + else: + return ServerClass(addr, HandlerClass) + + +def test(HandlerClass=SimpleHTTPRequestHandler, ServerClass=ThreadingHTTPServer, protocol="HTTP/1.0", port=8000, bind=None, + content_type=SimpleHTTPRequestHandler.default_content_type, tls_cert=None, tls_key=None, tls_password=None): """Test the HTTP request handler class. This runs an HTTP server on port 8000 (or the port argument). - """ - ServerClass.address_family, addr = _get_best_family(bind, port) - HandlerClass.protocol_version = protocol - - if tls_cert: - server = ServerClass(addr, HandlerClass, certfile=tls_cert, - keyfile=tls_key, password=tls_password) - else: - server = ServerClass(addr, HandlerClass) - - with server as httpd: + with _make_server( + HandlerClass=HandlerClass, ServerClass=ServerClass, + protocol=protocol, port=port, bind=bind, + tls_cert=tls_cert, tls_key=tls_key, tls_password=tls_password, + default_content_type=content_type, + ) as httpd: host, port = httpd.socket.getsockname()[:2] url_host = f'[{host}]' if ':' in host else host protocol = 'HTTPS' if tls_cert else 'HTTP' + t = _colorize.get_theme().http_server + url = f"{protocol.lower()}://{url_host}:{port}/" print( - f"Serving {protocol} on {host} port {port} " - f"({protocol.lower()}://{url_host}:{port}/) ..." + f"{t.serving}Serving {protocol} on {host} port {port}{t.reset} " + f"({t.url}{url}{t.reset}) ..." ) try: httpd.serve_forever() @@ -1361,13 +1080,12 @@ def test(HandlerClass=BaseHTTPRequestHandler, print("\nKeyboard interrupt received, exiting.") sys.exit(0) -if __name__ == '__main__': + +def _main(args=None): import argparse import contextlib - parser = argparse.ArgumentParser(color=True) - parser.add_argument('--cgi', action='store_true', - help='run as CGI server') + parser = argparse.ArgumentParser() parser.add_argument('-b', '--bind', metavar='ADDRESS', help='bind to this address ' '(default: all interfaces)') @@ -1378,6 +1096,10 @@ def test(HandlerClass=BaseHTTPRequestHandler, default='HTTP/1.0', help='conform to this HTTP version ' '(default: %(default)s)') + parser.add_argument('--content-type', + default=SimpleHTTPRequestHandler.default_content_type, + help='default content type for unknown extensions ' + '(default: %(default)s)') parser.add_argument('--tls-cert', metavar='PATH', help='path to the TLS certificate chain file') parser.add_argument('--tls-key', metavar='PATH', @@ -1387,7 +1109,11 @@ def test(HandlerClass=BaseHTTPRequestHandler, parser.add_argument('port', default=8000, type=int, nargs='?', help='bind to this port ' '(default: %(default)s)') - args = parser.parse_args() + parser.add_argument('-H', '--header', nargs=2, action='append', + metavar=('HEADER', 'VALUE'), + help='Add a custom response header ' + '(can be specified multiple times)') + args = parser.parse_args(args) if not args.tls_cert and args.tls_key: parser.error("--tls-key requires --tls-cert to be set") @@ -1403,11 +1129,6 @@ def test(HandlerClass=BaseHTTPRequestHandler, except OSError as e: parser.error(f"Failed to read TLS password file: {e}") - if args.cgi: - handler_class = CGIHTTPRequestHandler - else: - handler_class = SimpleHTTPRequestHandler - # ensure dual-stack is not disabled; ref #38907 class DualStackServerMixin: @@ -1420,7 +1141,8 @@ def server_bind(self): def finish_request(self, request, client_address): self.RequestHandlerClass(request, client_address, self, - directory=args.directory) + directory=args.directory, + extra_response_headers=args.header) class HTTPDualStackServer(DualStackServerMixin, ThreadingHTTPServer): pass @@ -1430,12 +1152,26 @@ class HTTPSDualStackServer(DualStackServerMixin, ThreadingHTTPSServer): ServerClass = HTTPSDualStackServer if args.tls_cert else HTTPDualStackServer test( - HandlerClass=handler_class, + HandlerClass=SimpleHTTPRequestHandler, ServerClass=ServerClass, port=args.port, bind=args.bind, protocol=args.protocol, + content_type=args.content_type, tls_cert=args.tls_cert, tls_key=args.tls_key, tls_password=tls_key_password, ) + + +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "0.6" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +if __name__ == '__main__': + _main() diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py index 8772a66791a..11c2a4b0ef4 100644 --- a/Lib/pkgutil.py +++ b/Lib/pkgutil.py @@ -9,6 +9,9 @@ import os.path import sys +lazy import re + + __all__ = [ 'get_importer', 'iter_importers', 'walk_packages', 'iter_modules', 'get_data', @@ -398,9 +401,10 @@ def get_data(package, resource): return loader.get_data(resource_name) -_NAME_PATTERN = None +_LENIENT_PATTERN = None +_STRICT_PATTERN = None -def resolve_name(name): +def resolve_name(name, *, strict=False): """ Resolve a name to an object. @@ -410,6 +414,7 @@ def resolve_name(name): W(.W)* W(.W)*:(W(.W)*)? + W(.W)*:(W(.W)*) The first form is intended for backward compatibility only. It assumes that some part of the dotted name is a package, and the rest is an object @@ -424,6 +429,11 @@ def resolve_name(name): hierarchy within that package. Only one import is needed in this form. If it ends with the colon, then a module object is returned. + The first two forms are accepted when `strict=False` (the default). + + The third form requires both the module name and callable, separated by + a colon. Only this form is accepted when `strict=True`. + The function will return an object (which might be a module), or raise one of the following exceptions: @@ -432,18 +442,26 @@ def resolve_name(name): AttributeError - if a failure occurred when traversing the object hierarchy within the imported package to get to the desired object. """ - global _NAME_PATTERN - if _NAME_PATTERN is None: - # Lazy import to speedup Python startup time - import re - dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' - _NAME_PATTERN = re.compile(f'^(?P{dotted_words})' - f'(?P:(?P{dotted_words})?)?$', - re.UNICODE) - - m = _NAME_PATTERN.match(name) - if not m: + global _LENIENT_PATTERN, _STRICT_PATTERN + dotted_words = r'(?!\d)(\w+)(\.(?!\d)(\w+))*' + if strict: + if _STRICT_PATTERN is None: + _STRICT_PATTERN = re.compile( + f'^(?P{dotted_words})' + f'(?P:(?P{dotted_words}))$', + re.UNICODE) + pattern = _STRICT_PATTERN + else: + if _LENIENT_PATTERN is None: + _LENIENT_PATTERN = re.compile( + f'^(?P{dotted_words})' + f'(?P:(?P{dotted_words})?)?$', + re.UNICODE) + pattern = _LENIENT_PATTERN + + if (m := pattern.match(name)) is None: raise ValueError(f'invalid format: {name!r}') + gd = m.groupdict() if gd.get('cln'): # there is a colon - a one-step import is all that's needed diff --git a/Lib/pydoc.py b/Lib/pydoc.py index 091f8af01b2..aa7f301dddb 100644 --- a/Lib/pydoc.py +++ b/Lib/pydoc.py @@ -78,20 +78,41 @@ class or function within a module or module in a package. If the from reprlib import Repr from traceback import format_exception_only -from _pyrepl.pager import (get_pager, pipe_pager, - plain_pager, tempfile_pager, tty_pager) +try: + from _pyrepl.pager import (get_pager, pipe_pager, + plain_pager, tempfile_pager, tty_pager) -# Expose plain() as pydoc.plain() -from _pyrepl.pager import plain # noqa: F401 + # Expose plain() as pydoc.plain() + from _pyrepl.pager import plain # noqa: F401 + # --------------------------------------------------------- old names + getpager = get_pager + pipepager = pipe_pager + plainpager = plain_pager + tempfilepager = tempfile_pager + ttypager = tty_pager -# --------------------------------------------------------- old names +except ModuleNotFoundError: + # Minimal alternatives for cases where _pyrepl is absent. -getpager = get_pager -pipepager = pipe_pager -plainpager = plain_pager -tempfilepager = tempfile_pager -ttypager = tty_pager + def plain(text: str) -> str: + """Remove boldface formatting from text.""" + return re.sub('.\b', '', text) + + def plain_pager(text: str, title: str = '') -> None: + """Simply print unformatted text. This is the ultimate fallback.""" + encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8' + text = text.encode(encoding, 'backslashreplace').decode(encoding) + text = plain(text) + sys.stdout.write(text) + + def get_pager(): + """Unconditionally return the plain pager, since _pyrepl is absent.""" + return plain_pager + + # --------------------------------------------------------- old names + getpager = get_pager + plainpager = plain_pager # --------------------------------------------------------- common routines @@ -108,96 +129,10 @@ def pathdirs(): normdirs.append(normdir) return dirs -def _findclass(func): - cls = sys.modules.get(func.__module__) - if cls is None: - return None - for name in func.__qualname__.split('.')[:-1]: - cls = getattr(cls, name) - if not inspect.isclass(cls): - return None - return cls - -def _finddoc(obj): - if inspect.ismethod(obj): - name = obj.__func__.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - getattr(getattr(self, name, None), '__func__') is obj.__func__): - # classmethod - cls = self - else: - cls = self.__class__ - elif inspect.isfunction(obj): - name = obj.__name__ - cls = _findclass(obj) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.isbuiltin(obj): - name = obj.__name__ - self = obj.__self__ - if (inspect.isclass(self) and - self.__qualname__ + '.' + name == obj.__qualname__): - # classmethod - cls = self - else: - cls = self.__class__ - # Should be tested before isdatadescriptor(). - elif isinstance(obj, property): - name = obj.__name__ - cls = _findclass(obj.fget) - if cls is None or getattr(cls, name) is not obj: - return None - elif inspect.ismethoddescriptor(obj) or inspect.isdatadescriptor(obj): - name = obj.__name__ - cls = obj.__objclass__ - if getattr(cls, name) is not obj: - return None - if inspect.ismemberdescriptor(obj): - slots = getattr(cls, '__slots__', None) - if isinstance(slots, dict) and name in slots: - return slots[name] - else: - return None - for base in cls.__mro__: - try: - doc = _getowndoc(getattr(base, name)) - except AttributeError: - continue - if doc is not None: - return doc - return None - -def _getowndoc(obj): - """Get the documentation string for an object if it is not - inherited from its class.""" - try: - doc = object.__getattribute__(obj, '__doc__') - if doc is None: - return None - if obj is not type: - typedoc = type(obj).__doc__ - if isinstance(typedoc, str) and typedoc == doc: - return None - return doc - except AttributeError: - return None - def _getdoc(object): - """Get the documentation string for an object. - - All tabs are expanded to spaces. To clean up docstrings that are - indented to line up with blocks of code, any whitespace than can be - uniformly removed from the second line onwards is removed.""" - doc = _getowndoc(object) - if doc is None: - try: - doc = _finddoc(object) - except (AttributeError, TypeError): - return None - if not isinstance(doc, str): - return None - return inspect.cleandoc(doc) + return inspect.getdoc(object, + fallback_to_class_doc=False, + inherit_class_doc=False) def getdoc(object): """Get the doc string or comments for an object.""" @@ -327,12 +262,12 @@ def visiblename(name, all=None, obj=None): """Decide whether to show documentation on a variable.""" # Certain special names are redundant or internal. # XXX Remove __initializing__? - if name in {'__author__', '__builtins__', '__cached__', '__credits__', - '__date__', '__doc__', '__file__', '__spec__', - '__loader__', '__module__', '__name__', '__package__', - '__path__', '__qualname__', '__slots__', '__version__', - '__static_attributes__', '__firstlineno__', - '__annotate_func__', '__annotations_cache__'}: + if name in {'__author__', '__builtins__', '__credits__', '__date__', + '__doc__', '__file__', '__spec__', '__loader__', '__module__', + '__name__', '__package__', '__path__', '__qualname__', + '__slots__', '__version__', '__static_attributes__', + '__firstlineno__', '__annotate_func__', + '__annotations_cache__'}: return 0 # Private names are hidden, but special names are displayed. if name.startswith('__') and name.endswith('__'): return 1 @@ -536,6 +471,13 @@ class Doc: PYTHONDOCS = os.environ.get("PYTHONDOCS", "https://docs.python.org/%d.%d/library" % sys.version_info[:2]) + STDLIB_DIR = sysconfig.get_path('stdlib') + # TODO: RUSTPYTHON; Cargo builds keep Lib outside sysconfig's install paths. + if (sys.implementation.name == 'rustpython' + and getattr(sys, '_is_python_build', False) + and not sysconfig.is_python_build() + and getattr(sys, '_stdlib_dir', None)): + STDLIB_DIR = sys._stdlib_dir def document(self, object, name=None, *args): """Generate documentation for an object.""" @@ -561,23 +503,12 @@ def fail(self, object, name=None, *args): docmodule = docclass = docroutine = docother = docproperty = docdata = fail - def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): + def getdocloc(self, object, basedir=None): """Return the location of module docs or None""" - - try: - file = inspect.getabsfile(object) - except TypeError: - file = '(built-in)' - + basedir = self.STDLIB_DIR if basedir is None else basedir docloc = os.environ.get("PYTHONDOCS", self.PYTHONDOCS) - basedir = os.path.normcase(basedir) - if (isinstance(object, type(os)) and - (object.__name__ in ('errno', 'exceptions', 'gc', - 'marshal', 'posix', 'signal', 'sys', - '_thread', 'zipimport') or - (file.startswith(basedir) and - not file.startswith(os.path.join(basedir, 'site-packages')))) and + if (self._is_stdlib_module(object, basedir) and object.__name__ not in ('xml.etree', 'test.test_pydoc.pydoc_mod')): try: @@ -597,6 +528,36 @@ def getdocloc(self, object, basedir=sysconfig.get_path('stdlib')): docloc = None return docloc + def _get_version(self, object): + if self._is_stdlib_module(object): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + version = getattr(object, '__version__', None) + else: + version = getattr(object, '__version__', None) + return '' if version is None else str(version) + + def _is_stdlib_module(self, object, basedir=None): + basedir = self.STDLIB_DIR if basedir is None else basedir + + try: + file = inspect.getabsfile(object) + except TypeError: + file = '(built-in)' + + if sysconfig.is_python_build(): + srcdir = sysconfig.get_config_var('srcdir') + if srcdir: + basedir = os.path.join(srcdir, 'Lib') + + basedir = os.path.normcase(basedir) + return (isinstance(object, type(os)) and + (object.__name__ in ('errno', 'exceptions', 'gc', + 'marshal', 'posix', 'signal', 'sys', + '_thread', 'zipimport') + or (file.startswith(basedir) and + not file.startswith(os.path.join(basedir, 'site-packages'))))) + # -------------------------------------------- HTML documentation generator class HTMLRepr(Repr): @@ -856,8 +817,8 @@ def docmodule(self, object, name=None, mod=None, *ignored): except TypeError: filelink = '(built-in)' info = [] - if hasattr(object, '__version__'): - version = str(object.__version__) + + if version := self._get_version(object): if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() info.append('version %s' % self.escape(version)) @@ -1392,8 +1353,7 @@ def docmodule(self, object, name=None, mod=None, *ignored): contents.append(self.docother(value, key, name, maxlen=70)) result = result + self.section('DATA', '\n'.join(contents)) - if hasattr(object, '__version__'): - version = str(object.__version__) + if version := self._get_version(object): if version[:11] == '$' + 'Revision: ' and version[-1:] == '$': version = version[11:-1].strip() result = result + self.section('VERSION', version) @@ -1824,7 +1784,6 @@ def writedocs(dir, pkgpath='', done=None): def _introdoc(): - import textwrap ver = '%d.%d' % sys.version_info[:2] if os.environ.get('PYTHON_BASIC_REPL'): pyrepl_keys = '' @@ -1893,6 +1852,7 @@ class Helper: 'in': ('in', 'SEQUENCEMETHODS'), 'is': 'COMPARISON', 'lambda': ('lambda', 'FUNCTIONS'), + 'lazy': ('lazy', 'MODULES'), 'match': ('match', 'if'), 'nonlocal': ('nonlocal', 'global NAMESPACES'), 'not': 'BOOLEAN', @@ -2185,7 +2145,6 @@ def showtopic(self, topic, more_xrefs=''): if more_xrefs: xrefs = (xrefs or '') + ' ' + more_xrefs if xrefs: - import textwrap text = 'Related help topics: ' + ', '.join(xrefs.split()) + '\n' wrapped_text = textwrap.wrap(text, 72) doc += '\n%s\n' % '\n'.join(wrapped_text) diff --git a/Lib/pydoc_data/module_docs.py b/Lib/pydoc_data/module_docs.py index 2d3543f2983..89e65350c04 100644 --- a/Lib/pydoc_data/module_docs.py +++ b/Lib/pydoc_data/module_docs.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Wed Aug 5 13:29:41 2026 +# Autogenerated by Sphinx on Fri Oct 9 13:16:32 2026 # as part of the release process. module_docs = { @@ -23,7 +23,6 @@ 'bisect': 'bisect#module-bisect', 'builtins': 'builtins#module-builtins', 'bz2': 'bz2#module-bz2', - 'cProfile': 'profile#module-cProfile', 'calendar': 'calendar#module-calendar', 'cgi': 'cgi#module-cgi', 'cgitb': 'cgitb#module-cgitb', @@ -149,6 +148,7 @@ 'mailcap': 'mailcap#module-mailcap', 'marshal': 'marshal#module-marshal', 'math': 'math#module-math', + 'math.integer': 'math.integer#module-math.integer', 'mimetypes': 'mimetypes#module-mimetypes', 'mmap': 'mmap#module-mmap', 'modulefinder': 'modulefinder#module-modulefinder', @@ -183,7 +183,10 @@ 'posix': 'posix#module-posix', 'pprint': 'pprint#module-pprint', 'profile': 'profile#module-profile', - 'pstats': 'profile#module-pstats', + 'profiling': 'profiling#module-profiling', + 'profiling.sampling': 'profiling.sampling#module-profiling.sampling', + 'profiling.tracing': 'profiling.tracing#module-profiling.tracing', + 'pstats': 'pstats#module-pstats', 'pty': 'pty#module-pty', 'pwd': 'pwd#module-pwd', 'py_compile': 'py_compile#module-py_compile', diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 753d9d43069..39e570dc34d 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,4 +1,4 @@ -# Autogenerated by Sphinx on Wed Aug 5 13:29:41 2026 +# Autogenerated by Sphinx on Fri Oct 9 13:16:32 2026 # as part of the release process. topics = { @@ -912,9 +912,10 @@ class to a new value, *value*. renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* "TypeError" will be raised if nonempty *__slots__* are defined for a - class derived from a ""variable-length" built-in type" such as - "int", "bytes", and "tuple". +* "TypeError" will be raised if *__slots__* other than *__dict__* and + *__weakref__* are defined for a class derived from a ""variable- + length" built-in type" such as "int", "bytes", and "type", except + "tuple". * Any non-string *iterable* may be assigned to *__slots__*. @@ -934,6 +935,10 @@ class derived from a ""variable-length" built-in type" such as * If an *iterator* is used for *__slots__* then a *descriptor* is created for each of the iterator’s values. However, the *__slots__* attribute will be an empty iterator. + +Changed in version 3.15: Allowed defining the *__dict__* and +*__weakref__* *__slots__* for any class. Allowed defining any +*__slots__* for a class derived from "tuple". ''', 'attribute-references': r'''Attribute references ******************** @@ -1394,7 +1399,7 @@ class and instance attributes applies as for regular assignments. "return" statement, the return value is "None". a built-in function or method: - The result is up to the interpreter; see Built-in Functions for the + The result is up to the interpreter; see Built-in functions for the descriptions of built-in functions and methods. a class object: @@ -1433,6 +1438,9 @@ class Foo: class Foo(object): pass +There may be one or more base classes; see Multiple inheritance below +for more information. + The class’s suite is then executed in a new execution frame (see Naming and binding), using a newly created local namespace and the original global namespace. (Usually, the suite contains mostly @@ -1496,6 +1504,115 @@ class attributes; they are shared by instances. Instance attributes **PEP 3129** - Class Decorators The proposal that added class decorators. Function and method decorators were introduced in **PEP 318**. + + +Multiple inheritance +==================== + +Python classes may have multiple base classes, a technique known as +*multiple inheritance*. The base classes are specified in the class +definition by listing them in parentheses after the class name, +separated by commas. For example, the following class definition: + + >>> class A: pass + >>> class B: pass + >>> class C(A, B): pass + +defines a class "C" that inherits from classes "A" and "B". + +The *method resolution order* (MRO) is the order in which base classes +are searched when looking up an attribute on a class. See The Python +2.3 Method Resolution Order for a description of how Python determines +the MRO for a class. + +Multiple inheritance is not always allowed. Attempting to define a +class with multiple inheritance will raise an error if one of the +bases does not allow subclassing, if a consistent MRO cannot be +created, if no valid metaclass can be determined, or if there is an +instance layout conflict. We’ll discuss each of these in turn. + +First, all base classes must allow subclassing. While most classes +allow subclassing, some built-in classes do not, such as "bool": + + >>> class SubBool(bool): # TypeError + ... pass + Traceback (most recent call last): + ... + TypeError: type 'bool' is not an acceptable base type + +In the resolved MRO of a class, the class’s bases appear in the order +they were specified in the class’s bases list. Additionally, the MRO +always lists a child class before any of its bases. A class definition +will fail if it is impossible to resolve a consistent MRO that +satisfies these rules from the list of bases provided: + + >>> class Base: pass + >>> class Child(Base): pass + >>> class Grandchild(Base, Child): pass # TypeError + Traceback (most recent call last): + ... + TypeError: Cannot create a consistent method resolution order (MRO) for bases Base, Child + +In the MRO of "Grandchild", "Base" must appear before "Child" because +it is first in the base class list, but it must also appear after +"Child" because it is a parent of "Child". This is a contradiction, so +the class cannot be defined. + +If some of the bases have a custom *metaclass*, the metaclass of the +resulting class is chosen among the metaclasses of the bases and the +explicitly specified metaclass of the child class. It must be a +metaclass that is a subclass of all other candidate metaclasses. If no +such metaclass exists among the candidates, the class cannot be +created, as explained in Determining the appropriate metaclass. + +Finally, the instance layouts of the bases must be compatible. This +means that it must be possible to compute a *solid base* for the +class. Exactly which classes are solid bases depends on the Python +implementation. + +**CPython implementation detail:** In CPython, a class is a solid base +if it has a nonempty "__slots__" definition. Many but not all classes +defined in C are also solid bases, including most builtins (such as +"int" or "BaseException") but excluding most concrete "Exception" +classes. Generally, a C class is a solid base if its underlying struct +is different in size from its base class. + +Every class has a solid base. "object", the base class, has itself as +its solid base. If there is a single base, the child class’s solid +base is that class if it is a solid base, or else the base class’s +solid base. If there are multiple bases, we first find the solid base +for each base class to produce a list of candidate solid bases. If +there is a unique solid base that is a subclass of all others, then +that class is the solid base. Otherwise, class creation fails. + +Example: + + >>> class Solid1: + ... __slots__ = ("solid1",) + >>> + >>> class Solid2: + ... __slots__ = ("solid2",) + >>> + >>> class SolidChild(Solid1): + ... __slots__ = ("solid_child",) + >>> + >>> class C1: # solid base is `object` + ... pass + >>> + >>> # OK: solid bases are `Solid1` and `object`, and `Solid1` is a subclass of `object`. + >>> class C2(Solid1, C1): # solid base is `Solid1` + ... pass + >>> + >>> # OK: solid bases are `SolidChild` and `Solid1`, and `SolidChild` is a subclass of `Solid1`. + >>> class C3(SolidChild, Solid1): # solid base is `SolidChild` + ... pass + >>> + >>> # Error: solid bases are `Solid1` and `Solid2`, but neither is a subclass of the other. + >>> class C4(Solid1, Solid2): # error: no single solid base + ... pass + Traceback (most recent call last): + ... + TypeError: multiple bases have instance lay-out conflict ''', 'comparisons': r'''Comparisons *********** @@ -1851,7 +1968,7 @@ class attributes; they are shared by instances. Instance attributes iterable. The first item provided by the iterator is then assigned to the target list using the standard rules for assignments (see Assignment statements), and the suite is executed. This repeats for -each item provided by the iterator. When the iterator is exhausted, +each item provided by the iterator. When the iterator is *exhausted*, the suite in the "else" clause, if present, is executed, and the loop terminates. @@ -2455,7 +2572,7 @@ def foo(): | "None" | "True" | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and @@ -2779,6 +2896,8 @@ def foo(): * "float" + * "frozendict" + * "frozenset" * "int" @@ -2832,7 +2951,7 @@ def foo(): | parameter_list_no_posonly parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] | parameter_list_starargs - parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + parameter_list_starargs: "*" star_parameter ("," defparameter)* ["," [parameter_star_kwargs]] | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] | parameter_star_kwargs parameter_star_kwargs: "**" parameter [","] @@ -2997,6 +3116,9 @@ class Foo: class Foo(object): pass +There may be one or more base classes; see Multiple inheritance below +for more information. + The class’s suite is then executed in a new execution frame (see Naming and binding), using a newly created local namespace and the original global namespace. (Usually, the suite contains mostly @@ -3062,6 +3184,115 @@ class attributes; they are shared by instances. Instance attributes decorators were introduced in **PEP 318**. +Multiple inheritance +-------------------- + +Python classes may have multiple base classes, a technique known as +*multiple inheritance*. The base classes are specified in the class +definition by listing them in parentheses after the class name, +separated by commas. For example, the following class definition: + + >>> class A: pass + >>> class B: pass + >>> class C(A, B): pass + +defines a class "C" that inherits from classes "A" and "B". + +The *method resolution order* (MRO) is the order in which base classes +are searched when looking up an attribute on a class. See The Python +2.3 Method Resolution Order for a description of how Python determines +the MRO for a class. + +Multiple inheritance is not always allowed. Attempting to define a +class with multiple inheritance will raise an error if one of the +bases does not allow subclassing, if a consistent MRO cannot be +created, if no valid metaclass can be determined, or if there is an +instance layout conflict. We’ll discuss each of these in turn. + +First, all base classes must allow subclassing. While most classes +allow subclassing, some built-in classes do not, such as "bool": + + >>> class SubBool(bool): # TypeError + ... pass + Traceback (most recent call last): + ... + TypeError: type 'bool' is not an acceptable base type + +In the resolved MRO of a class, the class’s bases appear in the order +they were specified in the class’s bases list. Additionally, the MRO +always lists a child class before any of its bases. A class definition +will fail if it is impossible to resolve a consistent MRO that +satisfies these rules from the list of bases provided: + + >>> class Base: pass + >>> class Child(Base): pass + >>> class Grandchild(Base, Child): pass # TypeError + Traceback (most recent call last): + ... + TypeError: Cannot create a consistent method resolution order (MRO) for bases Base, Child + +In the MRO of "Grandchild", "Base" must appear before "Child" because +it is first in the base class list, but it must also appear after +"Child" because it is a parent of "Child". This is a contradiction, so +the class cannot be defined. + +If some of the bases have a custom *metaclass*, the metaclass of the +resulting class is chosen among the metaclasses of the bases and the +explicitly specified metaclass of the child class. It must be a +metaclass that is a subclass of all other candidate metaclasses. If no +such metaclass exists among the candidates, the class cannot be +created, as explained in Determining the appropriate metaclass. + +Finally, the instance layouts of the bases must be compatible. This +means that it must be possible to compute a *solid base* for the +class. Exactly which classes are solid bases depends on the Python +implementation. + +**CPython implementation detail:** In CPython, a class is a solid base +if it has a nonempty "__slots__" definition. Many but not all classes +defined in C are also solid bases, including most builtins (such as +"int" or "BaseException") but excluding most concrete "Exception" +classes. Generally, a C class is a solid base if its underlying struct +is different in size from its base class. + +Every class has a solid base. "object", the base class, has itself as +its solid base. If there is a single base, the child class’s solid +base is that class if it is a solid base, or else the base class’s +solid base. If there are multiple bases, we first find the solid base +for each base class to produce a list of candidate solid bases. If +there is a unique solid base that is a subclass of all others, then +that class is the solid base. Otherwise, class creation fails. + +Example: + + >>> class Solid1: + ... __slots__ = ("solid1",) + >>> + >>> class Solid2: + ... __slots__ = ("solid2",) + >>> + >>> class SolidChild(Solid1): + ... __slots__ = ("solid_child",) + >>> + >>> class C1: # solid base is `object` + ... pass + >>> + >>> # OK: solid bases are `Solid1` and `object`, and `Solid1` is a subclass of `object`. + >>> class C2(Solid1, C1): # solid base is `Solid1` + ... pass + >>> + >>> # OK: solid bases are `SolidChild` and `Solid1`, and `SolidChild` is a subclass of `Solid1`. + >>> class C3(SolidChild, Solid1): # solid base is `SolidChild` + ... pass + >>> + >>> # Error: solid bases are `Solid1` and `Solid2`, but neither is a subclass of the other. + >>> class C4(Solid1, Solid2): # error: no single solid base + ... pass + Traceback (most recent call last): + ... + TypeError: multiple bases have instance lay-out conflict + + Coroutines ========== @@ -3816,11 +4047,11 @@ def f() -> annotation: ... object.__hash__(self) Called by built-in function "hash()" and for operations on members - of hashed collections including "set", "frozenset", and "dict". - The "__hash__()" method should return an integer. The only required - property is that objects which compare equal have the same hash - value; it is advised to mix together the hash values of the - components of the object that also play a part in comparison of + of hashed collections including "set", "frozenset", "dict", and + "frozendict". The "__hash__()" method should return an integer. The + only required property is that objects which compare equal have the + same hash value; it is advised to mix together the hash values of + the components of the object that also play a part in comparison of objects by packing them into a tuple and hashing the tuple. Example: @@ -4381,8 +4612,8 @@ class pdb.Pdb(completekey='tab', stdin=None, stdout=None, skip=None, nosigint=Fa To remove all commands from a breakpoint, type "commands" and follow it immediately with "end"; that is, give no commands. - With no *bpnumber* argument, "commands" refers to the last - breakpoint set. + With no *bpnumber* argument, "commands" refers to the most recently + set breakpoint that still exists. You can use breakpoint commands to start your program up again. Simply use the "continue" command, or "step", or any other command @@ -4686,7 +4917,7 @@ def middle(): def inner(x): 1 / x - out() + out() calling "pdb.pm()" will allow to move between exceptions: @@ -4742,47 +4973,49 @@ def inner(x): 'dict': r'''Dictionary displays ******************* -A dictionary display is a possibly empty series of dict items -(key/value pairs) enclosed in curly braces: - - dict_display: "{" [dict_item_list | dict_comprehension] "}" - dict_item_list: dict_item ("," dict_item)* [","] - dict_item: expression ":" expression | "**" or_expr - dict_comprehension: expression ":" expression comp_for - -A dictionary display yields a new dictionary object. - -If a comma-separated sequence of dict items is given, they are -evaluated from left to right to define the entries of the dictionary: -each key object is used as a key into the dictionary to store the -corresponding value. This means that you can specify the same key -multiple times in the dict item list, and the final dictionary’s value -for that key will be the last one given. - -A double asterisk "**" denotes *dictionary unpacking*. Its operand -must be a *mapping*. Each mapping item is added to the new -dictionary. Later values replace values already set by earlier dict -items and earlier dictionary unpackings. +A *dictionary display* is a possibly empty series of *dict items* +enclosed in curly braces. Each dict item is a colon-separated pair of +expressions: the *key* and its associated *value*. For example: + + >>> {1: 'one', 2: 'two'} + {1: 'one', 2: 'two'} + +At runtime, when a dictionary comprehension is evaluated, the +expressions are evaluated from left to right. Each key object is used +as a key into the dictionary to store the corresponding value. This +means that you can specify the same key multiple times in the +comprehension, and the final dictionary’s value for a given key will +be the last one given. For example: + + >>> { + ... 1: 'this will be overridden', + ... 2: 'two', + ... 1: 'also overridden', + ... 1: 'one', + ... } + {1: 'one', 2: 'two'} + +Instead of a key-value pair, a dict item may be an expression prefixed +by a double asterisk "**". This denotes *dictionary unpacking*. At +runtime, the expression must evaluate to a *mapping*; each item of the +mapping is added to the new dictionary. As with key-value pairs, later +values replace values already set by earlier items and unpackings. +This may be used to override a set of defaults: + + >>> defaults = {'color': 'blue', 'count': 8} + >>> overrides = {'color': 'yellow'} + >>> {**defaults, **overrides} + {'color': 'yellow', 'count': 8} Added in version 3.5: Unpacking into dictionary displays, originally proposed by **PEP 448**. -A dict comprehension, in contrast to list and set comprehensions, -needs two expressions separated with a colon followed by the usual -“for” and “if” clauses. When the comprehension is run, the resulting -key and value elements are inserted in the new dictionary in the order -they are produced. - -Restrictions on the types of the key values are listed earlier in -section The standard type hierarchy. (To summarize, the key type -should be *hashable*, which excludes all mutable objects.) Clashes -between duplicate keys are not detected; the last value (textually -rightmost in the display) stored for a given key value prevails. - -Changed in version 3.8: Prior to Python 3.8, in dict comprehensions, -the evaluation order of key and value was not well-defined. In -CPython, the value was evaluated before the key. Starting with 3.8, -the key is evaluated before the value, as proposed by **PEP 572**. +The formal grammar for dict displays is: + + dict: '{' [double_starred_kvpairs] '}' + double_starred_kvpairs: ','.double_starred_kvpair+ [','] + double_starred_kvpair: '**' or_expr | kvpair + kvpair: expression ':' expression ''', 'dynamic-features': r'''Interaction with dynamic features ********************************* @@ -5412,8 +5645,22 @@ class of the instance or a *non-virtual base class* thereof. The is the number of expressions in the list. The expressions are evaluated from left to right. -An asterisk "*" denotes *iterable unpacking*. Its operand must be an -*iterable*. The iterable is expanded into a sequence of items, which +A trailing comma is required only to create a one-item tuple, such as +"1,"; it is optional in all other cases. A single expression without a +trailing comma doesn’t create a tuple, but rather yields the value of +that expression. (To create an empty tuple, use an empty pair of +parentheses: "()".) + + +Iterable unpacking +================== + +In an expression list or tuple, list or set display, any expression +may be prefixed with an asterisk ("*"). This denotes *iterable +unpacking*. + +At runtime, the asterisk-prefixed expression must evaluate to an +*iterable*. The iterable is expanded into a sequence of items, which are included in the new tuple, list, or set, at the site of the unpacking. @@ -5422,12 +5669,6 @@ class of the instance or a *non-virtual base class* thereof. The Added in version 3.11: Any item in an expression list may be starred. See **PEP 646**. - -A trailing comma is required only to create a one-item tuple, such as -"1,"; it is optional in all other cases. A single expression without a -trailing comma doesn’t create a tuple, but rather yields the value of -that expression. (To create an empty tuple, use an empty pair of -parentheses: "()".) ''', 'floating': r'''Floating-point literals *********************** @@ -5497,7 +5738,7 @@ class of the instance or a *non-virtual base class* thereof. The iterable. The first item provided by the iterator is then assigned to the target list using the standard rules for assignments (see Assignment statements), and the suite is executed. This repeats for -each item provided by the iterator. When the iterator is exhausted, +each item provided by the iterator. When the iterator is *exhausted*, the suite in the "else" clause, if present, is executed, and the loop terminates. @@ -6096,7 +6337,7 @@ class of the instance or a *non-virtual base class* thereof. The | parameter_list_no_posonly parameter_list_no_posonly: defparameter ("," defparameter)* ["," [parameter_list_starargs]] | parameter_list_starargs - parameter_list_starargs: "*" [star_parameter] ("," defparameter)* ["," [parameter_star_kwargs]] + parameter_list_starargs: "*" star_parameter ("," defparameter)* ["," [parameter_star_kwargs]] | "*" ("," defparameter)+ ["," [parameter_star_kwargs]] | parameter_star_kwargs parameter_star_kwargs: "**" parameter [","] @@ -6369,6 +6610,8 @@ def whats_on_the_telly(penguin=None): * "type", when used in the "type" statement. +* "lazy", when used before an "import" statement. + These syntactically act as keywords in their specific contexts, but this distinction is done at the parser level, not when tokenizing. @@ -6378,6 +6621,8 @@ def whats_on_the_telly(penguin=None): Changed in version 3.12: "type" is now a soft keyword. +Changed in version 3.15: "lazy" is now a soft keyword. + Reserved classes of identifiers =============================== @@ -6587,10 +6832,10 @@ def whats_on_the_telly(penguin=None): 'import': r'''The "import" statement ********************** - import_stmt: "import" module ["as" identifier] ("," module ["as" identifier])* - | "from" relative_module "import" identifier ["as" identifier] + import_stmt: ["lazy"] "import" module ["as" identifier] ("," module ["as" identifier])* + | ["lazy"] "from" relative_module "import" identifier ["as" identifier] ("," identifier ["as" identifier])* - | "from" relative_module "import" "(" identifier ["as" identifier] + | ["lazy"] "from" relative_module "import" "(" identifier ["as" identifier] ("," identifier ["as" identifier])* [","] ")" | "from" relative_module "import" "*" module: (identifier ".")* identifier @@ -6703,6 +6948,91 @@ def whats_on_the_telly(penguin=None): "sys.path", "sys.meta_path", "sys.path_hooks". +Lazy imports +============ + +The "lazy" keyword is a soft keyword that only has special meaning +when it appears immediately before an "import" or "from" statement. +When an import statement is preceded by the "lazy" keyword, the import +becomes *lazy*: the module is not loaded immediately at the import +statement. Instead, a lazy proxy object is created and bound to the +name. The actual module is loaded on first use of that name. + +Lazy imports are only permitted at module scope. Using "lazy" inside a +function, class body, or "try"/"except"/"finally" block raises a +"SyntaxError". Star imports cannot be lazy ("lazy from module import +*" is a syntax error), and future statements cannot be lazy. + +When using "lazy from ... import", each imported name is bound to a +lazy proxy object. The first access to any of these names triggers +loading of the entire module and resolves only that specific name to +its actual value. Other names remain as lazy proxies until they are +accessed. + +Example: + + lazy import json + import sys + + print('json' in sys.modules) # False - json module not yet loaded + + # First use triggers loading + result = json.dumps({"hello": "world"}) + + print('json' in sys.modules) # True - now loaded + +If an error occurs during module loading (such as "ImportError" or +"SyntaxError"), it is raised at the point where the lazy import is +first used, not at the import statement itself. + +See **PEP 810** for the full specification of lazy imports. + +Added in version 3.15. + + +Compatibility via "__lazy_modules__" +------------------------------------ + +As an alternative to using the "lazy" keyword, a module can opt into +lazy loading for specific imports by defining a module-level +"__lazy_modules__" variable. When present, it must be a container of +fully qualified module name strings. Any regular (non-"lazy") +"import" statement at module scope whose target appears in +"__lazy_modules__" is treated as a lazy import, exactly as if the +"lazy" keyword had been used. + +This provides a way to enable lazy loading for specific dependencies +without changing individual "import" statements. This is useful when +supporting Python versions older than 3.15 while using lazy imports in +3.15+: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the +lookup, so "__lazy_modules__" must always contain fully qualified +module names. + +For "from"-style imports, the relevant name is the module following +"from", not the names of its members: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or "try"/"except"/"finally" +blocks are always eager, regardless of "__lazy_modules__". + +Added in version 3.15. + + Future statements ================= @@ -6902,21 +7232,109 @@ def (parameters): See section Function definitions for the syntax of parameter lists. Note that functions created with lambda expressions cannot contain statements or annotations. +''', + 'lazy': r'''Lazy imports +************ + +The "lazy" keyword is a soft keyword that only has special meaning +when it appears immediately before an "import" or "from" statement. +When an import statement is preceded by the "lazy" keyword, the import +becomes *lazy*: the module is not loaded immediately at the import +statement. Instead, a lazy proxy object is created and bound to the +name. The actual module is loaded on first use of that name. + +Lazy imports are only permitted at module scope. Using "lazy" inside a +function, class body, or "try"/"except"/"finally" block raises a +"SyntaxError". Star imports cannot be lazy ("lazy from module import +*" is a syntax error), and future statements cannot be lazy. + +When using "lazy from ... import", each imported name is bound to a +lazy proxy object. The first access to any of these names triggers +loading of the entire module and resolves only that specific name to +its actual value. Other names remain as lazy proxies until they are +accessed. + +Example: + + lazy import json + import sys + + print('json' in sys.modules) # False - json module not yet loaded + + # First use triggers loading + result = json.dumps({"hello": "world"}) + + print('json' in sys.modules) # True - now loaded + +If an error occurs during module loading (such as "ImportError" or +"SyntaxError"), it is raised at the point where the lazy import is +first used, not at the import statement itself. + +See **PEP 810** for the full specification of lazy imports. + +Added in version 3.15. + + +Compatibility via "__lazy_modules__" +==================================== + +As an alternative to using the "lazy" keyword, a module can opt into +lazy loading for specific imports by defining a module-level +"__lazy_modules__" variable. When present, it must be a container of +fully qualified module name strings. Any regular (non-"lazy") +"import" statement at module scope whose target appears in +"__lazy_modules__" is treated as a lazy import, exactly as if the +"lazy" keyword had been used. + +This provides a way to enable lazy loading for specific dependencies +without changing individual "import" statements. This is useful when +supporting Python versions older than 3.15 while using lazy imports in +3.15+: + + __lazy_modules__ = ["json", "pathlib"] + + import json # loaded lazily (name is in __lazy_modules__) + import os # loaded eagerly (name not in __lazy_modules__) + + import pathlib # loaded lazily + +Relative imports are resolved to their absolute name before the +lookup, so "__lazy_modules__" must always contain fully qualified +module names. + +For "from"-style imports, the relevant name is the module following +"from", not the names of its members: + + # In mypackage/mymodule.py + __lazy_modules__ = ["mypackage", "mypackage.sub.utils"] + + from . import helper # loaded lazily: . resolves to mypackage + from .sub.utils import func # loaded lazily: .sub.utils resolves to mypackage.sub.utils + import json # loaded eagerly (not in __lazy_modules__) + +Imports inside functions, class bodies, or "try"/"except"/"finally" +blocks are always eager, regardless of "__lazy_modules__". + +Added in version 3.15. ''', 'lists': r'''List displays ************* -A list display is a possibly empty series of expressions enclosed in -square brackets: +A *list display* is a possibly empty series of expressions enclosed in +square brackets. For example: - list_display: "[" [flexible_expression_list | comprehension] "]" + >>> ["one", "two", "three"] + ['one', 'two', 'three'] + >>> ["one"] # One-element list + ['one'] + >>> [] # empty list + [] -A list display yields a new list object, the contents being specified -by either a list of expressions or a comprehension. When a comma- -separated list of expressions is supplied, its elements are evaluated -from left to right and placed into the list object in that order. -When a comprehension is supplied, the list is constructed from the -elements resulting from the comprehension. +See Container displays for general information on displays. + +The formal grammar for list displays is: + + list: '[' [flexible_expression_list] ']' ''', 'match': r'''The "match" statement ********************* @@ -7154,7 +7572,7 @@ def (parameters): | "None" | "True" | "False" - signed_number: ["-"] NUMBER + signed_number: ["+" | "-"] NUMBER The rule "strings" and the token "NUMBER" are defined in the standard Python grammar. Triple-quoted strings are supported. Raw strings and @@ -7478,6 +7896,8 @@ def (parameters): * "float" + * "frozendict" + * "frozenset" * "int" @@ -9054,11 +9474,11 @@ class C: pass # a class with no methods (yet) object.__hash__(self) Called by built-in function "hash()" and for operations on members - of hashed collections including "set", "frozenset", and "dict". - The "__hash__()" method should return an integer. The only required - property is that objects which compare equal have the same hash - value; it is advised to mix together the hash values of the - components of the object that also play a part in comparison of + of hashed collections including "set", "frozenset", "dict", and + "frozendict". The "__hash__()" method should return an integer. The + only required property is that objects which compare equal have the + same hash value; it is advised to mix together the hash values of + the components of the object that also play a part in comparison of objects by packing them into a tuple and hashing the tuple. Example: @@ -9449,9 +9869,10 @@ class to a new value, *value*. renders the meaning of the program undefined. In the future, a check may be added to prevent this. -* "TypeError" will be raised if nonempty *__slots__* are defined for a - class derived from a ""variable-length" built-in type" such as - "int", "bytes", and "tuple". +* "TypeError" will be raised if *__slots__* other than *__dict__* and + *__weakref__* are defined for a class derived from a ""variable- + length" built-in type" such as "int", "bytes", and "type", except + "tuple". * Any non-string *iterable* may be assigned to *__slots__*. @@ -9472,6 +9893,10 @@ class derived from a ""variable-length" built-in type" such as created for each of the iterator’s values. However, the *__slots__* attribute will be an empty iterator. +Changed in version 3.15: Allowed defining the *__dict__* and +*__weakref__* *__slots__* for any class. Allowed defining any +*__slots__* for a class derived from "tuple". + Customizing class creation ========================== @@ -10530,7 +10955,7 @@ class is used in a class pattern with positional arguments, each >>> 'straße'.casefold() 'strasse' - The casefolding algorithm is described in section 3.13 ‘Default + The casefolding algorithm is described in section 3.13.3 ‘Default Case Folding’ of the Unicode Standard. Added in version 3.3. @@ -10754,7 +11179,7 @@ class is used in a class pattern with positional arguments, each characters are those characters defined in the Unicode character database as “Letter”, i.e., those with general category property being one of “Lm”, “Lt”, “Lu”, “Ll”, or “Lo”. Note that this is - different from the Alphabetic property defined in the section 4.10 + different from the Alphabetic property defined in section 4.10 ‘Letters, Alphabetic, and Ideographic’ of the Unicode Standard. For example: @@ -10985,8 +11410,8 @@ class is used in a class pattern with positional arguments, each >>> 'Lower Method Example'.lower() 'lower method example' - The lowercasing algorithm used is described in section 3.13 - ‘Default Case Folding’ of the Unicode Standard. + The lowercasing algorithm used is described in section 3.13.2 + ‘Default Case Conversion’ of the Unicode Standard. str.lstrip(chars=None, /) @@ -11026,6 +11451,8 @@ class is used in a class pattern with positional arguments, each a third argument, it must be a string, whose characters will be mapped to "None" in the result. + Changed in version 3.15: *dict* can now be a "frozendict". + str.partition(sep, /) Split the string at the first occurrence of *sep*, and return a @@ -11162,9 +11589,9 @@ class is used in a class pattern with positional arguments, each Return a list of the words in the string, using *sep* as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* ones. If *sep* is not specified or - "None", any whitespace string is a separator. Except for splitting - from the right, "rsplit()" behaves like "split()" which is - described in detail below. + "None", any "whitespace" string is a separator. Except for + splitting from the right, "rsplit()" behaves like "split()" which + is described in detail below. str.rstrip(chars=None, /) @@ -11216,7 +11643,7 @@ class is used in a class pattern with positional arguments, each ['1', '2', '3<4'] If *sep* is not specified or is "None", a different splitting - algorithm is applied: runs of consecutive whitespace are regarded + algorithm is applied: runs of consecutive "whitespace" are regarded as a single separator, and the result will contain no empty strings at the start or end if the string has leading or trailing whitespace. Consequently, splitting an empty string or a string @@ -11422,6 +11849,12 @@ class is used in a class pattern with positional arguments, each You can use "str.maketrans()" to create a translation map from character-to-character mappings in different formats. + The following example uses a mapping to replace "'a'" with "'X'", + "'b'" with "'Y'", and delete "'c'": + + >>> 'abc123'.translate({ord('a'): 'X', ord('b'): 'Y', ord('c'): None}) + 'XY123' + See also the "codecs" module for a more flexible approach to custom character mappings. @@ -11433,8 +11866,8 @@ class is used in a class pattern with positional arguments, each category of the resulting character(s) is not “Lu” (Letter, uppercase), but e.g. “Lt” (Letter, titlecase). - The uppercasing algorithm used is described in section 3.13 - ‘Default Case Folding’ of the Unicode Standard. + The uppercasing algorithm used is described in section 3.13.2 + ‘Default Case Conversion’ of the Unicode Standard. str.zfill(width, /) @@ -12717,7 +13150,7 @@ def foo(): target of assignments or "del" statements. The built-in function "len()" returns the number of items in a mapping. -There is currently a single intrinsic mapping type: +There are two intrinsic mapping types: Dictionaries @@ -12751,6 +13184,18 @@ def foo(): rather than a language guarantee. +Frozen dictionaries +------------------- + +These represent an immutable dictionary. They are created by the +built-in "frozendict()" constructor. A frozendict is *hashable* if +all of its keys and values are hashable, in which case it can be used +as an element of a set, or as a key in another mapping. "frozendict" +is not a subclass of "dict"; it inherits directly from "object". + +Added in version 3.15. + + Callable types ============== @@ -12914,12 +13359,12 @@ def foo(): Generator functions ------------------- -A function or method which uses the "yield" statement (see section The -yield statement) is called a *generator function*. Such a function, +A function or method which contains a "yield" expression (see section +Yield expressions) is called a *generator function*. Such a function, when called, always returns an *iterator* object which can be used to execute the body of the function: calling the iterator’s "iterator.__next__()" method will cause the function to execute until -it provides a value using the "yield" statement. When the function +it provides a value using the "yield" expression. When the function executes a "return" statement or falls off the end, a "StopIteration" exception is raised and the iterator will have reached the end of the set of values to be returned. @@ -12938,11 +13383,11 @@ def foo(): Asynchronous generator functions -------------------------------- -A function or method which is defined using "async def" and which uses -the "yield" statement is called a *asynchronous generator function*. -Such a function, when called, returns an *asynchronous iterator* -object which can be used in an "async for" statement to execute the -body of the function. +A function or method which is defined using "async def" and which +contains a "yield" expression is called a *asynchronous generator +function*. Such a function, when called, returns an *asynchronous +iterator* object which can be used in an "async for" statement to +execute the body of the function. Calling the asynchronous iterator’s "aiterator.__anext__" method will return an *awaitable* which when awaited will execute until it @@ -13108,7 +13553,7 @@ def foo(): "ImportWarning" when falling back to "__package__" during import resolution. - Deprecated since version 3.13, will be removed in version 3.15: + Deprecated since version 3.13, removed in version 3.15: "__package__" will cease to be set or taken into consideration by the import system or standard library. @@ -13152,41 +13597,21 @@ def foo(): module.__file__ -module.__cached__ - - "__file__" and "__cached__" are both optional attributes that may - or may not be set. Both attributes should be a "str" when they are - available. - - "__file__" indicates the pathname of the file from which the module - was loaded (if loaded from a file), or the pathname of the shared - library file for extension modules loaded dynamically from a shared - library. It might be missing for certain types of modules, such as - C modules that are statically linked into the interpreter, and the - import system may opt to leave it unset if it has no semantic - meaning (for example, a module loaded from a database). - - If "__file__" is set then the "__cached__" attribute might also be - set, which is the path to any compiled version of the code (for - example, a byte-compiled file). The file does not need to exist to - set this attribute; the path can simply point to where the compiled - file *would* exist (see **PEP 3147**). - - Note that "__cached__" may be set even if "__file__" is not set. - However, that scenario is quite atypical. Ultimately, the *loader* - is what makes use of the module spec provided by the *finder* (from - which "__file__" and "__cached__" are derived). So if a loader can - load from a cached module but otherwise does not load from a file, - that atypical scenario may be appropriate. + "__file__" is an optional attribute that may or may not be set. + Both attributes should be a "str" when they are available. - It is **strongly** recommended that you use - "module.__spec__.cached" instead of "module.__cached__". + An optional attribute, "__file__" indicates the pathname of the + file from which the module was loaded (if loaded from a file), or + the pathname of the shared library file for extension modules + loaded dynamically from a shared library. It might be missing for + certain types of modules, such as C modules that are statically + linked into the interpreter, and the import system may opt to leave + it unset if it has no semantic meaning (for example, a module + loaded from a database). - Deprecated since version 3.13, will be removed in version 3.15: - Setting "__cached__" on a module while failing to set - "__spec__.cached" is deprecated. In Python 3.15, "__cached__" will - cease to be set or taken into consideration by the import system or - standard library. +Changed in version 3.15: The "__cached__" attribute is no longer set +on modules or taken into consideration by the import system or +standard library. Other writable attributes on module objects @@ -13216,6 +13641,19 @@ def foo(): Added in version 3.14. +module.__lazy_modules__ + + A container (an object implementing "__contains__()") of fully + qualified module name strings. When defined at module scope, any + regular "import" statement in that module whose target module name + appears in this container is treated as a lazy import, as if the + "lazy" keyword had been used. Imports inside functions, class + bodies, or "try"/"except"/"finally" blocks are unaffected. + + See Compatibility via __lazy_modules__ for details and examples. + + Added in version 3.15. + Module dictionaries ------------------- @@ -13512,11 +13950,14 @@ class instance has a namespace implemented as a dictionary which is +----------------------------------------------------+----------------------------------------------------+ | codeobject.co_firstlineno | The line number of the first line of the function | +----------------------------------------------------+----------------------------------------------------+ -| codeobject.co_lnotab | A string encoding the mapping from *bytecode* | -| | offsets to line numbers. For details, see the | -| | source code of the interpreter. Deprecated since | -| | version 3.12: This attribute of code objects is | -| | deprecated, and may be removed in Python 3.15. | +| codeobject.co_linetable | A "bytes" object containing encoded source | +| | location information. The exact format is an | +| | implementation detail and may change between | +| | Python versions. Use "co_lines()" and | +| | "co_positions()" for supported access to line and | +| | position information. To create a modified copy of | +| | a code object, use "replace()". Added in version | +| | 3.10. | +----------------------------------------------------+----------------------------------------------------+ | codeobject.co_stacksize | The required stack size of the code object | +----------------------------------------------------+----------------------------------------------------+ @@ -13785,6 +14226,10 @@ class instance has a namespace implemented as a dictionary which is Slice objects are used to represent slices for "__getitem__()" methods. They are also created by the built-in "slice()" function. +Added in version 3.15: The "slice()" type now supports subscription. +For example, "slice[float]" may be used in type annotations to +indicate a slice containing "float" objects. + Special read-only attributes: "start" is the lower bound; "stop" is the upper bound; "step" is the step value; each is "None" if omitted. These attributes can have any type. @@ -13838,13 +14283,15 @@ class instance has a namespace implemented as a dictionary which is See Function definitions for more information. ''', - 'typesmapping': r'''Mapping Types — "dict" -********************** + 'typesmapping': r'''Mapping types — "dict", "frozendict" +************************************ -A *mapping* object maps *hashable* values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard -mapping type, the *dictionary*. (For other containers see the built- -in "list", "set", and "tuple" classes, and the "collections" module.) +A *mapping* object maps *hashable* values to arbitrary objects. There +are currently two standard mapping types, the *dictionary* and +"frozendict". (For other containers see the built-in "list", "set", +and "tuple" classes, and the "collections" module.) See Time +complexity of operations on built-in types for the costs of the +various dictionary operations. A dictionary’s keys are *almost* arbitrary values. Values that are not *hashable*, that is, values containing lists, dictionaries or @@ -14121,10 +14568,10 @@ class dict(iterable, /, **kwargs) Changed in version 3.8: Dictionaries are now reversible. -See also: + See also: - "types.MappingProxyType" can be used to create a read-only view of a - "dict". + "frozendict" and "types.MappingProxyType" can be used to create a + read-only view of a "dict". See also: @@ -14234,13 +14681,65 @@ class dict(iterable, /, **kwargs) mappingproxy({'bacon': 1, 'spam': 500}) >>> values.mapping['spam'] 500 + + +Frozen dictionaries +=================== + +class frozendict(**kwargs) +class frozendict(mapping, /, **kwargs) +class frozendict(iterable, /, **kwargs) + + Return a new frozen dictionary initialized from an optional + positional argument and a possibly empty set of keyword arguments. + + A "frozendict" has a similar API to the "dict" API, with the + following differences: + + * "dict" has more methods than "frozendict": + + * "__delitem__()" + + * "__setitem__()" + + * "clear()" + + * "pop()" + + * "popitem()" + + * "setdefault()" + + * "update()" + + * A "frozendict" can be hashed with "hash(frozendict)" if all keys + and values can be hashed. + + * "frozendict |= other" does not modify the "frozendict" in-place + but creates a new frozen dictionary. + + "frozendict" is not a "dict" subclass but inherits directly from + "object". + + Like dictionaries, frozendicts are generic over two types, + signifying (respectively) the types of the frozendict’s keys and + values. + + classmethod fromkeys(iterable, value=None, /) + + Similar to "dict.fromkeys()", but call again the type + constructor with an initialized "frozendict" if the type is a + "frozendict" subclass or if the constructor returned a + "frozendict". + + Added in version 3.15. ''', 'typesmethods': r'''Methods ******* Methods are functions that are called using the attribute notation. There are two flavors: built-in methods (such as "append()" on lists) -and class instance method. Built-in methods are described with the +and class instance methods. Built-in methods are described with the types that support them. If you access a method (a function defined in a class namespace) @@ -14324,6 +14823,9 @@ class dict(iterable, /, **kwargs) operations have the same priority as the corresponding numeric operations. [3] +See Time complexity of operations on built-in types for the costs of +the various sequence operations. + +----------------------------+----------------------------------+------------+ | Operation | Result | Notes | |============================|==================================|============| @@ -14589,8 +15091,8 @@ class dict(iterable, /, **kwargs) sequence.pop(index=-1, /) - Retrieve the item at *index* and also removes it from *sequence*. - By default, the last item in *sequence* is removed and returned. + Retrieve the item at *index* and also remove it from *sequence*. By + default, the last item in *sequence* is removed and returned. sequence.remove(value, /) @@ -14948,8 +15450,8 @@ class range(start, stop, step=1, /) sequence.pop(index=-1, /) - Retrieve the item at *index* and also removes it from *sequence*. - By default, the last item in *sequence* is removed and returned. + Retrieve the item at *index* and also remove it from *sequence*. By + default, the last item in *sequence* is removed and returned. sequence.remove(value, /) diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 2379dad1976..d1df2ec42f0 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -132,7 +132,6 @@ def test_unquote(self): @support.requires_resource('cpu') def test_unquote_large(self): n = 10**6 - n = 10**4 # TODO: RUSTPYTHON; This takes more than 10 minutes to run. lower to 4 for encoded in r'\\', r'\134': with self.subTest(encoded): data = 'a="b=' + encoded*n + ';"' @@ -154,7 +153,8 @@ def test_load(self): self.assertEqual(C.output(['path']), 'Set-Cookie: Customer="WILE_E_COYOTE"; Path=/acme') cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme; Version=1', safe='', encoding='utf-8') - self.assertEqual(C.js_output(), fr""" + with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): + self.assertEqual(C.js_output(), fr""" """) cookie_encoded = urllib.parse.quote('Customer="WILE_E_COYOTE"; Path=/acme', safe='', encoding='utf-8') - self.assertEqual(C.js_output(['path']), fr""" + with self.assertWarnsRegex(DeprecationWarning, r"BaseCookie\.js_output"): + self.assertEqual(C.js_output(['path']), fr""" """ % (expected_encoded_cookie,) - self.assertEqual(M.js_output(), expected_js_output) + with self.assertWarnsRegex(DeprecationWarning, r"Morsel\.js_output"): + self.assertEqual(M.js_output(), expected_js_output) for i in ["foo bar", "foo@bar"]: # Try some illegal characters self.assertRaises(cookies.CookieError, @@ -652,7 +656,8 @@ def test_control_characters_output(self): cookie = cookies.SimpleCookie() cookie["cookie"] = morsel with self.assertRaises(cookies.CookieError): - cookie.js_output() + with self.assertWarnsRegex(DeprecationWarning, r"Morsel\.js_output"): + cookie.js_output() morsel = cookies.Morsel() morsel.set("key", "value", "coded-value") @@ -660,8 +665,29 @@ def test_control_characters_output(self): cookie = cookies.SimpleCookie() cookie["cookie"] = morsel with self.assertRaises(cookies.CookieError): - cookie.js_output() + with self.assertWarnsRegex(DeprecationWarning, r"Morsel\.js_output"): + cookie.js_output() + def test_morsel_js_output_deprecated(self): + morsel = cookies.Morsel() + morsel.set("key", "value", "value") + with self.assertWarnsRegex(DeprecationWarning, r"Morsel\.js_output") as cm: + result = morsel.js_output() + self.assertEqual(cm.filename, __file__) + self.assertIn("document.cookie", result) + + + def test_basecookie_js_output_warns_once(self): + C = cookies.SimpleCookie() + C["key"] = "value" + with self.assertWarns(DeprecationWarning) as cm: + C.js_output() + deprecation_warnings = [ + w for w in cm.warnings if issubclass(w.category, DeprecationWarning) + ] + self.assertEqual(len(deprecation_warnings), 1) + self.assertRegex(str(deprecation_warnings[0].message), r"BaseCookie\.js_output") + self.assertEqual(cm.filename, __file__) def load_tests(loader, tests, pattern): tests.addTest(doctest.DocTestSuite(cookies)) diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index ab06f3cf054..44c36013528 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -431,6 +431,81 @@ def test_headers_debuglevel(self): self.assertEqual(lines[2], "header: Second: val1") self.assertEqual(lines[3], "header: Second: val2") + def test_max_response_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = [f"Name{i}: Value{i}".encode() for i in range(max_headers)] + body = b"HTTP/1.1 200 OK\r\n" + b"\r\n".join(headers) + + with self.subTest(max_headers=None): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + with self.assertRaisesRegex( + client.HTTPException, f"got more than 100 headers" + ): + resp.begin() + + with self.subTest(max_headers=max_headers): + sock = FakeSocket(body) + resp = client.HTTPResponse(sock) + resp.begin(_max_headers=max_headers) + + def test_max_connection_headers(self): + max_headers = client._MAXHEADERS + 20 + headers = ( + f"Name{i}: Value{i}".encode() for i in range(max_headers - 1) + ) + body = ( + b"HTTP/1.1 200 OK\r\n" + + b"\r\n".join(headers) + + b"\r\nContent-Length: 12\r\n\r\nDummy body\r\n" + ) + + with self.subTest(max_headers=None): + conn = client.HTTPConnection("example.com") + conn.sock = FakeSocket(body) + conn.request("GET", "/") + with self.assertRaisesRegex( + client.HTTPException, f"got more than {client._MAXHEADERS} headers" + ): + response = conn.getresponse() + + with self.subTest(max_headers=None): + conn = client.HTTPConnection( + "example.com", max_response_headers=max_headers + ) + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + response.read() + + def test_max_connection_trailers(self): + # max_response_headers also limits trailer lines of a chunked + # response, which are read and discarded by read(). + max_trailers = client._MAXHEADERS + 20 + trailer_lines = "".join( + f"X-Trailer{i}: {i}\r\n" for i in range(max_trailers - 1) + ) + body = chunked_start + last_chunk + trailer_lines + chunked_end + + with self.subTest(max_response_headers=None): + conn = client.HTTPConnection("example.com") + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + response.read() + + with self.subTest(max_response_headers=max_trailers): + conn = client.HTTPConnection( + "example.com", max_response_headers=max_trailers + ) + conn.sock = FakeSocket(body) + conn.request("GET", "/") + response = conn.getresponse() + self.assertEqual(response.read(), chunked_expected) class HttpMethodTests(TestCase): def test_invalid_method_names(self): @@ -1332,6 +1407,35 @@ def test_overflowing_header_limit_after_100(self): self.assertIn('got more than ', str(cm.exception)) self.assertIn('headers', str(cm.exception)) + def test_too_many_interim_responses(self): + # A server streaming "100 Continue" responses forever must not + # hang getresponse(). + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' + * (client._MAXINTERIMRESPONSES + 1) + ) + resp = client.HTTPResponse(FakeSocket(body)) + with self.assertRaises(client.HTTPException) as cm: + resp.begin() + self.assertIn('got more than ', str(cm.exception)) + self.assertIn('interim responses', str(cm.exception)) + + def test_multiple_interim_responses(self): + # A reasonable number of interim responses before the final + # response is skipped as before. + body = ( + 'HTTP/1.1 100 Continue\r\n\r\n' * 3 + + 'HTTP/1.1 200 OK\r\n' + 'Content-Length: 5\r\n' + '\r\n' + 'hello' + ) + resp = client.HTTPResponse(FakeSocket(body), method="GET") + resp.begin() + self.assertEqual(resp.status, 200) + self.assertEqual(resp.read(), b'hello') + resp.close() + def test_overflowing_chunked_line(self): body = ( 'HTTP/1.1 200 OK\r\n' @@ -1403,6 +1507,35 @@ def test_chunked_trailers(self): self.assertEqual(sock.file.read(), b"") #we read to the end resp.close() + def test_chunked_too_many_trailers(self): + """A response streaming endless trailer lines must raise, not hang""" + too_many_trailers = "".join( + f"X-Trailer{i}: {i}\r\n" for i in range(client._MAXHEADERS + 1) + ) + # An unbounded read() reaches the trailers via the final 0 chunk. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read() + resp.close() + + # A bounded read(amt) larger than the body hits the same limit. + sock = FakeSocket( + chunked_start + last_chunk + too_many_trailers + chunked_end) + resp = client.HTTPResponse(sock, method="GET") + resp.begin() + with self.assertRaisesRegex( + client.HTTPException, + f"got more than {client._MAXHEADERS} trailers", + ): + resp.read(len(chunked_expected) + 1) + resp.close() + def test_chunked_sync(self): """Check that we don't read past the end of the chunked-encoding stream""" expected = chunked_expected diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index 7bf4e8279c3..70349c6c886 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -3,9 +3,9 @@ Written by Cody A.W. Somerville , Josip Dzolonga, and Michael Otteneder for the 2007/08 GHOP contest. """ -from collections import OrderedDict + from http.server import BaseHTTPRequestHandler, HTTPServer, HTTPSServer, \ - SimpleHTTPRequestHandler, CGIHTTPRequestHandler + SimpleHTTPRequestHandler from http import server, HTTPStatus import contextlib @@ -13,7 +13,6 @@ import socket import sys import re -import base64 import ntpath import pathlib import shutil @@ -22,18 +21,23 @@ import html import http, http.client import urllib.parse +import urllib.request import tempfile import time import datetime import threading from unittest import mock from io import BytesIO, StringIO +from _colorize import get_theme import unittest from test import support from test.support import ( - is_apple, import_helper, os_helper, requires_subprocess, threading_helper + force_not_colorized, + is_apple, import_helper, os_helper, threading_helper ) +from test.support.script_helper import kill_python, spawn_python +from test.support.socket_helper import find_unused_port try: import ssl @@ -382,6 +386,8 @@ def test_simple_get(self): def test_invalid_request(self): self.sock.send(b'POST /index.html\r\n') res = self.sock.recv(1024) + # The error response is not sent in the bare HTTP/0.9 style. + self.assertStartsWith(res, b'HTTP/1.0 400 ') self.assertIn(b"Bad HTTP/0.9 request type ('POST')", res) def test_single_request(self): @@ -478,6 +484,7 @@ def do_GET(self): def do_ERROR(self): self.send_error(HTTPStatus.NOT_FOUND, 'File not found') + @force_not_colorized def test_get(self): self.con = http.client.HTTPConnection(self.HOST, self.PORT) self.con.connect() @@ -488,6 +495,7 @@ def test_get(self): self.assertEndsWith(err.getvalue(), '"GET / HTTP/1.1" 200 -\n') + @force_not_colorized def test_err(self): self.con = http.client.HTTPConnection(self.HOST, self.PORT) self.con.connect() @@ -501,8 +509,49 @@ def test_err(self): self.assertEndsWith(lines[1], '"ERROR / HTTP/1.1" 404 -') +@support.force_colorized_test_class +class RequestHandlerColorizedLoggingTestCase(RequestHandlerLoggingTestCase): + + def test_get(self): + t = get_theme(force_color=True).http_server + self.con = http.client.HTTPConnection(self.HOST, self.PORT) + self.con.connect() + + with support.captured_stderr() as err: + self.con.request("GET", "/") + self.con.getresponse() + + output = err.getvalue() + self.assertIn(f"{t.path}/{t.reset}", output) + self.assertIn(f"{t.status_ok}200", output) + self.assertIn(t.reset, output) + + def test_err(self): + t = get_theme(force_color=True).http_server + self.con = http.client.HTTPConnection(self.HOST, self.PORT) + self.con.connect() + + with support.captured_stderr() as err: + self.con.request("ERROR", "/") + self.con.getresponse() + + lines = err.getvalue().split("\n") + self.assertIn( + f"{t.error}code 404, message File not found{t.reset}", lines[0] + ) + self.assertIn(f"{t.status_client_error}404", lines[1]) + + +class CustomHeaderSimpleHTTPRequestHandler(SimpleHTTPRequestHandler): + extra_response_headers = None + + def __init__(self, *args, **kwargs): + kwargs.setdefault('extra_response_headers', self.extra_response_headers) + super().__init__(*args, **kwargs) + + class SimpleHTTPServerTestCase(BaseTestCase): - class request_handler(NoLogRequestHandler, SimpleHTTPRequestHandler): + class request_handler(NoLogRequestHandler, CustomHeaderSimpleHTTPRequestHandler): pass def setUp(self): @@ -529,12 +578,12 @@ def setUp(self): def tearDown(self): try: os.chdir(self.cwd) - try: - shutil.rmtree(self.tempdir) - except: - pass finally: super().tearDown() + try: + shutil.rmtree(self.tempdir) + except: + pass def check_status_and_reason(self, response, status, data=None): def close_conn(): @@ -561,6 +610,15 @@ def close_conn(): reader.close() return body + def check_status_and_headers(self, response, status, headers=None): + # Drain the body so the server-side handler can close the file + # before tearDown removes the tempdir (matters on Windows). + response.read() + self.assertEqual(response.status, status) + if headers: + for name, value in headers.items(): + self.assertEqual(response.getheader(name), value) + def check_list_dir_dirname(self, dirname, quotedname=None): fullpath = os.path.join(self.tempdir, dirname) try: @@ -859,368 +917,66 @@ def test_path_without_leading_slash(self): self.assertEqual(response.getheader("Location"), self.tempdir_name + "/?hi=1") - -cgi_file1 = """\ -#!%s - -print("Content-type: text/html") -print() -print("Hello World") -""" - -cgi_file2 = """\ -#!%s -import os -import sys -import urllib.parse - -print("Content-type: text/html") -print() - -content_length = int(os.environ["CONTENT_LENGTH"]) -query_string = sys.stdin.buffer.read(content_length) -params = {key.decode("utf-8"): val.decode("utf-8") - for key, val in urllib.parse.parse_qsl(query_string)} - -print("%%s, %%s, %%s" %% (params["spam"], params["eggs"], params["bacon"])) -""" - -cgi_file4 = """\ -#!%s -import os - -print("Content-type: text/html") -print() - -print(os.environ["%s"]) -""" - -cgi_file6 = """\ -#!%s -import os - -print("X-ambv: was here") -print("Content-type: text/html") -print() -print("
")
-for k, v in os.environ.items():
-    try:
-        k.encode('ascii')
-        v.encode('ascii')
-    except UnicodeEncodeError:
-        continue  # see: BPO-44647
-    print(f"{k}={v}")
-print("
") -""" - -cgi_file7 = """\ -#!%s -import os -import sys - -print("Content-type: text/plain") -print() - -content_length = int(os.environ["CONTENT_LENGTH"]) -body = sys.stdin.buffer.read(content_length) - -print(f"{content_length} {len(body)}") -""" - - -@unittest.skipIf(hasattr(os, 'geteuid') and os.geteuid() == 0, - "This test can't be run reliably as root (issue #13308).") -@requires_subprocess() -class CGIHTTPServerTestCase(BaseTestCase): - class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler): - _test_case_self = None # populated by each setUp() method call. - - def __init__(self, *args, **kwargs): - with self._test_case_self.assertWarnsRegex( - DeprecationWarning, - r'http\.server\.CGIHTTPRequestHandler'): - # This context also happens to catch and silence the - # threading DeprecationWarning from os.fork(). - super().__init__(*args, **kwargs) - - linesep = os.linesep.encode('ascii') - - def setUp(self): - self.request_handler._test_case_self = self # practical, but yuck. - BaseTestCase.setUp(self) - self.cwd = os.getcwd() - self.parent_dir = tempfile.mkdtemp() - self.cgi_dir = os.path.join(self.parent_dir, 'cgi-bin') - self.cgi_child_dir = os.path.join(self.cgi_dir, 'child-dir') - self.sub_dir_1 = os.path.join(self.parent_dir, 'sub') - self.sub_dir_2 = os.path.join(self.sub_dir_1, 'dir') - self.cgi_dir_in_sub_dir = os.path.join(self.sub_dir_2, 'cgi-bin') - os.mkdir(self.cgi_dir) - os.mkdir(self.cgi_child_dir) - os.mkdir(self.sub_dir_1) - os.mkdir(self.sub_dir_2) - os.mkdir(self.cgi_dir_in_sub_dir) - self.nocgi_path = None - self.file1_path = None - self.file2_path = None - self.file3_path = None - self.file4_path = None - self.file5_path = None - self.file6_path = None - self.file7_path = None - - # The shebang line should be pure ASCII: use symlink if possible. - # See issue #7668. - self._pythonexe_symlink = None - if os_helper.can_symlink(): - self.pythonexe = os.path.join(self.parent_dir, 'python') - self._pythonexe_symlink = support.PythonSymlink(self.pythonexe).__enter__() - else: - self.pythonexe = sys.executable - - try: - # The python executable path is written as the first line of the - # CGI Python script. The encoding cookie cannot be used, and so the - # path should be encodable to the default script encoding (utf-8) - self.pythonexe.encode('utf-8') - except UnicodeEncodeError: - self.tearDown() - self.skipTest("Python executable path is not encodable to utf-8") - - self.nocgi_path = os.path.join(self.parent_dir, 'nocgi.py') - with open(self.nocgi_path, 'w', encoding='utf-8') as fp: - fp.write(cgi_file1 % self.pythonexe) - os.chmod(self.nocgi_path, 0o777) - - self.file1_path = os.path.join(self.cgi_dir, 'file1.py') - with open(self.file1_path, 'w', encoding='utf-8') as file1: - file1.write(cgi_file1 % self.pythonexe) - os.chmod(self.file1_path, 0o777) - - self.file2_path = os.path.join(self.cgi_dir, 'file2.py') - with open(self.file2_path, 'w', encoding='utf-8') as file2: - file2.write(cgi_file2 % self.pythonexe) - os.chmod(self.file2_path, 0o777) - - self.file3_path = os.path.join(self.cgi_child_dir, 'file3.py') - with open(self.file3_path, 'w', encoding='utf-8') as file3: - file3.write(cgi_file1 % self.pythonexe) - os.chmod(self.file3_path, 0o777) - - self.file4_path = os.path.join(self.cgi_dir, 'file4.py') - with open(self.file4_path, 'w', encoding='utf-8') as file4: - file4.write(cgi_file4 % (self.pythonexe, 'QUERY_STRING')) - os.chmod(self.file4_path, 0o777) - - self.file5_path = os.path.join(self.cgi_dir_in_sub_dir, 'file5.py') - with open(self.file5_path, 'w', encoding='utf-8') as file5: - file5.write(cgi_file1 % self.pythonexe) - os.chmod(self.file5_path, 0o777) - - self.file6_path = os.path.join(self.cgi_dir, 'file6.py') - with open(self.file6_path, 'w', encoding='utf-8') as file6: - file6.write(cgi_file6 % self.pythonexe) - os.chmod(self.file6_path, 0o777) - - self.file7_path = os.path.join(self.cgi_dir, 'file7.py') - with open(self.file7_path, 'w', encoding='utf-8') as file7: - file7.write(cgi_file7 % self.pythonexe) - os.chmod(self.file7_path, 0o777) - - os.chdir(self.parent_dir) - - def tearDown(self): - self.request_handler._test_case_self = None - try: - os.chdir(self.cwd) - if self._pythonexe_symlink: - self._pythonexe_symlink.__exit__(None, None, None) - if self.nocgi_path: - os.remove(self.nocgi_path) - if self.file1_path: - os.remove(self.file1_path) - if self.file2_path: - os.remove(self.file2_path) - if self.file3_path: - os.remove(self.file3_path) - if self.file4_path: - os.remove(self.file4_path) - if self.file5_path: - os.remove(self.file5_path) - if self.file6_path: - os.remove(self.file6_path) - if self.file7_path: - os.remove(self.file7_path) - os.rmdir(self.cgi_child_dir) - os.rmdir(self.cgi_dir) - os.rmdir(self.cgi_dir_in_sub_dir) - os.rmdir(self.sub_dir_2) - os.rmdir(self.sub_dir_1) - # The 'gmon.out' file can be written in the current working - # directory if C-level code profiling with gprof is enabled. - os_helper.unlink(os.path.join(self.parent_dir, 'gmon.out')) - os.rmdir(self.parent_dir) - finally: - BaseTestCase.tearDown(self) - - def test_url_collapse_path(self): - # verify tail is the last portion and head is the rest on proper urls - test_vectors = { - '': '//', - '..': IndexError, - '/.//..': IndexError, - '/': '//', - '//': '//', - '/\\': '//\\', - '/.//': '//', - 'cgi-bin/file1.py': '/cgi-bin/file1.py', - '/cgi-bin/file1.py': '/cgi-bin/file1.py', - 'a': '//a', - '/a': '//a', - '//a': '//a', - './a': '//a', - './C:/': '/C:/', - '/a/b': '/a/b', - '/a/b/': '/a/b/', - '/a/b/.': '/a/b/', - '/a/b/c/..': '/a/b/', - '/a/b/c/../d': '/a/b/d', - '/a/b/c/../d/e/../f': '/a/b/d/f', - '/a/b/c/../d/e/../../f': '/a/b/f', - '/a/b/c/../d/e/.././././..//f': '/a/b/f', - '../a/b/c/../d/e/.././././..//f': IndexError, - '/a/b/c/../d/e/../../../f': '/a/f', - '/a/b/c/../d/e/../../../../f': '//f', - '/a/b/c/../d/e/../../../../../f': IndexError, - '/a/b/c/../d/e/../../../../f/..': '//', - '/a/b/c/../d/e/../../../../f/../.': '//', - } - for path, expected in test_vectors.items(): - if isinstance(expected, type) and issubclass(expected, Exception): - self.assertRaises(expected, - server._url_collapse_path, path) - else: - actual = server._url_collapse_path(path) - self.assertEqual(expected, actual, - msg='path = %r\nGot: %r\nWanted: %r' % - (path, actual, expected)) - - def test_headers_and_content(self): - res = self.request('/cgi-bin/file1.py') - self.assertEqual( - (res.read(), res.getheader('Content-type'), res.status), - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK)) - - def test_issue19435(self): - res = self.request('///////////nocgi.py/../cgi-bin/nothere.sh') - self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - - def test_post(self): - params = urllib.parse.urlencode( - {'spam' : 1, 'eggs' : 'python', 'bacon' : 123456}) - headers = {'Content-type' : 'application/x-www-form-urlencoded'} - res = self.request('/cgi-bin/file2.py', 'POST', params, headers) - - self.assertEqual(res.read(), b'1, python, 123456' + self.linesep) - - def test_large_content_length(self): - for w in range(15, 25): - size = 1 << w - body = b'X' * size - headers = {'Content-Length' : str(size)} - res = self.request('/cgi-bin/file7.py', 'POST', body, headers) - self.assertEqual(res.read(), b'%d %d' % (size, size) + self.linesep) - - @unittest.skipIf(sys.platform == "darwin", "TODO: RUSTPYTHON; flaky: the 1ms handler timeout also bounds reading the request line") - def test_large_content_length_truncated(self): - with support.swap_attr(self.request_handler, 'timeout', 0.001): - for w in range(18, 65): - size = 1 << w - headers = {'Content-Length' : str(size)} - res = self.request('/cgi-bin/file1.py', 'POST', b'x', headers) - self.assertEqual(res.read(), b'Hello World' + self.linesep) - - def test_invaliduri(self): - res = self.request('/cgi-bin/invalid') - res.read() - self.assertEqual(res.status, HTTPStatus.NOT_FOUND) - - def test_authorization(self): - headers = {b'Authorization' : b'Basic ' + - base64.b64encode(b'username:pass')} - res = self.request('/cgi-bin/file1.py', 'GET', headers=headers) - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_no_leading_slash(self): - # http://bugs.python.org/issue2254 - res = self.request('cgi-bin/file1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_os_environ_is_not_altered(self): - signature = "Test CGI Server" - os.environ['SERVER_SOFTWARE'] = signature - res = self.request('/cgi-bin/file1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - self.assertEqual(os.environ['SERVER_SOFTWARE'], signature) - - def test_urlquote_decoding_in_cgi_check(self): - res = self.request('/cgi-bin%2ffile1.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_nested_cgi_path_issue21323(self): - res = self.request('/cgi-bin/child-dir/file3.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_query_with_multiple_question_mark(self): - res = self.request('/cgi-bin/file4.py?a=b?c=d') - self.assertEqual( - (b'a=b?c=d' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_query_with_continuous_slashes(self): - res = self.request('/cgi-bin/file4.py?k=aa%2F%2Fbb&//q//p//=//a//b//') - self.assertEqual( - (b'k=aa%2F%2Fbb&//q//p//=//a//b//' + self.linesep, - 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - - def test_cgi_path_in_sub_directories(self): - try: - CGIHTTPRequestHandler.cgi_directories.append('/sub/dir/cgi-bin') - res = self.request('/sub/dir/cgi-bin/file5.py') - self.assertEqual( - (b'Hello World' + self.linesep, 'text/html', HTTPStatus.OK), - (res.read(), res.getheader('Content-type'), res.status)) - finally: - CGIHTTPRequestHandler.cgi_directories.remove('/sub/dir/cgi-bin') - - def test_accept(self): - browser_accept = \ - 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' - tests = ( - ((('Accept', browser_accept),), browser_accept), - ((), ''), - # Hack case to get two values for the one header - ((('Accept', 'text/html'), ('ACCEPT', 'text/plain')), - 'text/html,text/plain'), - ) - for headers, expected in tests: - headers = OrderedDict(headers) - with self.subTest(headers): - res = self.request('/cgi-bin/file6.py', 'GET', headers=headers) - self.assertEqual(http.HTTPStatus.OK, res.status) - expected = f"HTTP_ACCEPT={expected}".encode('ascii') - self.assertIn(expected, res.read()) + def test_extra_response_headers_list_dir(self): + with mock.patch.object(self.request_handler, 'extra_response_headers', [ + ('X-Test1', 'test1'), + ('X-Test2', 'test2'), + ]): + response = self.request(self.base_url + '/') + self.check_status_and_headers(response, HTTPStatus.OK, { + "X-Test1": "test1", + "X-Test2": "test2", + }) + + def test_extra_response_headers_get_file(self): + with mock.patch.object(self.request_handler, 'extra_response_headers', [ + ('Set-Cookie', 'test1=value1'), + ('Set-Cookie', 'test2=value2'), + ('X-Test1', 'value3'), + ]): + data = b"Dummy index file\r\n" + with open(os.path.join(self.tempdir_name, 'index.html'), 'wb') as f: + f.write(data) + response = self.request(self.base_url + '/') + self.check_status_and_headers(response, HTTPStatus.OK, { + "Set-Cookie": "test1=value1, test2=value2", + "X-Test1": "value3", + }) + + def test_extra_response_headers_missing_on_404(self): + with mock.patch.object(self.request_handler, 'extra_response_headers', [ + ('X-Test1', 'value'), + ]): + response = self.request(self.base_url + '/missing.html') + self.check_status_and_headers(response, HTTPStatus.NOT_FOUND, { + "X-Test1": None, + }) + + def test_extra_response_headers_dont_overwrite_default_headers(self): + with mock.patch.object(self.request_handler, 'extra_response_headers', [ + ('Content-Type', 'test/not_allowed'), + ('Server', 'not_allowed'), + ('Set-Cookie', 'test=allowed'), + ]): + # The Content-Type header should not be overwritten by the extra_response_headers + # But cookies in the extra_allowed_duplicate_headers are allowed, + # including Set-Cookie + response = self.request(self.base_url + '/') + self.check_status_and_headers(response, HTTPStatus.OK, { + "Set-Cookie": "test=allowed", + }) + self.assertNotEqual(response.getheader("Content-Type"), 'test/not_allowed') + self.assertNotEqual(response.getheader("Server"), 'not_allowed') + + def test_multiple_requests_dont_duplicate_extra_response_headers(self): + with mock.patch.object(self.request_handler, 'extra_response_headers', [ + ('x-test', 'test-value'), + ]): + for _ in range(2): + response = self.request(self.base_url + '/') + self.check_status_and_headers(response, HTTPStatus.OK, { + "x-test": "test-value", + }) class SocketlessRequestHandler(SimpleHTTPRequestHandler): @@ -1242,6 +998,7 @@ def do_GET(self): def log_message(self, format, *args): pass + class RejectingSocketlessRequestHandler(SocketlessRequestHandler): def handle_expect_100(self): self.send_error(HTTPStatus.EXPECTATION_FAILED) @@ -1295,6 +1052,7 @@ def verify_http_server_response(self, response): match = self.HTTPResponseMatch.search(response) self.assertIsNotNone(match) + @force_not_colorized def test_unprintable_not_logged(self): # We call the method from the class directly as our Socketless # Handler subclass overrode it... nice for everything BUT this test. @@ -1346,6 +1104,19 @@ def test_http_0_9(self): self.assertEqual(result[0], b'Data\r\n') self.verify_get_called() + @support.subTests('request,code', [ + (b'GET / FUBAR\r\n\r\n', 400), # bad version + (b'GET / HTTP/2.0\r\n\r\n', 505), # unsupported version + (b'GET\r\n', 400), # bad syntax + (b'POST /\r\n', 400), # bad HTTP/0.9 request type + ]) + def test_request_line_error_has_status_line(self, request, code): + self.handler = SocketlessRequestHandler() + result = self.send_typical_request(request) + self.assertStartsWith(result[0], b'HTTP/1.1 %d ' % code) + self.verify_expected_headers(result[1:result.index(b'\r\n')]) + self.assertFalse(self.handler.get_called) + def test_extra_space(self): result = self.send_typical_request( b'GET /spaced out HTTP/1.1\r\n' @@ -1683,6 +1454,303 @@ def test_server_test_ipv4(self, _): self.assertEqual(mock_server.address_family, socket.AF_INET) +class CommandLineTestCase(unittest.TestCase): + default_port = 8000 + default_bind = None + default_protocol = 'HTTP/1.0' + default_handler = SimpleHTTPRequestHandler + default_server = unittest.mock.ANY + tls_cert = certdata_file('ssl_cert.pem') + tls_key = certdata_file('ssl_key.pem') + tls_password = 'somepass' + tls_cert_options = ['--tls-cert'] + tls_key_options = ['--tls-key'] + tls_password_options = ['--tls-password-file'] + args = { + 'HandlerClass': default_handler, + 'ServerClass': default_server, + 'protocol': default_protocol, + 'port': default_port, + 'bind': default_bind, + 'content_type': 'application/octet-stream', + 'tls_cert': None, + 'tls_key': None, + 'tls_password': None, + } + + def setUp(self): + super().setUp() + self.tls_password_file = tempfile.mktemp() + with open(self.tls_password_file, 'wb') as f: + f.write(self.tls_password.encode()) + self.addCleanup(os_helper.unlink, self.tls_password_file) + + def invoke_httpd(self, *args, stdout=None, stderr=None): + stdout = StringIO() if stdout is None else stdout + stderr = StringIO() if stderr is None else stderr + with contextlib.redirect_stdout(stdout), \ + contextlib.redirect_stderr(stderr): + server._main(args) + return stdout.getvalue(), stderr.getvalue() + + @mock.patch('http.server.test') + def test_port_flag(self, mock_func): + ports = [8000, 65535] + for port in ports: + with self.subTest(port=port): + self.invoke_httpd(str(port)) + call_args = self.args | dict(port=port) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_directory_flag(self, mock_func): + options = ['-d', '--directory'] + directories = ['.', '/foo', '\\bar', '/', + 'C:\\', 'C:\\foo', 'C:\\bar', + '/home/user', './foo/foo2', 'D:\\foo\\bar'] + for flag in options: + for directory in directories: + with self.subTest(flag=flag, directory=directory): + self.invoke_httpd(flag, directory) + mock_func.assert_called_once_with(**self.args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_bind_flag(self, mock_func): + options = ['-b', '--bind'] + bind_addresses = ['localhost', '127.0.0.1', '::1', + '0.0.0.0', '8.8.8.8'] + for flag in options: + for bind_address in bind_addresses: + with self.subTest(flag=flag, bind_address=bind_address): + self.invoke_httpd(flag, bind_address) + call_args = self.args | dict(bind=bind_address) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_protocol_flag(self, mock_func): + options = ['-p', '--protocol'] + protocols = ['HTTP/1.0', 'HTTP/1.1', 'HTTP/2.0', 'HTTP/3.0'] + for flag in options: + for protocol in protocols: + with self.subTest(flag=flag, protocol=protocol): + self.invoke_httpd(flag, protocol) + call_args = self.args | dict(protocol=protocol) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_content_type_flag(self, mock_func): + content_types = ['text/html', 'text/plain', 'application/json'] + for content_type in content_types: + with self.subTest(content_type=content_type): + self.invoke_httpd('--content-type', content_type) + call_args = self.args | dict(content_type=content_type) + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_header_flag(self, mock_func): + call_args = self.args + self.invoke_httpd('--header', 'h1', 'v1', '-H', 'h2', 'v2') + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + def test_extra_header_flag_too_few_args(self): + with self.assertRaises(SystemExit): + self.invoke_httpd('--header', 'h1') + + def test_extra_header_flag_too_many_args(self): + with self.assertRaises(SystemExit): + self.invoke_httpd('--header', 'h1', 'v1', 'h2') + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_tls_cert_and_key_flags(self, mock_func): + for tls_cert_option in self.tls_cert_options: + for tls_key_option in self.tls_key_options: + self.invoke_httpd(tls_cert_option, self.tls_cert, + tls_key_option, self.tls_key) + call_args = self.args | { + 'tls_cert': self.tls_cert, + 'tls_key': self.tls_key, + } + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_tls_cert_and_key_and_password_flags(self, mock_func): + for tls_cert_option in self.tls_cert_options: + for tls_key_option in self.tls_key_options: + for tls_password_option in self.tls_password_options: + self.invoke_httpd(tls_cert_option, + self.tls_cert, + tls_key_option, + self.tls_key, + tls_password_option, + self.tls_password_file) + call_args = self.args | { + 'tls_cert': self.tls_cert, + 'tls_key': self.tls_key, + 'tls_password': self.tls_password, + } + mock_func.assert_called_once_with(**call_args) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_missing_tls_cert_flag(self, mock_func): + for tls_key_option in self.tls_key_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_key_option, self.tls_key) + mock_func.reset_mock() + + for tls_password_option in self.tls_password_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_password_option, self.tls_password) + mock_func.reset_mock() + + @unittest.skipIf(ssl is None, "requires ssl") + @mock.patch('http.server.test') + def test_invalid_password_file(self, mock_func): + non_existent_file = 'non_existent_file' + for tls_password_option in self.tls_password_options: + for tls_cert_option in self.tls_cert_options: + with self.assertRaises(SystemExit): + self.invoke_httpd(tls_cert_option, + self.tls_cert, + tls_password_option, + non_existent_file) + + @mock.patch('http.server.test') + def test_no_arguments(self, mock_func): + self.invoke_httpd() + mock_func.assert_called_once_with(**self.args) + mock_func.reset_mock() + + @mock.patch('http.server.test') + def test_help_flag(self, _): + options = ['-h', '--help'] + for option in options: + stdout, stderr = StringIO(), StringIO() + with self.assertRaises(SystemExit): + self.invoke_httpd(option, stdout=stdout, stderr=stderr) + self.assertIn('usage', stdout.getvalue()) + self.assertEqual(stderr.getvalue(), '') + + @mock.patch('http.server.test') + def test_unknown_flag(self, _): + stdout, stderr = StringIO(), StringIO() + with self.assertRaises(SystemExit): + self.invoke_httpd('--unknown-flag', stdout=stdout, stderr=stderr) + self.assertEqual(stdout.getvalue(), '') + self.assertIn('error', stderr.getvalue()) + + @mock.patch('http.server.test') + def test_extra_response_headers_arg(self, mock_test): + # Call the main function with extra response headers cli args + server._main( + ['-H', 'Set-Cookie', 'k=v', '-H', 'Set-Cookie', 'k2=v2:v3 v4', '8080'] + ) + # Get the ServerClass (DualStackServerMixin subclass) that _main() + # passed to test(), and verify its finish_request passes + # extra_response_headers to the handler. + _, kwargs = mock_test.call_args + server_class = kwargs['ServerClass'] + + mock_handler_class = mock.MagicMock() + mock_server = mock.Mock() + mock_server.RequestHandlerClass = mock_handler_class + server_class.finish_request(mock_server, mock.Mock(), '127.0.0.1') + mock_handler_class.assert_called_once_with( + mock.ANY, mock.ANY, mock_server, + directory=mock.ANY, + extra_response_headers=[ + ['Set-Cookie', 'k=v'], ['Set-Cookie', 'k2=v2:v3 v4'] + ] + ) + + +class CommandLineRunTimeTestCase(unittest.TestCase): + served_data = os.urandom(32) + served_filename = 'served_filename' + tls_cert = certdata_file('ssl_cert.pem') + tls_key = certdata_file('ssl_key.pem') + tls_password = b'somepass' + tls_password_file = 'ssl_key_password' + + def setUp(self): + super().setUp() + server_dir_context = os_helper.temp_cwd() + server_dir = self.enterContext(server_dir_context) + with open(self.served_filename, 'wb') as f: + f.write(self.served_data) + with open(self.tls_password_file, 'wb') as f: + f.write(self.tls_password) + + def fetch_file(self, path, context=None): + req = urllib.request.Request(path, method='GET') + with urllib.request.urlopen(req, context=context) as res: + return res.read() + + def parse_cli_output(self, output): + match = re.search(r'Serving (HTTP|HTTPS) on (.+) port (\d+)', output) + if match is None: + return None, None, None + return match.group(1).lower(), match.group(2), int(match.group(3)) + + def wait_for_server(self, proc, protocol, bind, port): + """Check that the server has been successfully started.""" + line = proc.stdout.readline().strip() + if support.verbose: + print() + print('python -m http.server: ', line) + return self.parse_cli_output(line) == (protocol, bind, port) + + def test_http_client(self): + bind, port = '127.0.0.1', find_unused_port() + proc = spawn_python('-u', '-m', 'http.server', str(port), '-b', bind, + bufsize=1, text=True) + self.addCleanup(kill_python, proc) + self.addCleanup(proc.terminate) + self.assertTrue(self.wait_for_server(proc, 'http', bind, port)) + res = self.fetch_file(f'http://{bind}:{port}/{self.served_filename}') + self.assertEqual(res, self.served_data) + + @unittest.skipIf(ssl is None, "requires ssl") + def test_https_client(self): + context = ssl.create_default_context() + # allow self-signed certificates + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + + bind, port = '127.0.0.1', find_unused_port() + proc = spawn_python('-u', '-m', 'http.server', str(port), '-b', bind, + '--tls-cert', self.tls_cert, + '--tls-key', self.tls_key, + '--tls-password-file', self.tls_password_file, + bufsize=1, text=True) + self.addCleanup(kill_python, proc) + self.addCleanup(proc.terminate) + self.assertTrue(self.wait_for_server(proc, 'https', bind, port)) + url = f'https://{bind}:{port}/{self.served_filename}' + res = self.fetch_file(url, context=context) + self.assertEqual(res, self.served_data) + + +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(http.server, "__version__") + self.assertEqual(cm.filename, __file__) + + def setUpModule(): unittest.addModuleCleanup(os.chdir, os.getcwd()) diff --git a/Lib/test/test_pkgutil.py b/Lib/test/test_pkgutil.py index d4faaaeca00..4623b7eb443 100644 --- a/Lib/test/test_pkgutil.py +++ b/Lib/test/test_pkgutil.py @@ -1,3 +1,5 @@ +import logging +import logging.handlers from pathlib import Path from test.support.import_helper import unload from test.support.warnings_helper import check_warnings @@ -232,9 +234,6 @@ def test_walk_packages_raises_on_string_or_bytes_input(self): list(pkgutil.walk_packages(bytes_input)) def test_name_resolution(self): - import logging - import logging.handlers - success_cases = ( ('os', os), ('os.path', os.path), @@ -322,6 +321,53 @@ def test_name_resolution(self): with self.assertRaises(exc): pkgutil.resolve_name(s) + def test_name_resolution_strict(self): + # PEP 829: strict=True accepts only the pkg.mod:callable form + # (W(.W)*:W(.W)*) -- both the colon and the callable are required. + success_cases = ( + ('os.path:pathsep', os.path.pathsep), + ('logging.handlers:SysLogHandler', + logging.handlers.SysLogHandler), + ('logging.handlers:SysLogHandler.LOG_ALERT', + logging.handlers.SysLogHandler.LOG_ALERT), + ('builtins:int', int), + ('builtins:int.from_bytes', int.from_bytes), + ('os:path', os.path), + ) + + # All of these are accepted under strict=False but must be + # rejected under strict=True. + failure_cases = ( + 'os', # no colon (non-strict form) + 'os.path', # no colon + 'logging:', # colon, empty callable + 'os.foo:', # colon, empty callable + ':int', # empty package + 'os.path:join:extra', # extra colon + 'os.path.9abc:join', # invalid identifier in package + 'os.path:9abc', # invalid identifier in callable + '', # empty + '?abc:foo', # invalid character + ) + + for s, expected in success_cases: + with self.subTest(s=s): + self.assertEqual( + pkgutil.resolve_name(s, strict=True), expected) + + for s in failure_cases: + with self.subTest(s=s): + with self.assertRaises(ValueError): + pkgutil.resolve_name(s, strict=True) + + # Cache independence: a strict=True call must not poison + # strict=False (and vice versa). Exercise both orderings. + self.assertEqual( + pkgutil.resolve_name('os:path', strict=True), os.path) + self.assertEqual(pkgutil.resolve_name('os.path'), os.path) + self.assertEqual( + pkgutil.resolve_name('os:path', strict=True), os.path) + def test_name_resolution_import_rebinding(self): # The same data is also used for testing import in test_import and # mock.patch in test_unittest. diff --git a/Lib/test/test_pydoc/test_pydoc.py b/Lib/test/test_pydoc/test_pydoc.py index bac96087a64..b0b219cfab4 100644 --- a/Lib/test/test_pydoc/test_pydoc.py +++ b/Lib/test/test_pydoc/test_pydoc.py @@ -8,10 +8,7 @@ import pydoc import py_compile import keyword -try: # TODO: RUSTPYTHON; Implement `_pickle` - import _pickle -except ImportError: - _pickle = None +import _pickle import pkgutil import re import tempfile @@ -581,7 +578,7 @@ class object # of the known subclasses of object. (doc.docclass() used to # fail if HeapType was imported before running this test, like # when running tests sequentially.) - from _testcapi import HeapType + from _testcapi import HeapType # noqa: F401 except ImportError: pass text = doc.docclass(object) @@ -1009,6 +1006,8 @@ def test_synopsis_sourceless(self): os = import_helper.import_fresh_module('os') expected = os.__doc__.splitlines()[0] filename = os.__spec__.cached + if not filename: + raise unittest.SkipTest('requires .pyc files') synopsis = pydoc.synopsis(filename) self.assertEqual(synopsis, expected) @@ -1016,10 +1015,10 @@ def test_synopsis_sourceless(self): def test_synopsis_sourceless_empty_doc(self): with os_helper.temp_cwd() as test_dir: init_path = os.path.join(test_dir, 'foomod42.py') - cached_path = importlib.util.cache_from_source(init_path) + cached_path = init_path + 'c' with open(init_path, 'w') as fobj: fobj.write("foo = 1") - py_compile.compile(init_path) + py_compile.compile(init_path, cached_path) synopsis = pydoc.synopsis(init_path, {}) self.assertIsNone(synopsis) synopsis_cached = pydoc.synopsis(cached_path, {}) @@ -1363,47 +1362,6 @@ def test_url_search_package_error(self): finally: sys.path[:] = saved_paths - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules(self): - # See Helper.listmodules(). - num_header_lines = 2 - num_module_lines_min = 5 # Playing it safe. - num_footer_lines = 3 - expected = num_header_lines + num_module_lines_min + num_footer_lines - - output = StringIO() - helper = pydoc.Helper(output=output) - helper('modules') - result = output.getvalue().strip() - num_lines = len(result.splitlines()) - - self.assertGreaterEqual(num_lines, expected) - - @unittest.skip('causes undesirable side-effects (#20128)') - def test_modules_search(self): - # See Helper.listmodules(). - expected = 'pydoc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules pydoc') - result = help_io.getvalue() - - self.assertIn(expected, result) - - @unittest.skip('some buildbots are not cooperating (#20128)') - def test_modules_search_builtin(self): - expected = 'gc - ' - - output = StringIO() - helper = pydoc.Helper(output=output) - with captured_stdout() as help_io: - helper('modules garbage') - result = help_io.getvalue() - - self.assertStartsWith(result, expected) - def test_importfile(self): try: loaded_pydoc = pydoc.importfile(pydoc.__file__) @@ -1483,7 +1441,7 @@ def test_special_form(self): self.assertIn('NoReturn = typing.NoReturn', doc) self.assertIn(typing.NoReturn.__doc__.strip().splitlines()[0], doc) else: - self.assertIn('NoReturn = class _SpecialForm(_Final)', doc) + self.assertIn('NoReturn = class _SpecialForm(_Final, _NotIterable)', doc) def test_typing_pydoc(self): def foo(data: typing.List[typing.Any], @@ -2215,7 +2173,7 @@ def mock_getline(prompt): def test_keywords(self): self.assertEqual(sorted(pydoc.Helper.keywords), - sorted(keyword.kwlist + ['case', 'match'])) + sorted(keyword.kwlist + ['case', 'match', 'lazy'])) def test_interact_empty_line_continues(self): # gh-138568: test pressing Enter without input should continue in help session @@ -2414,6 +2372,32 @@ def test_sys_path_adjustment_when_curdir_already_included(self): trailing_argv0dir = trailing_curdir + [self.argv0dir] self.assertIsNone(self._get_revised_path(trailing_argv0dir)) + def test__get_version(self): + import json + import warnings + + class MyModule: + __name__ = 'my_module' + + @property + def __version__(self): + warnings._deprecated("__version__", remove=(3, 20)) + return "1.2.3" + + module = MyModule() + doc = pydoc.Doc() + with warnings.catch_warnings(record=True) as w: # TODO: remove in 3.20 + warnings.simplefilter("always") + version = doc._get_version(json) + self.assertEqual(version, "2.0.9") + self.assertEqual(len(w), 0) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + version = doc._get_version(module) + self.assertEqual(version, "1.2.3") + self.assertEqual(len(w), 1) + def setUpModule(): thread_info = threading_helper.threading_setup() diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 1e5f79998e7..ab59727a8fd 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -807,6 +807,61 @@ def test_short_content_raises_ContentTooShortError_without_reporthook(self): self.unfakehttp() +class urlcleanup_Tests(unittest.TestCase, FakeHTTPMixin): + """Test urllib.request.urlcleanup()""" + + def setUp(self): + self.addCleanup(urllib.request.urlcleanup) + + def urlretrieve(self): + self.fakehttp(b'HTTP/1.1 200 OK\r\n\r\ndata') + try: + filename, headers = urllib.request.urlretrieve( + support.TEST_HTTP_URL) + finally: + self.unfakehttp() + self.addCleanup(os_helper.unlink, filename) + return filename + + def fake_urlopen(self, data): + self.fakehttp(b'HTTP/1.1 200 OK\r\n\r\n' + data) + try: + with urllib.request.urlopen(support.TEST_HTTP_URL) as fp: + return fp.read() + finally: + self.unfakehttp() + + def test_temporary_files(self): + filename = self.urlretrieve() + self.assertTrue(os.path.exists(filename)) + + urllib.request.urlcleanup() + self.assertFalse(os.path.exists(filename)) + + # A file created after the cleanup is not deleted. + os_helper.create_empty_file(filename) + urllib.request.urlcleanup() + self.assertTrue(os.path.exists(filename)) + + def test_opener(self): + # The implicitly created opener supports http. + self.assertEqual(self.fake_urlopen(b'first'), b'first') + + # An installed opener replaces it and supports only its handlers. + opener = urllib.request.OpenerDirector() + opener.add_handler(urllib.request.DataHandler()) + opener.add_handler(urllib.request.UnknownHandler()) + urllib.request.install_opener(opener) + with urllib.request.urlopen('data:,hello') as fp: + self.assertEqual(fp.read(), b'hello') + with self.assertRaises(urllib.error.URLError): + self.fake_urlopen(b'') + + # urlcleanup() resets the opener. + urllib.request.urlcleanup() + self.assertEqual(self.fake_urlopen(b'second'), b'second') + + class QuotingTests(unittest.TestCase): r"""Tests for urllib.quote() and urllib.quote_plus() diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index eeea9cda2f4..7efbc81a160 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -270,6 +270,50 @@ def test_password_manager_default_port(self): self.assertEqual(find_user_pass("i", "http://j.example.com:80"), (None, None)) + def test_password_manager_scheme(self): + mgr = urllib.request.HTTPPasswordMgr() + mgr.add_password( + "realm", "https://example.com/", "user", "password") + + self.assertEqual( + mgr.find_user_password("realm", "https://example.com/"), + ("user", "password")) + self.assertEqual( + mgr.find_user_password("realm", "http://example.com/"), + (None, None)) + # Support an authority without a scheme. + self.assertEqual( + mgr.find_user_password("realm", "example.com"), + ("user", "password")) + # An authority without a scheme continues to match any scheme. + mgr.add_password( + "realm", "schemeless.example.com", "user", "password") + for scheme in "http", "https": + with self.subTest(scheme=scheme): + self.assertEqual( + mgr.find_user_password( + "realm", f"{scheme}://schemeless.example.com/"), + ("user", "password")) + + # A network-path reference also has no scheme. + mgr.add_password( + "realm", "//network-path.example.com/", "user", "password") + self.assertEqual( + mgr.find_user_password( + "realm", "https://network-path.example.com/"), + ("user", "password")) + + def test_password_manager_reduced_uri(self): + mgr = urllib.request.HTTPPasswordMgr() + + self.assertEqual( + mgr.reduce_uri("http://example.com/path"), + ("example.com:80", "/path")) + self.assertTrue( + mgr.is_suburi( + ("example.com", "/path"), + ("example.com", "/path/subpath"))) + class MockOpener: addheaders = [] @@ -577,6 +621,23 @@ class NonHandler(object): self.assertRaises(TypeError, OpenerDirector().add_handler, NonHandler()) + def test_no_protocol_methods(self): + # test the case that methods starts with handler type without the protocol + # like open*() or _open*(). + # These methods should be ignored + + o = OpenerDirector() + meth_spec = [ + ["open"], + ["_open"], + ["error"] + ] + + add_ordered_mock_handlers(o, meth_spec) + + self.assertEqual(len(o.handle_open), 0) + self.assertEqual(len(o.handle_error), 0) + def test_badly_named_methods(self): # test work-around for three methods that accidentally follow the # naming conventions for handler methods @@ -1808,6 +1869,18 @@ def test_basic_prior_auth_auto_send(self): # expect request to be sent with auth header self.assertTrue(http_handler.has_auth_header) + def test_basic_prior_auth_different_scheme(self): + pwd_manager = HTTPPasswordMgrWithPriorAuth() + auth_handler = HTTPBasicAuthHandler(pwd_manager) + auth_handler.add_password( + None, "https://example.com/", "user", "password", + is_authenticated=True) + + request = Request("http://example.com/") + auth_handler.http_request(request) + + self.assertFalse(request.has_header("Authorization")) + def test_basic_prior_auth_send_after_first_success(self): # Auto send auth header after authentication is successful once diff --git a/Lib/test/test_urlparse.py b/Lib/test/test_urlparse.py index 98f0b190ee7..a5b7966c778 100644 --- a/Lib/test/test_urlparse.py +++ b/Lib/test/test_urlparse.py @@ -1,7 +1,10 @@ +import copy +import pickle import sys import unicodedata import unittest import urllib.parse +from urllib.parse import urldefrag, urlparse, urlsplit, urlunparse, urlunsplit from test import support RFC1808_BASE = "http://a/b/c/d;p?q#f" @@ -107,19 +110,46 @@ class UrlParseTestCase(unittest.TestCase): def checkRoundtrips(self, url, parsed, split, url2=None): if url2 is None: url2 = url - result = urllib.parse.urlparse(url) + self.checkRoundtrips1(url, parsed, split, missing_as_none=True) + empty = url[:0] + parsed = tuple(x or empty for x in parsed) + split = tuple(x or empty for x in split) + self.checkRoundtrips1(url, parsed, split, url2, missing_as_none=False) + + result = urlparse(url, missing_as_none=True) + self.assertEqual(urlunparse(result, keep_empty=False), url2) + self.assertEqual(urlunparse(tuple(result), keep_empty=False), url2) + result = urlparse(url, missing_as_none=False) + with self.assertRaises(ValueError): + urlunparse(result, keep_empty=True) + urlunparse(tuple(result), keep_empty=True) + + result = urlsplit(url, missing_as_none=True) + self.assertEqual(urlunsplit(result, keep_empty=False), url2) + self.assertEqual(urlunsplit(tuple(result), keep_empty=False), url2) + result = urlsplit(url, missing_as_none=False) + with self.assertRaises(ValueError): + urlunsplit(result, keep_empty=True) + urlunsplit(tuple(result), keep_empty=True) + + def checkRoundtrips1(self, url, parsed, split, url2=None, *, missing_as_none): + if url2 is None: + url2 = url + result = urlparse(url, missing_as_none=missing_as_none) self.assertSequenceEqual(result, parsed) t = (result.scheme, result.netloc, result.path, result.params, result.query, result.fragment) self.assertSequenceEqual(t, parsed) # put it back together and it should be the same - result2 = urllib.parse.urlunparse(result) - self.assertSequenceEqual(result2, url2) - self.assertSequenceEqual(result2, result.geturl()) + result2 = urlunparse(result) + self.assertEqual(result2, url2) + self.assertEqual(result2, result.geturl()) + self.assertEqual(urlunparse(result, keep_empty=missing_as_none), url2) + self.assertEqual(urlunparse(tuple(result), keep_empty=missing_as_none), result2) # the result of geturl() is a fixpoint; we can always parse it # again to get the same result: - result3 = urllib.parse.urlparse(result.geturl()) + result3 = urlparse(result.geturl(), missing_as_none=missing_as_none) self.assertEqual(result3.geturl(), result.geturl()) self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) @@ -134,17 +164,18 @@ def checkRoundtrips(self, url, parsed, split, url2=None): self.assertEqual(result3.port, result.port) # check the roundtrip using urlsplit() as well - result = urllib.parse.urlsplit(url) + result = urlsplit(url, missing_as_none=missing_as_none) self.assertSequenceEqual(result, split) t = (result.scheme, result.netloc, result.path, - result.query, result.fragment) + result.query, result.fragment) self.assertSequenceEqual(t, split) - result2 = urllib.parse.urlunsplit(result) - self.assertSequenceEqual(result2, url2) - self.assertSequenceEqual(result2, result.geturl()) + result2 = urlunsplit(result) + self.assertEqual(result2, url2) + self.assertEqual(result2, result.geturl()) + self.assertEqual(urlunsplit(tuple(result), keep_empty=missing_as_none), result2) # check the fixpoint property of re-parsing the result of geturl() - result3 = urllib.parse.urlsplit(result.geturl()) + result3 = urlsplit(result.geturl(), missing_as_none=missing_as_none) self.assertEqual(result3.geturl(), result.geturl()) self.assertSequenceEqual(result3, result) self.assertEqual(result3.scheme, result.scheme) @@ -177,94 +208,94 @@ def test_qs(self, orig, expect): @support.subTests('bytes', (False, True)) @support.subTests('url,parsed,split', [ ('path/to/file', - ('', '', 'path/to/file', '', '', ''), - ('', '', 'path/to/file', '', '')), + (None, None, 'path/to/file', None, None, None), + (None, None, 'path/to/file', None, None)), ('/path/to/file', - ('', '', '/path/to/file', '', '', ''), - ('', '', '/path/to/file', '', '')), + (None, None, '/path/to/file', None, None, None), + (None, None, '/path/to/file', None, None)), ('//path/to/file', - ('', 'path', '/to/file', '', '', ''), - ('', 'path', '/to/file', '', '')), + (None, 'path', '/to/file', None, None, None), + (None, 'path', '/to/file', None, None)), ('////path/to/file', - ('', '', '//path/to/file', '', '', ''), - ('', '', '//path/to/file', '', '')), + (None, '', '//path/to/file', None, None, None), + (None, '', '//path/to/file', None, None)), ('/////path/to/file', - ('', '', '///path/to/file', '', '', ''), - ('', '', '///path/to/file', '', '')), + (None, '', '///path/to/file', None, None, None), + (None, '', '///path/to/file', None, None)), ('scheme:path/to/file', - ('scheme', '', 'path/to/file', '', '', ''), - ('scheme', '', 'path/to/file', '', '')), + ('scheme', None, 'path/to/file', None, None, None), + ('scheme', None, 'path/to/file', None, None)), ('scheme:/path/to/file', - ('scheme', '', '/path/to/file', '', '', ''), - ('scheme', '', '/path/to/file', '', '')), + ('scheme', None, '/path/to/file', None, None, None), + ('scheme', None, '/path/to/file', None, None)), ('scheme://path/to/file', - ('scheme', 'path', '/to/file', '', '', ''), - ('scheme', 'path', '/to/file', '', '')), + ('scheme', 'path', '/to/file', None, None, None), + ('scheme', 'path', '/to/file', None, None)), ('scheme:////path/to/file', - ('scheme', '', '//path/to/file', '', '', ''), - ('scheme', '', '//path/to/file', '', '')), + ('scheme', '', '//path/to/file', None, None, None), + ('scheme', '', '//path/to/file', None, None)), ('scheme://///path/to/file', - ('scheme', '', '///path/to/file', '', '', ''), - ('scheme', '', '///path/to/file', '', '')), + ('scheme', '', '///path/to/file', None, None, None), + ('scheme', '', '///path/to/file', None, None)), ('file:tmp/junk.txt', - ('file', '', 'tmp/junk.txt', '', '', ''), - ('file', '', 'tmp/junk.txt', '', '')), + ('file', None, 'tmp/junk.txt', None, None, None), + ('file', None, 'tmp/junk.txt', None, None)), ('file:///tmp/junk.txt', - ('file', '', '/tmp/junk.txt', '', '', ''), - ('file', '', '/tmp/junk.txt', '', '')), + ('file', '', '/tmp/junk.txt', None, None, None), + ('file', '', '/tmp/junk.txt', None, None)), ('file:////tmp/junk.txt', - ('file', '', '//tmp/junk.txt', '', '', ''), - ('file', '', '//tmp/junk.txt', '', '')), + ('file', '', '//tmp/junk.txt', None, None, None), + ('file', '', '//tmp/junk.txt', None, None)), ('file://///tmp/junk.txt', - ('file', '', '///tmp/junk.txt', '', '', ''), - ('file', '', '///tmp/junk.txt', '', '')), + ('file', '', '///tmp/junk.txt', None, None, None), + ('file', '', '///tmp/junk.txt', None, None)), ('http:tmp/junk.txt', - ('http', '', 'tmp/junk.txt', '', '', ''), - ('http', '', 'tmp/junk.txt', '', '')), + ('http', None, 'tmp/junk.txt', None, None, None), + ('http', None, 'tmp/junk.txt', None, None)), ('http://example.com/tmp/junk.txt', - ('http', 'example.com', '/tmp/junk.txt', '', '', ''), - ('http', 'example.com', '/tmp/junk.txt', '', '')), + ('http', 'example.com', '/tmp/junk.txt', None, None, None), + ('http', 'example.com', '/tmp/junk.txt', None, None)), ('http:///example.com/tmp/junk.txt', - ('http', '', '/example.com/tmp/junk.txt', '', '', ''), - ('http', '', '/example.com/tmp/junk.txt', '', '')), + ('http', '', '/example.com/tmp/junk.txt', None, None, None), + ('http', '', '/example.com/tmp/junk.txt', None, None)), ('http:////example.com/tmp/junk.txt', - ('http', '', '//example.com/tmp/junk.txt', '', '', ''), - ('http', '', '//example.com/tmp/junk.txt', '', '')), + ('http', '', '//example.com/tmp/junk.txt', None, None, None), + ('http', '', '//example.com/tmp/junk.txt', None, None)), ('imap://mail.python.org/mbox1', - ('imap', 'mail.python.org', '/mbox1', '', '', ''), - ('imap', 'mail.python.org', '/mbox1', '', '')), + ('imap', 'mail.python.org', '/mbox1', None, None, None), + ('imap', 'mail.python.org', '/mbox1', None, None)), ('mms://wms.sys.hinet.net/cts/Drama/09006251100.asf', ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf', - '', '', ''), + None, None, None), ('mms', 'wms.sys.hinet.net', '/cts/Drama/09006251100.asf', - '', '')), + None, None)), ('nfs://server/path/to/file.txt', - ('nfs', 'server', '/path/to/file.txt', '', '', ''), - ('nfs', 'server', '/path/to/file.txt', '', '')), + ('nfs', 'server', '/path/to/file.txt', None, None, None), + ('nfs', 'server', '/path/to/file.txt', None, None)), ('svn+ssh://svn.zope.org/repos/main/ZConfig/trunk/', ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', - '', '', ''), + None, None, None), ('svn+ssh', 'svn.zope.org', '/repos/main/ZConfig/trunk/', - '', '')), + None, None)), ('git+ssh://git@github.com/user/project.git', ('git+ssh', 'git@github.com','/user/project.git', - '','',''), + None,None,None), ('git+ssh', 'git@github.com','/user/project.git', - '', '')), + None, None)), ('itms-services://?action=download-manifest&url=https://example.com/app', - ('itms-services', '', '', '', - 'action=download-manifest&url=https://example.com/app', ''), + ('itms-services', '', '', None, + 'action=download-manifest&url=https://example.com/app', None), ('itms-services', '', '', - 'action=download-manifest&url=https://example.com/app', '')), + 'action=download-manifest&url=https://example.com/app', None)), ('+scheme:path/to/file', - ('', '', '+scheme:path/to/file', '', '', ''), - ('', '', '+scheme:path/to/file', '', '')), + (None, None, '+scheme:path/to/file', None, None, None), + (None, None, '+scheme:path/to/file', None, None)), ('sch_me:path/to/file', - ('', '', 'sch_me:path/to/file', '', '', ''), - ('', '', 'sch_me:path/to/file', '', '')), + (None, None, 'sch_me:path/to/file', None, None, None), + (None, None, 'sch_me:path/to/file', None, None)), ('schème:path/to/file', - ('', '', 'schème:path/to/file', '', '', ''), - ('', '', 'schème:path/to/file', '', '')), + (None, None, 'schème:path/to/file', None, None, None), + (None, None, 'schème:path/to/file', None, None)), ]) def test_roundtrips(self, bytes, url, parsed, split): if bytes: @@ -279,24 +310,24 @@ def test_roundtrips(self, bytes, url, parsed, split): @support.subTests('url,url2,parsed,split', [ ('///path/to/file', '/path/to/file', - ('', '', '/path/to/file', '', '', ''), - ('', '', '/path/to/file', '', '')), + (None, '', '/path/to/file', None, None, None), + (None, '', '/path/to/file', None, None)), ('scheme:///path/to/file', 'scheme:/path/to/file', - ('scheme', '', '/path/to/file', '', '', ''), - ('scheme', '', '/path/to/file', '', '')), + ('scheme', '', '/path/to/file', None, None, None), + ('scheme', '', '/path/to/file', None, None)), ('file:/tmp/junk.txt', 'file:///tmp/junk.txt', - ('file', '', '/tmp/junk.txt', '', '', ''), - ('file', '', '/tmp/junk.txt', '', '')), + ('file', None, '/tmp/junk.txt', None, None, None), + ('file', None, '/tmp/junk.txt', None, None)), ('http:/tmp/junk.txt', 'http:///tmp/junk.txt', - ('http', '', '/tmp/junk.txt', '', '', ''), - ('http', '', '/tmp/junk.txt', '', '')), + ('http', None, '/tmp/junk.txt', None, None, None), + ('http', None, '/tmp/junk.txt', None, None)), ('https:/tmp/junk.txt', 'https:///tmp/junk.txt', - ('https', '', '/tmp/junk.txt', '', '', ''), - ('https', '', '/tmp/junk.txt', '', '')), + ('https', None, '/tmp/junk.txt', None, None, None), + ('https', None, '/tmp/junk.txt', None, None)), ]) def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): if bytes: @@ -310,17 +341,17 @@ def test_roundtrips_normalization(self, bytes, url, url2, parsed, split): @support.subTests('scheme', ('http', 'https')) @support.subTests('url,parsed,split', [ ('://www.python.org', - ('www.python.org', '', '', '', ''), - ('www.python.org', '', '', '')), + ('www.python.org', '', None, None, None), + ('www.python.org', '', None, None)), ('://www.python.org#abc', - ('www.python.org', '', '', '', 'abc'), - ('www.python.org', '', '', 'abc')), + ('www.python.org', '', None, None, 'abc'), + ('www.python.org', '', None, 'abc')), ('://www.python.org?q=abc', - ('www.python.org', '', '', 'q=abc', ''), - ('www.python.org', '', 'q=abc', '')), + ('www.python.org', '', None, 'q=abc', None), + ('www.python.org', '', 'q=abc', None)), ('://www.python.org/#abc', - ('www.python.org', '/', '', '', 'abc'), - ('www.python.org', '/', '', 'abc')), + ('www.python.org', '/', None, None, 'abc'), + ('www.python.org', '/', None, 'abc')), ('://a/b/c/d;p?q#f', ('a', '/b/c/d', 'p', 'q', 'f'), ('a', '/b/c/d;p', 'q', 'f')), @@ -342,16 +373,21 @@ def test_http_roundtrips(self, bytes, scheme, url, parsed, split): def checkJoin(self, base, relurl, expected, *, relroundtrip=True): with self.subTest(base=base, relurl=relurl): self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - baseb = base.encode('ascii') - relurlb = relurl.encode('ascii') - expectedb = expected.encode('ascii') + baseb = str_encode(base) + relurlb = str_encode(relurl) + expectedb = str_encode(expected) self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) if relroundtrip: - relurl = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurl)) - self.assertEqual(urllib.parse.urljoin(base, relurl), expected) - relurlb = urllib.parse.urlunsplit(urllib.parse.urlsplit(relurlb)) - self.assertEqual(urllib.parse.urljoin(baseb, relurlb), expectedb) + relurl2 = urlunsplit(urlsplit(relurl)) + self.assertEqual(urllib.parse.urljoin(base, relurl2), expected) + relurlb2 = urlunsplit(urlsplit(relurlb)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb2), expectedb) + + relurl3 = urlunsplit(urlsplit(relurl, missing_as_none=True)) + self.assertEqual(urllib.parse.urljoin(base, relurl3), expected) + relurlb3 = urlunsplit(urlsplit(relurlb, missing_as_none=True)) + self.assertEqual(urllib.parse.urljoin(baseb, relurlb3), expectedb) @support.subTests('bytes', (False, True)) @support.subTests('u', ['Python', './Python','x-newscheme://foo.com/stuff','x://y','x:/y','x:/','/',]) @@ -387,7 +423,7 @@ def test_RFC1808(self): self.checkJoin(RFC1808_BASE, '../../g', 'http://a/g') # "abnormal" cases from RFC 1808: - self.checkJoin(RFC1808_BASE, '', 'http://a/b/c/d;p?q#f') + self.checkJoin(RFC1808_BASE, None, 'http://a/b/c/d;p?q#f') self.checkJoin(RFC1808_BASE, 'g.', 'http://a/b/c/g.') self.checkJoin(RFC1808_BASE, '.g', 'http://a/b/c/.g') self.checkJoin(RFC1808_BASE, 'g..', 'http://a/b/c/g..') @@ -411,8 +447,10 @@ def test_RFC1808(self): def test_RFC2368(self): # Issue 11467: path that starts with a number is not parsed correctly - self.assertEqual(urllib.parse.urlparse('mailto:1337@example.org'), + self.assertEqual(urlparse('mailto:1337@example.org'), ('mailto', '', '1337@example.org', '', '', '')) + self.assertEqual(urlparse('mailto:1337@example.org', missing_as_none=True), + ('mailto', None, '1337@example.org', None, None, None)) def test_RFC2396(self): # cases from RFC 2396 @@ -741,18 +779,18 @@ def test_RFC2732_invalid(self, bytes, invalid_url): @support.subTests('bytes', (False, True)) @support.subTests('url,defrag,frag', [ ('http://python.org#frag', 'http://python.org', 'frag'), - ('http://python.org', 'http://python.org', ''), + ('http://python.org', 'http://python.org', None), ('http://python.org/#frag', 'http://python.org/', 'frag'), - ('http://python.org/', 'http://python.org/', ''), + ('http://python.org/', 'http://python.org/', None), ('http://python.org/?q#frag', 'http://python.org/?q', 'frag'), - ('http://python.org/?q', 'http://python.org/?q', ''), + ('http://python.org/?q', 'http://python.org/?q', None), ('http://python.org/p#frag', 'http://python.org/p', 'frag'), - ('http://python.org/p?q', 'http://python.org/p?q', ''), + ('http://python.org/p?q', 'http://python.org/p?q', None), (RFC1808_BASE, 'http://a/b/c/d;p?q', 'f'), - (RFC2396_BASE, 'http://a/b/c/d;p?q', ''), + (RFC2396_BASE, 'http://a/b/c/d;p?q', None), ('http://a/b/c;p?q#f', 'http://a/b/c;p?q', 'f'), ('http://a/b/c;p?q#', 'http://a/b/c;p?q', ''), - ('http://a/b/c;p?q', 'http://a/b/c;p?q', ''), + ('http://a/b/c;p?q', 'http://a/b/c;p?q', None), ('http://a/b/c;p?#f', 'http://a/b/c;p?', 'f'), ('http://a/b/c;p#f', 'http://a/b/c;p', 'f'), ('http://a/b/c;?q#f', 'http://a/b/c;?q', 'f'), @@ -764,14 +802,19 @@ def test_RFC2732_invalid(self, bytes, invalid_url): ('//a/b/c;p?q#f', '//a/b/c;p?q', 'f'), ('://a/b/c;p?q#f', '://a/b/c;p?q', 'f'), ]) - def test_urldefrag(self, bytes, url, defrag, frag): + @support.subTests('missing_as_none', (False, True)) + def test_urldefrag(self, bytes, url, defrag, frag, missing_as_none): if bytes: url = str_encode(url) defrag = str_encode(defrag) frag = str_encode(frag) - result = urllib.parse.urldefrag(url) - hash = '#' if isinstance(url, str) else b'#' - self.assertEqual(result.geturl(), url.rstrip(hash)) + result = urllib.parse.urldefrag(url, missing_as_none=missing_as_none) + if not missing_as_none: + hash = '#' if isinstance(url, str) else b'#' + url = url.rstrip(hash) + if frag is None: + frag = url[:0] + self.assertEqual(result.geturl(), url) self.assertEqual(result, (defrag, frag)) self.assertEqual(result.url, defrag) self.assertEqual(result.fragment, frag) @@ -1001,26 +1044,27 @@ def test_attributes_bad_scheme(self, bytes, parse, scheme): if not url.isascii(): self.skipTest('non-ASCII bytes') url = url.encode("ascii") - p = parse(url) - self.assertEqual(p.scheme, b"" if bytes else "") + p = parse(url, missing_as_none=True) + self.assertIsNone(p.scheme) - def test_attributes_without_netloc(self): + @support.subTests('missing_as_none', (False, True)) + def test_attributes_without_netloc(self, missing_as_none): # This example is straight from RFC 3261. It looks like it # should allow the username, hostname, and port to be filled # in, but doesn't. Since it's a URI and doesn't use the # scheme://netloc syntax, the netloc and related attributes # should be left empty. uri = "sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15" - p = urllib.parse.urlsplit(uri) - self.assertEqual(p.netloc, "") + p = urllib.parse.urlsplit(uri, missing_as_none=missing_as_none) + self.assertEqual(p.netloc, None if missing_as_none else "") self.assertEqual(p.username, None) self.assertEqual(p.password, None) self.assertEqual(p.hostname, None) self.assertEqual(p.port, None) self.assertEqual(p.geturl(), uri) - p = urllib.parse.urlparse(uri) - self.assertEqual(p.netloc, "") + p = urllib.parse.urlparse(uri, missing_as_none=missing_as_none) + self.assertEqual(p.netloc, None if missing_as_none else "") self.assertEqual(p.username, None) self.assertEqual(p.password, None) self.assertEqual(p.hostname, None) @@ -1029,16 +1073,16 @@ def test_attributes_without_netloc(self): # You guessed it, repeating the test with bytes input uri = b"sip:alice@atlanta.com;maddr=239.255.255.1;ttl=15" - p = urllib.parse.urlsplit(uri) - self.assertEqual(p.netloc, b"") + p = urllib.parse.urlsplit(uri, missing_as_none=missing_as_none) + self.assertEqual(p.netloc, None if missing_as_none else b"") self.assertEqual(p.username, None) self.assertEqual(p.password, None) self.assertEqual(p.hostname, None) self.assertEqual(p.port, None) self.assertEqual(p.geturl(), uri) - p = urllib.parse.urlparse(uri) - self.assertEqual(p.netloc, b"") + p = urllib.parse.urlparse(uri, missing_as_none=missing_as_none) + self.assertEqual(p.netloc, None if missing_as_none else b"") self.assertEqual(p.username, None) self.assertEqual(p.password, None) self.assertEqual(p.hostname, None) @@ -1052,67 +1096,86 @@ def test_noslash(self): self.assertEqual(urllib.parse.urlparse(b"http://example.com?blahblah=/foo"), (b'http', b'example.com', b'', b'', b'blahblah=/foo', b'')) - def test_withoutscheme(self): + @support.subTests('missing_as_none', (False, True)) + def test_withoutscheme(self, missing_as_none): # Test urlparse without scheme # Issue 754016: urlparse goes wrong with IP:port without scheme # RFC 1808 specifies that netloc should start with //, urlparse expects # the same, otherwise it classifies the portion of url as path. - self.assertEqual(urllib.parse.urlparse("path"), - ('','','path','','','')) - self.assertEqual(urllib.parse.urlparse("//www.python.org:80"), - ('','www.python.org:80','','','','')) - self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"), - ('http','www.python.org:80','','','','')) + none = None if missing_as_none else '' + self.assertEqual(urlparse("path", missing_as_none=missing_as_none), + (none, none, 'path', none, none, none)) + self.assertEqual(urlparse("//www.python.org:80", missing_as_none=missing_as_none), + (none, 'www.python.org:80', '', none, none, none)) + self.assertEqual(urlparse("http://www.python.org:80", missing_as_none=missing_as_none), + ('http', 'www.python.org:80', '', none, none, none)) # Repeat for bytes input - self.assertEqual(urllib.parse.urlparse(b"path"), - (b'',b'',b'path',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"//www.python.org:80"), - (b'',b'www.python.org:80',b'',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"), - (b'http',b'www.python.org:80',b'',b'',b'',b'')) - - def test_portseparator(self): + none = None if missing_as_none else b'' + self.assertEqual(urlparse(b"path", missing_as_none=missing_as_none), + (none, none, b'path', none, none, none)) + self.assertEqual(urlparse(b"//www.python.org:80", missing_as_none=missing_as_none), + (none, b'www.python.org:80', b'', none, none, none)) + self.assertEqual(urlparse(b"http://www.python.org:80", missing_as_none=missing_as_none), + (b'http', b'www.python.org:80', b'', none, none, none)) + + @support.subTests('missing_as_none', (False, True)) + def test_portseparator(self, missing_as_none): # Issue 754016 makes changes for port separator ':' from scheme separator - self.assertEqual(urllib.parse.urlparse("http:80"), ('http','','80','','','')) - self.assertEqual(urllib.parse.urlparse("https:80"), ('https','','80','','','')) - self.assertEqual(urllib.parse.urlparse("path:80"), ('path','','80','','','')) - self.assertEqual(urllib.parse.urlparse("http:"),('http','','','','','')) - self.assertEqual(urllib.parse.urlparse("https:"),('https','','','','','')) - self.assertEqual(urllib.parse.urlparse("http://www.python.org:80"), - ('http','www.python.org:80','','','','')) + none = None if missing_as_none else '' + self.assertEqual(urlparse("http:80", missing_as_none=missing_as_none), + ('http', none, '80', none, none, none)) + self.assertEqual(urlparse("https:80", missing_as_none=missing_as_none), + ('https', none, '80', none, none, none)) + self.assertEqual(urlparse("path:80", missing_as_none=missing_as_none), + ('path', none, '80', none, none, none)) + self.assertEqual(urlparse("http:", missing_as_none=missing_as_none), + ('http', none, '', none, none, none)) + self.assertEqual(urlparse("https:", missing_as_none=missing_as_none), + ('https', none, '', none, none, none)) + self.assertEqual(urlparse("http://www.python.org:80", missing_as_none=missing_as_none), + ('http', 'www.python.org:80', '', none, none, none)) # As usual, need to check bytes input as well - self.assertEqual(urllib.parse.urlparse(b"http:80"), (b'http',b'',b'80',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"https:80"), (b'https',b'',b'80',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"path:80"), (b'path',b'',b'80',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"http:"),(b'http',b'',b'',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"https:"),(b'https',b'',b'',b'',b'',b'')) - self.assertEqual(urllib.parse.urlparse(b"http://www.python.org:80"), - (b'http',b'www.python.org:80',b'',b'',b'',b'')) + none = None if missing_as_none else b'' + self.assertEqual(urlparse(b"http:80", missing_as_none=missing_as_none), + (b'http', none, b'80', none, none, none)) + self.assertEqual(urlparse(b"https:80", missing_as_none=missing_as_none), + (b'https', none, b'80', none, none, none)) + self.assertEqual(urlparse(b"path:80", missing_as_none=missing_as_none), + (b'path', none, b'80', none, none, none)) + self.assertEqual(urlparse(b"http:", missing_as_none=missing_as_none), + (b'http', none, b'', none, none, none)) + self.assertEqual(urlparse(b"https:", missing_as_none=missing_as_none), + (b'https', none, b'', none, none, none)) + self.assertEqual(urlparse(b"http://www.python.org:80", missing_as_none=missing_as_none), + (b'http', b'www.python.org:80', b'', none, none, none)) def test_usingsys(self): # Issue 3314: sys module is used in the error self.assertRaises(TypeError, urllib.parse.urlencode, "foo") - def test_anyscheme(self): + @support.subTests('missing_as_none', (False, True)) + def test_anyscheme(self, missing_as_none): # Issue 7904: s3://foo.com/stuff has netloc "foo.com". - self.assertEqual(urllib.parse.urlparse("s3://foo.com/stuff"), - ('s3', 'foo.com', '/stuff', '', '', '')) - self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff"), - ('x-newscheme', 'foo.com', '/stuff', '', '', '')) - self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query#fragment"), - ('x-newscheme', 'foo.com', '/stuff', '', 'query', 'fragment')) - self.assertEqual(urllib.parse.urlparse("x-newscheme://foo.com/stuff?query"), - ('x-newscheme', 'foo.com', '/stuff', '', 'query', '')) + none = None if missing_as_none else '' + self.assertEqual(urlparse("s3://foo.com/stuff", missing_as_none=missing_as_none), + ('s3', 'foo.com', '/stuff', none, none, none)) + self.assertEqual(urlparse("x-newscheme://foo.com/stuff", missing_as_none=missing_as_none), + ('x-newscheme', 'foo.com', '/stuff', none, none, none)) + self.assertEqual(urlparse("x-newscheme://foo.com/stuff?query#fragment", missing_as_none=missing_as_none), + ('x-newscheme', 'foo.com', '/stuff', none, 'query', 'fragment')) + self.assertEqual(urlparse("x-newscheme://foo.com/stuff?query", missing_as_none=missing_as_none), + ('x-newscheme', 'foo.com', '/stuff', none, 'query', none)) # And for bytes... - self.assertEqual(urllib.parse.urlparse(b"s3://foo.com/stuff"), - (b's3', b'foo.com', b'/stuff', b'', b'', b'')) - self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff"), - (b'x-newscheme', b'foo.com', b'/stuff', b'', b'', b'')) - self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query#fragment"), - (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'fragment')) - self.assertEqual(urllib.parse.urlparse(b"x-newscheme://foo.com/stuff?query"), - (b'x-newscheme', b'foo.com', b'/stuff', b'', b'query', b'')) + none = None if missing_as_none else b'' + self.assertEqual(urlparse(b"s3://foo.com/stuff", missing_as_none=missing_as_none), + (b's3', b'foo.com', b'/stuff', none, none, none)) + self.assertEqual(urlparse(b"x-newscheme://foo.com/stuff", missing_as_none=missing_as_none), + (b'x-newscheme', b'foo.com', b'/stuff', none, none, none)) + self.assertEqual(urlparse(b"x-newscheme://foo.com/stuff?query#fragment", missing_as_none=missing_as_none), + (b'x-newscheme', b'foo.com', b'/stuff', none, b'query', b'fragment')) + self.assertEqual(urlparse(b"x-newscheme://foo.com/stuff?query", missing_as_none=missing_as_none), + (b'x-newscheme', b'foo.com', b'/stuff', none, b'query', none)) @support.subTests('func', (urllib.parse.urlparse, urllib.parse.urlsplit)) def test_default_scheme(self, func): @@ -1125,8 +1188,11 @@ def test_default_scheme(self, func): self.assertEqual(func("path", scheme="ftp").scheme, "ftp") self.assertEqual(func(b"path", scheme=b"ftp").scheme, b"ftp") self.assertEqual(func("path").scheme, "") + self.assertEqual(func("path", missing_as_none=True).scheme, None) self.assertEqual(func(b"path").scheme, b"") + self.assertEqual(func(b"path", missing_as_none=True).scheme, None) self.assertEqual(func(b"path", "").scheme, b"") + self.assertEqual(func(b"path", "", missing_as_none=True).scheme, b"") @support.subTests('url,attr,expected_frag', ( ("http:#frag", "path", "frag"), @@ -1151,9 +1217,16 @@ def test_parse_fragments(self, url, attr, expected_frag, func): "#" + expected_frag) self.assertEqual(func(url, "", False).fragment, "") + result = func(url, allow_fragments=False, missing_as_none=True) + self.assertIsNone(result.fragment) + self.assertTrue( + getattr(result, attr).endswith("#" + expected_frag)) + self.assertIsNone(func(url, "", False, missing_as_none=True).fragment) + result = func(url, allow_fragments=True) self.assertEqual(result.fragment, expected_frag) - self.assertNotEndsWith(getattr(result, attr), expected_frag) + self.assertFalse( + getattr(result, attr).endswith(expected_frag)) self.assertEqual(func(url, "", True).fragment, expected_frag) self.assertEqual(func(url).fragment, expected_frag) @@ -1182,19 +1255,10 @@ def test_mixed_types_rejected(self): with self.assertRaisesRegex(TypeError, "Cannot mix str"): urllib.parse.urljoin(b"http://python.org", "http://python.org") - @support.subTests('result_type', [ - urllib.parse.DefragResult, - urllib.parse.SplitResult, - urllib.parse.ParseResult, - ]) - def test_result_pairs(self, result_type): - # Check encoding and decoding between result pairs - str_type = result_type - num_args = len(str_type._fields) + def _check_result_type(self, str_type, str_args): bytes_type = str_type._encoded_counterpart self.assertIs(bytes_type._decoded_counterpart, str_type) - str_args = ('',) * num_args - bytes_args = (b'',) * num_args + bytes_args = tuple_encode(str_args) str_result = str_type(*str_args) bytes_result = bytes_type(*bytes_args) encoding = 'ascii' @@ -1213,6 +1277,169 @@ def test_result_pairs(self, result_type): self.assertEqual(str_result.encode(encoding), bytes_result) self.assertEqual(str_result.encode(encoding, errors), bytes_args) self.assertEqual(str_result.encode(encoding, errors), bytes_result) + for result in str_result, bytes_result: + self.assertEqual(copy.copy(result), result) + self.assertEqual(copy.deepcopy(result), result) + self.assertEqual(copy.replace(result), result) + self.assertEqual(result._replace(), result) + + def test_result_pairs__(self): + # Check encoding and decoding between result pairs + self._check_result_type(urllib.parse.DefragResult, ('', '')) + self._check_result_type(urllib.parse.DefragResult, ('', None)) + self._check_result_type(urllib.parse.SplitResult, ('', '', '', '', '')) + self._check_result_type(urllib.parse.SplitResult, (None, None, '', None, None)) + self._check_result_type(urllib.parse.ParseResult, ('', '', '', '', '', '')) + self._check_result_type(urllib.parse.ParseResult, (None, None, '', None, None, None)) + + def test_result_encoding_decoding(self): + def check(str_result, bytes_result): + self.assertEqual(str_result.encode(), bytes_result) + self.assertEqual(str_result.encode().geturl(), bytes_result.geturl()) + self.assertEqual(bytes_result.decode(), str_result) + self.assertEqual(bytes_result.decode().geturl(), str_result.geturl()) + + url = 'http://example.com/?#' + burl = url.encode() + for func in urldefrag, urlsplit, urlparse: + check(func(url, missing_as_none=True), func(burl, missing_as_none=True)) + check(func(url), func(burl)) + + def test_result_copying(self): + def check(result): + result2 = copy.copy(result) + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + result2 = copy.deepcopy(result) + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + result2 = copy.replace(result) + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + result2 = result._replace() + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + + url = 'http://example.com/?#' + burl = url.encode() + for func in urldefrag, urlsplit, urlparse: + check(func(url)) + check(func(url, missing_as_none=True)) + check(func(burl)) + check(func(burl, missing_as_none=True)) + + def test_result_pickling(self): + def check(result): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + pickled = pickle.dumps(result, proto) + result2 = pickle.loads(pickled) + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + + url = 'http://example.com/?#' + burl = url.encode() + for func in urldefrag, urlsplit, urlparse: + check(func(url)) + check(func(url, missing_as_none=True)) + check(func(burl)) + check(func(burl, missing_as_none=True)) + + def test_result_compat_unpickling(self): + def check(result, pickles): + for pickled in pickles: + result2 = pickle.loads(pickled) + self.assertEqual(result2, result) + self.assertEqual(result2.geturl(), result.geturl()) + + url = 'http://example.com/?#' + burl = url.encode() + # Pre-3.15 data. + check(urldefrag(url), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResult\nc__builtin__\ntuple\n(Vhttp://example.com/?\nV\nttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResult\nc__builtin__\ntuple\n(X\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00ttR.', + b'\x80\x02curlparse\nDefragResult\nX\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00\x86\x81.', + b'\x80\x03curllib.parse\nDefragResult\nX\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00\x86\x81.', + b'\x80\x04\x958\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0cDefragResult\x93\x8c\x14http://example.com/?\x8c\x00\x86\x81.', + )) + check(urldefrag(burl), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\n(Vhttp://example.com/?\nVlatin1\ntRc__builtin__\nbytes\n(tRttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\n(X\x14\x00\x00\x00http://example.com/?X\x06\x00\x00\x00latin1tRc__builtin__\nbytes\n)RttR.', + b'\x80\x02curlparse\nDefragResultBytes\nc_codecs\nencode\nX\x14\x00\x00\x00http://example.com/?X\x06\x00\x00\x00latin1\x86Rc__builtin__\nbytes\n)R\x86\x81.', + b'\x80\x03curllib.parse\nDefragResultBytes\nC\x14http://example.com/?C\x00\x86\x81.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x11DefragResultBytes\x93C\x14http://example.com/?C\x00\x86\x81.', + )) + check(urlsplit(url), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResult\nc__builtin__\ntuple\n(Vhttp\nVexample.com\nV/\nV\np0\ng0\nttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResult\nc__builtin__\ntuple\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00ttR.', + b'\x80\x02curlparse\nSplitResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00t\x81.', + b'\x80\x03curllib.parse\nSplitResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00t\x81.', + b'\x80\x04\x95;\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0bSplitResult\x93(\x8c\x04http\x8c\x0bexample.com\x8c\x01/\x8c\x00\x94h\x00t\x81.', + )) + check(urlsplit(burl), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\np0\n(Vhttp\nVlatin1\np1\ntRg0\n(Vexample.com\ng1\ntRg0\n(V/\ng1\ntRc__builtin__\nbytes\n(tRp2\ng2\nttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\nq\x00(X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01tRh\x00(X\x0b\x00\x00\x00example.comh\x01tRh\x00(X\x01\x00\x00\x00/h\x01tRc__builtin__\nbytes\n)Rq\x02h\x02ttR.', + b'\x80\x02curlparse\nSplitResultBytes\n(c_codecs\nencode\nq\x00X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01\x86Rh\x00X\x0b\x00\x00\x00example.comh\x01\x86Rh\x00X\x01\x00\x00\x00/h\x01\x86Rc__builtin__\nbytes\n)Rq\x02h\x02t\x81.', + b'\x80\x03curllib.parse\nSplitResultBytes\n(C\x04httpC\x0bexample.comC\x01/C\x00q\x00h\x00t\x81.', + b'\x80\x04\x95@\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x10SplitResultBytes\x93(C\x04httpC\x0bexample.comC\x01/C\x00\x94h\x00t\x81.', + )) + check(urlparse(url), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResult\nc__builtin__\ntuple\n(Vhttp\nVexample.com\nV/\nV\np0\ng0\ng0\nttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResult\nc__builtin__\ntuple\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00h\x00ttR.', + b'\x80\x02curlparse\nParseResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00h\x00t\x81.', + b'\x80\x03curllib.parse\nParseResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00h\x00t\x81.', + b'\x80\x04\x95=\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0bParseResult\x93(\x8c\x04http\x8c\x0bexample.com\x8c\x01/\x8c\x00\x94h\x00h\x00t\x81.', + )) + check(urlparse(burl), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\np0\n(Vhttp\nVlatin1\np1\ntRg0\n(Vexample.com\ng1\ntRg0\n(V/\ng1\ntRc__builtin__\nbytes\n(tRp2\ng2\ng2\nttR.', + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\nq\x00(X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01tRh\x00(X\x0b\x00\x00\x00example.comh\x01tRh\x00(X\x01\x00\x00\x00/h\x01tRc__builtin__\nbytes\n)Rq\x02h\x02h\x02ttR.', + b'\x80\x02curlparse\nParseResultBytes\n(c_codecs\nencode\nq\x00X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01\x86Rh\x00X\x0b\x00\x00\x00example.comh\x01\x86Rh\x00X\x01\x00\x00\x00/h\x01\x86Rc__builtin__\nbytes\n)Rq\x02h\x02h\x02t\x81.', + b'\x80\x03curllib.parse\nParseResultBytes\n(C\x04httpC\x0bexample.comC\x01/C\x00q\x00h\x00h\x00t\x81.', + b'\x80\x04\x95B\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x10ParseResultBytes\x93(C\x04httpC\x0bexample.comC\x01/C\x00\x94h\x00h\x00t\x81.', + )) + + # 3.15 data with missing_as_none=True. + check(urldefrag(url, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResult\nc__builtin__\ntuple\n(Vhttp://example.com/?\nV\nttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResult\nc__builtin__\ntuple\n(X\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00ttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nDefragResult\nX\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00\x86\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nDefragResult\nX\x14\x00\x00\x00http://example.com/?X\x00\x00\x00\x00\x86\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95K\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0cDefragResult\x93\x8c\x14http://example.com/?\x8c\x00\x86\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) + check(urldefrag(burl, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\n(Vhttp://example.com/?\nVlatin1\ntRc__builtin__\nbytes\n(tRttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nDefragResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\n(X\x14\x00\x00\x00http://example.com/?X\x06\x00\x00\x00latin1tRc__builtin__\nbytes\n)RttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nDefragResultBytes\nc_codecs\nencode\nX\x14\x00\x00\x00http://example.com/?X\x06\x00\x00\x00latin1\x86Rc__builtin__\nbytes\n)R\x86\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nDefragResultBytes\nC\x14http://example.com/?C\x00\x86\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95P\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x11DefragResultBytes\x93C\x14http://example.com/?C\x00\x86\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) + check(urlsplit(url, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResult\nc__builtin__\ntuple\n(Vhttp\nVexample.com\nV/\nV\np0\ng0\nttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResult\nc__builtin__\ntuple\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00ttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nSplitResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nSplitResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/X\x00\x00\x00\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95N\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0bSplitResult\x93(\x8c\x04http\x8c\x0bexample.com\x8c\x01/\x8c\x00\x94h\x00t\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) + check(urlsplit(burl, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\np0\n(Vhttp\nVlatin1\np1\ntRg0\n(Vexample.com\ng1\ntRg0\n(V/\ng1\ntRc__builtin__\nbytes\n(tRp2\ng2\nttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nSplitResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\nq\x00(X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01tRh\x00(X\x0b\x00\x00\x00example.comh\x01tRh\x00(X\x01\x00\x00\x00/h\x01tRc__builtin__\nbytes\n)Rq\x02h\x02ttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nSplitResultBytes\n(c_codecs\nencode\nq\x00X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01\x86Rh\x00X\x0b\x00\x00\x00example.comh\x01\x86Rh\x00X\x01\x00\x00\x00/h\x01\x86Rc__builtin__\nbytes\n)Rq\x02h\x02t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nSplitResultBytes\n(C\x04httpC\x0bexample.comC\x01/C\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95S\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x10SplitResultBytes\x93(C\x04httpC\x0bexample.comC\x01/C\x00\x94h\x00t\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) + check(urlparse(url, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResult\nc__builtin__\ntuple\n(Vhttp\nVexample.com\nV/\nNV\np0\ng0\nttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResult\nc__builtin__\ntuple\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/NX\x00\x00\x00\x00q\x00h\x00ttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nParseResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/NX\x00\x00\x00\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nParseResult\n(X\x04\x00\x00\x00httpX\x0b\x00\x00\x00example.comX\x01\x00\x00\x00/NX\x00\x00\x00\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95O\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x0bParseResult\x93(\x8c\x04http\x8c\x0bexample.com\x8c\x01/N\x8c\x00\x94h\x00t\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) + check(urlparse(burl, missing_as_none=True), ( + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\np0\n(Vhttp\nVlatin1\np1\ntRg0\n(Vexample.com\ng1\ntRg0\n(V/\ng1\ntRNc__builtin__\nbytes\n(tRp2\ng2\nttR(N(dV_keep_empty\nI01\nstb.', + b'ccopy_reg\n_reconstructor\n(curlparse\nParseResultBytes\nc__builtin__\ntuple\n(c_codecs\nencode\nq\x00(X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01tRh\x00(X\x0b\x00\x00\x00example.comh\x01tRh\x00(X\x01\x00\x00\x00/h\x01tRNc__builtin__\nbytes\n)Rq\x02h\x02ttR(N}X\x0b\x00\x00\x00_keep_emptyI01\nstb.', + b'\x80\x02curlparse\nParseResultBytes\n(c_codecs\nencode\nq\x00X\x04\x00\x00\x00httpX\x06\x00\x00\x00latin1q\x01\x86Rh\x00X\x0b\x00\x00\x00example.comh\x01\x86Rh\x00X\x01\x00\x00\x00/h\x01\x86RNc__builtin__\nbytes\n)Rq\x02h\x02t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x03curllib.parse\nParseResultBytes\n(C\x04httpC\x0bexample.comC\x01/NC\x00q\x00h\x00t\x81N}X\x0b\x00\x00\x00_keep_empty\x88s\x86b.', + b'\x80\x04\x95T\x00\x00\x00\x00\x00\x00\x00\x8c\x0curllib.parse\x8c\x10ParseResultBytes\x93(C\x04httpC\x0bexample.comC\x01/NC\x00\x94h\x00t\x81N}\x8c\x0b_keep_empty\x88s\x86b.', + )) def test_parse_qs_encoding(self): result = urllib.parse.parse_qs("key=\u0141%E9", encoding="latin-1") @@ -1462,6 +1689,11 @@ def test_telurl_params(self): self.assertEqual(p1.path, '+1-201-555-0123') self.assertEqual(p1.params, '') + p1 = urllib.parse.urlparse('tel:+1-201-555-0123', missing_as_none=True) + self.assertEqual(p1.scheme, 'tel') + self.assertEqual(p1.path, '+1-201-555-0123') + self.assertEqual(p1.params, None) + p1 = urllib.parse.urlparse('tel:7042;phone-context=example.com') self.assertEqual(p1.scheme, 'tel') self.assertEqual(p1.path, '7042') @@ -1689,63 +1921,63 @@ def test_splittype_deprecation(self): urllib.parse.splittype('') self.assertEqual(str(cm.warning), 'urllib.parse.splittype() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splithost_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splithost('') self.assertEqual(str(cm.warning), 'urllib.parse.splithost() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splituser_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splituser('') self.assertEqual(str(cm.warning), 'urllib.parse.splituser() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitpasswd_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitpasswd('') self.assertEqual(str(cm.warning), 'urllib.parse.splitpasswd() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitport('') self.assertEqual(str(cm.warning), 'urllib.parse.splitport() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitnport_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitnport('') self.assertEqual(str(cm.warning), 'urllib.parse.splitnport() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitquery_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitquery('') self.assertEqual(str(cm.warning), 'urllib.parse.splitquery() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splittag_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splittag('') self.assertEqual(str(cm.warning), 'urllib.parse.splittag() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitattr_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: urllib.parse.splitattr('') self.assertEqual(str(cm.warning), 'urllib.parse.splitattr() is deprecated as of 3.8, ' - 'use urllib.parse.urlparse() instead') + 'use urllib.parse.urlsplit() instead') def test_splitvalue_deprecation(self): with self.assertWarns(DeprecationWarning) as cm: @@ -1762,6 +1994,8 @@ def test_to_bytes_deprecation(self): def str_encode(s): + if s is None: + return None return s.encode('ascii') def tuple_encode(t): diff --git a/Lib/test/test_webbrowser.py b/Lib/test/test_webbrowser.py index bfbcf112b0b..82f14ca968f 100644 --- a/Lib/test/test_webbrowser.py +++ b/Lib/test/test_webbrowser.py @@ -5,9 +5,10 @@ import subprocess import sys import unittest +import warnings import webbrowser -from functools import partial from test import support +from test.support import force_not_colorized_test_class from test.support import import_helper from test.support import is_apple_mobile from test.support import os_helper @@ -335,6 +336,87 @@ def close(self): return None +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSTest(unittest.TestCase): + + def setUp(self): + env = self.enterContext(os_helper.EnvironmentVarGuard()) + env.unset("BROWSER") + + def test_default(self): + browser = webbrowser.get() + self.assertIsInstance(browser, webbrowser.MacOS) + self.assertEqual(browser.name, 'default') + + def test_default_http_open(self): + # http/https URLs use /usr/bin/open directly — no bundle ID needed. + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_default_non_http_uses_bundle_id(self): + # Non-http(s) URLs (e.g. file://) must be routed through the browser + # via -b to prevent OS file handler dispatch. + file_url = 'file:///tmp/test.html' + browser = webbrowser.MacOS('default') + with mock.patch('webbrowser._macos_default_browser_bundle_id', + return_value='com.google.Chrome'), \ + mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(file_url) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.google.Chrome', file_url], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_known_browser_uses_bundle_id(self): + # Named browsers with a known bundle ID use /usr/bin/open -b. + browser = webbrowser.MacOS('safari') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + result = browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-b', 'com.apple.Safari', URL], + stderr=subprocess.DEVNULL, + ) + self.assertTrue(result) + + def test_named_unknown_browser_falls_back_to_dash_a(self): + # Named browsers not in the bundle ID map fall back to -a. + browser = webbrowser.MacOS('lynx') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=0) + browser.open(URL) + mock_run.assert_called_once_with( + ['/usr/bin/open', '-a', 'lynx', URL], + stderr=subprocess.DEVNULL, + ) + + def test_open_failure(self): + browser = webbrowser.MacOS('default') + with mock.patch('subprocess.run') as mock_run: + mock_run.return_value = mock.Mock(returncode=1) + result = browser.open(URL) + self.assertFalse(result) + + +@unittest.skipUnless(sys.platform == "darwin", "macOS specific test") +@requires_subprocess() +class MacOSXOSAScriptDeprecationTest(unittest.TestCase): + + def test_deprecation_warning(self): + with self.assertWarns(DeprecationWarning): + webbrowser.MacOSXOSAScript('default') + + @unittest.skipUnless(sys.platform == "darwin", "macOS specific test") @requires_subprocess() class MacOSXOSAScriptTest(unittest.TestCase): @@ -345,17 +427,14 @@ def setUp(self): env.unset("BROWSER") support.patch(self, os, "popen", self.mock_popen) + self.enterContext(warnings.catch_warnings()) + warnings.simplefilter("ignore", DeprecationWarning) self.browser = webbrowser.MacOSXOSAScript("default") def mock_popen(self, cmd, mode): self.popen_pipe = MockPopenPipe(cmd, mode) return self.popen_pipe - def test_default(self): - browser = webbrowser.get() - assert isinstance(browser, webbrowser.MacOSXOSAScript) - self.assertEqual(browser.name, "default") - def test_default_open(self): url = "https://python.org" self.browser.open(url) @@ -381,7 +460,9 @@ def test_default_browser_lookup(self): self.assertIn(f'open location "{url}"', script) def test_explicit_browser(self): - browser = webbrowser.MacOSXOSAScript("safari") + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + browser = webbrowser.MacOSXOSAScript("safari") browser.open("https://python.org") script = self.popen_pipe.pipe.getvalue() self.assertIn('tell application "safari"', script) @@ -527,6 +608,7 @@ def test_environment_preferred(self): self.assertEqual(webbrowser.get().name, sys.executable) +@force_not_colorized_test_class class CliTest(unittest.TestCase): def test_parse_args(self): for command, url, new_win in [ diff --git a/Lib/test/test_wsgiref.py b/Lib/test/test_wsgiref.py index d84e7647adb..484a2db1637 100644 --- a/Lib/test/test_wsgiref.py +++ b/Lib/test/test_wsgiref.py @@ -1,6 +1,6 @@ from unittest import mock from test import support -from test.support import socket_helper, control_characters_c0 +from test.support import force_not_colorized, socket_helper, control_characters_c0 from test.test_httpservers import NoLogRequestHandler from unittest import TestCase from wsgiref.util import setup_testing_defaults @@ -66,6 +66,26 @@ def header_app(environ, start_response): ]).encode('iso-8859-1')] +def input_app(func_name, *args): + def app(e,s): + req = getattr(e['wsgi.input'], func_name)(*args) + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + if type(req) is list: + resp = b";".join(req) + else: + resp = req + return [resp] + return app + + +def errors_app(func_name, *args): + def app(e,s): + getattr(e['wsgi.errors'], func_name)(*args) + s("200 OK", [("Content-Type", "text/plain; charset=utf-8")]) + return [b"data"] + return app + + def run_amock(app=hello_app, data=b"GET / HTTP/1.0\n\n"): server = make_server("", 80, app, MockServer, MockHandler) inp = BufferedReader(BytesIO(data)) @@ -109,7 +129,7 @@ def check_hello(self, out, has_length=True): sys.version.split()[0]) self.assertEqual(out, ("HTTP/1.0 200 OK\r\n" - "Server: WSGIServer/0.2 " + pyver +"\r\n" + "Server: WSGIServer " + pyver + "\r\n" "Content-Type: text/plain\r\n" "Date: Mon, 05 Jun 2006 18:49:54 GMT\r\n" + (has_length and "Content-Length: 13\r\n" or "") + @@ -192,6 +212,96 @@ def bad_app(e,s): err.splitlines()[-2], "AssertionError" ) + def test_wsgi_input_read(self): + bad_app = input_app("read") + good_app = input_app("read", 5) + + out, err = run_amock(validator(bad_app)) + self.assertEndsWith(out, + b"A server error occurred. Please contact the administrator." + ) + + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + out, err = run_amock(validator(good_app), b"GET / HTTP/1.0\n\nTest 1\nTest 2\n") + self.assertEndsWith(out, b"Test ") + + def test_wsgi_input_readlines(self): + bad_app = input_app("readlines", 3, 5) + good_app = input_app("readlines", 1) + + out, err = run_amock(validator(bad_app)) + self.assertEndsWith(out, + b"A server error occurred. Please contact the administrator." + ) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + out, err = run_amock(validator(good_app), b"GET / HTTP/1.0\n\nTest Line 1\nTest Line 2\n") + self.assertEndsWith(out, b"Test Line 1\n") + + def test_wsgi_input_readline(self): + bad_app = input_app("readline", 3, 4) + good_app = input_app("readline", 2) + + out, err = run_amock(validator(bad_app)) + self.assertEndsWith(out, + b"A server error occurred. Please contact the administrator." + ) + self.assertEqual( + err.splitlines()[-2], "AssertionError" + ) + + out, err = run_amock(validator(good_app), b"GET / HTTP/1.0\n\nTest 1\nTest 2\n") + self.assertEndsWith(out, b"Te") + + def test_wsgi_input_close(self): + app = input_app("close") + out, err = run_amock(validator(app), b"GET / HTTP/1.0\n\nTest 1\nTest 2\n") + self.assertEqual(err.splitlines()[-2], 'AssertionError: input.close() must not be called') + self.assertEndsWith(out, b"A server error occurred. Please contact the administrator.") + + def test_wsgi_input_iter(self): + def app(e,s): + req = [] + for line in e['wsgi.input']: + req.append(line) + s("200 OK", [('Content-Type', 'text/plain; charser=utf-8')]) + return [b';'.join(req)] + + out, err = run_amock(validator(app), b"GET / HTTP/1.0\n\nTest 1\nTest 2\n") + self.assertEndsWith(out, b"Test 1\n;Test 2\n") + + def test_wsgi_errors_write(self): + bad_app = errors_app("write", b"Test") + good_app = errors_app("write", "Test") + + out, err = run_amock(validator(bad_app), b"GET / HTTP/1.0\n\n") + self.assertEqual(err.splitlines()[-2], 'AssertionError') + + out, err = run_amock(validator(good_app), b"GET / HTTP/1.0\n\n") + self.assertStartsWith(err, "Test") + + def test_wsgi_errors_writelines(self): + bad_app = errors_app("writelines", [1, "Test"]) + good_app = errors_app("writelines", ["Test", "Test"]) + + out, err = run_amock(validator(bad_app), b"GET / HTTP/1.0\n\n") + self.assertEqual(err.splitlines()[-2], 'AssertionError') + + out, err = run_amock(validator(good_app), b"GET / HTTP/1.0\n\n") + self.assertStartsWith(err, "TestTest") + + def test_wsgi_errors_close(self): + app = errors_app("close") + + out, err = run_amock(validator(app), b"GET / HTTP/1.0\n\n") + self.assertEqual(err.splitlines()[-2], + 'AssertionError: errors.close() must not be called') + + @force_not_colorized def test_bytes_validation(self): def app(e, s): s("200 OK", [ @@ -206,7 +316,7 @@ def app(e, s): pyver = py + b"/" + ver self.assertEqual( b"HTTP/1.0 200 OK\r\n" - b"Server: WSGIServer/0.2 "+ pyver + b"\r\n" + b"Server: WSGIServer " + pyver + b"\r\n" b"Content-Type: text/plain; charset=utf-8\r\n" b"Date: Wed, 24 Dec 2008 13:29:32 GMT\r\n" b"\r\n" @@ -448,10 +558,6 @@ def testHopByHop(self): for alt in hop, hop.title(), hop.upper(), hop.lower(): self.assertFalse(util.is_hop_by_hop(alt)) - @unittest.expectedFailure # TODO: RUSTPYTHON - def test_filewrapper_getitem_deprecation(self): - return super().test_filewrapper_getitem_deprecation() - class HeaderTests(TestCase): def testMappingInterface(self): @@ -879,5 +985,17 @@ def testRaisesControlCharacters(self): base.start_response('200 OK', [('x', c0)]) +class TestModule(unittest.TestCase): + def test_deprecated__version__(self): + from wsgiref import simple_server + + with self.assertWarnsRegex( + DeprecationWarning, + "'__version__' is deprecated and slated for removal in Python 3.20", + ) as cm: + getattr(simple_server, "__version__") + self.assertEqual(cm.filename, __file__) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/urllib/parse.py b/Lib/urllib/parse.py index c9f8a33e72c..4247b9a4b07 100644 --- a/Lib/urllib/parse.py +++ b/Lib/urllib/parse.py @@ -97,11 +97,9 @@ def clear_cache(): _byte_quoter_factory.cache_clear() # Helpers for bytes handling -# For 3.2, we deliberately require applications that -# handle improperly quoted URLs to do their own -# decoding and encoding. If valid use cases are -# presented, we may relax this by using latin-1 -# decoding internally for 3.3 +# We deliberately require applications that +# handle improperly quoted URLs to do their +# own decoding and encoding. _implicit_encoding = 'ascii' _implicit_errors = 'strict' @@ -114,7 +112,10 @@ def _encode_result(obj, encoding=_implicit_encoding, def _decode_args(args, encoding=_implicit_encoding, errors=_implicit_errors): - return tuple(x.decode(encoding, errors) if x else '' for x in args) + return tuple(x.decode(encoding, errors) if x + else '' if x is not None + else None + for x in args) def _coerce_args(*args): # Invokes decode if necessary to create str args @@ -122,13 +123,20 @@ def _coerce_args(*args): # an appropriate result coercion function # - noop for str inputs # - encoding function otherwise - str_input = isinstance(args[0], str) - for arg in args[1:]: - # We special-case the empty string to support the - # "scheme=''" default argument to some functions - if arg and isinstance(arg, str) != str_input: - raise TypeError("Cannot mix str and non-str arguments") - if str_input: + str_input = None + for arg in args: + if arg: + if str_input is None: + str_input = isinstance(arg, str) + else: + if isinstance(arg, str) != str_input: + raise TypeError("Cannot mix str and non-str arguments") + if str_input is None: + for arg in args: + if arg is not None: + str_input = isinstance(arg, str) + break + if str_input is not False: return args + (_noop,) return _decode_args(args) + (_encode_result,) @@ -138,7 +146,14 @@ class _ResultMixinStr(object): __slots__ = () def encode(self, encoding='ascii', errors='strict'): - return self._encoded_counterpart(*(x.encode(encoding, errors) for x in self)) + result = self._encoded_counterpart(*(x.encode(encoding, errors) + if x is not None else None + for x in self)) + try: + result._keep_empty = self._keep_empty + except AttributeError: + pass + return result class _ResultMixinBytes(object): @@ -146,7 +161,14 @@ class _ResultMixinBytes(object): __slots__ = () def decode(self, encoding='ascii', errors='strict'): - return self._decoded_counterpart(*(x.decode(encoding, errors) for x in self)) + result = self._decoded_counterpart(*(x.decode(encoding, errors) + if x is not None else None + for x in self)) + try: + result._keep_empty = self._keep_empty + except AttributeError: + pass + return result class _NetlocResultMixinBase(object): @@ -193,6 +215,8 @@ class _NetlocResultMixinStr(_NetlocResultMixinBase, _ResultMixinStr): @property def _userinfo(self): netloc = self.netloc + if netloc is None: + return None, None userinfo, have_info, hostinfo = netloc.rpartition('@') if have_info: username, have_password, password = userinfo.partition(':') @@ -205,6 +229,8 @@ def _userinfo(self): @property def _hostinfo(self): netloc = self.netloc + if netloc is None: + return None, None _, _, hostinfo = netloc.rpartition('@') _, have_open_br, bracketed = hostinfo.partition('[') if have_open_br: @@ -223,6 +249,8 @@ class _NetlocResultMixinBytes(_NetlocResultMixinBase, _ResultMixinBytes): @property def _userinfo(self): netloc = self.netloc + if netloc is None: + return None, None userinfo, have_info, hostinfo = netloc.rpartition(b'@') if have_info: username, have_password, password = userinfo.partition(b':') @@ -235,6 +263,8 @@ def _userinfo(self): @property def _hostinfo(self): netloc = self.netloc + if netloc is None: + return None, None _, _, hostinfo = netloc.rpartition(b'@') _, have_open_br, bracketed = hostinfo.partition(b'[') if have_open_br: @@ -247,11 +277,70 @@ def _hostinfo(self): return hostname, port -_DefragResultBase = namedtuple('_DefragResultBase', 'url fragment') -_SplitResultBase = namedtuple( - '_SplitResultBase', 'scheme netloc path query fragment') -_ParseResultBase = namedtuple( - '_ParseResultBase', 'scheme netloc path params query fragment') +_UNSPECIFIED = sentinel("_UNSPECIFIED", repr="") +_MISSING_AS_NONE_DEFAULT = False + +class _ResultBase: + __slots__ = () + + def __replace__(self, /, **kwargs): + result = super().__replace__(**kwargs) + try: + result._keep_empty = self._keep_empty + except AttributeError: + pass + return result + + def _replace(self, /, **kwargs): + result = super()._replace(**kwargs) + try: + result._keep_empty = self._keep_empty + except AttributeError: + pass + return result + + def __copy__(self): + return self + + def __deepcopy__(self, memo): + return self + + def __getstate__(self): + state = super().__getstate__() + try: + if state[1]['_keep_empty'] == _MISSING_AS_NONE_DEFAULT: + del state[1]['_keep_empty'] + if state == (None, {}): + state = None + except LookupError: + pass + return state + + +class _DefragResultBase(_ResultBase, namedtuple('_DefragResultBase', 'url fragment')): + __slots__ = ('_keep_empty',) + + def geturl(self): + if self.fragment or (self.fragment is not None and + getattr(self, '_keep_empty', _MISSING_AS_NONE_DEFAULT)): + return self.url + self._HASH + self.fragment + else: + return self.url + +class _SplitResultBase(_ResultBase, namedtuple( + '_SplitResultBase', 'scheme netloc path query fragment')): + __slots__ = ('_keep_empty',) + + def geturl(self): + return urlunsplit(self) + +class _ParseResultBase(_ResultBase, namedtuple( + '_ParseResultBase', 'scheme netloc path params query fragment')): + __slots__ = ('_keep_empty',) + + def geturl(self): + return urlunparse(self) + _DefragResultBase.__doc__ = """ DefragResult(url, fragment) @@ -322,40 +411,24 @@ def _hostinfo(self): # Structured result objects for string data class DefragResult(_DefragResultBase, _ResultMixinStr): __slots__ = () - def geturl(self): - if self.fragment: - return self.url + '#' + self.fragment - else: - return self.url + _HASH = '#' class SplitResult(_SplitResultBase, _NetlocResultMixinStr): __slots__ = () - def geturl(self): - return urlunsplit(self) class ParseResult(_ParseResultBase, _NetlocResultMixinStr): __slots__ = () - def geturl(self): - return urlunparse(self) # Structured result objects for bytes data class DefragResultBytes(_DefragResultBase, _ResultMixinBytes): __slots__ = () - def geturl(self): - if self.fragment: - return self.url + b'#' + self.fragment - else: - return self.url + _HASH = b'#' class SplitResultBytes(_SplitResultBase, _NetlocResultMixinBytes): __slots__ = () - def geturl(self): - return urlunsplit(self) class ParseResultBytes(_ParseResultBase, _NetlocResultMixinBytes): __slots__ = () - def geturl(self): - return urlunparse(self) # Set up the encode/decode result pairs def _fix_result_transcoding(): @@ -371,7 +444,7 @@ def _fix_result_transcoding(): _fix_result_transcoding() del _fix_result_transcoding -def urlparse(url, scheme='', allow_fragments=True): +def urlparse(url, scheme=None, allow_fragments=True, *, missing_as_none=_MISSING_AS_NONE_DEFAULT): """Parse a URL into 6 components: :///;?# @@ -394,23 +467,33 @@ def urlparse(url, scheme='', allow_fragments=True): urlsplit() should generally be used instead of urlparse(). """ url, scheme, _coerce_result = _coerce_args(url, scheme) + if url is None: + url = '' scheme, netloc, url, params, query, fragment = _urlparse(url, scheme, allow_fragments) - result = ParseResult(scheme or '', netloc or '', url, params or '', query or '', fragment or '') - return _coerce_result(result) + if not missing_as_none: + if scheme is None: scheme = '' + if netloc is None: netloc = '' + if params is None: params = '' + if query is None: query = '' + if fragment is None: fragment = '' + result = ParseResult(scheme, netloc, url, params, query, fragment) + result = _coerce_result(result) + result._keep_empty = missing_as_none + return result def _urlparse(url, scheme=None, allow_fragments=True): scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) if (scheme or '') in uses_params and ';' in url: - url, params = _splitparams(url, allow_none=True) + url, params = _splitparams(url, missing_as_none=True) else: params = None return (scheme, netloc, url, params, query, fragment) -def _splitparams(url, allow_none=False): +def _splitparams(url, missing_as_none=False): if '/' in url: i = url.find(';', url.rfind('/')) if i < 0: - return url, None if allow_none else '' + return url, None if missing_as_none else '' else: i = url.find(';') return url[:i], url[i+1:] @@ -472,7 +555,7 @@ def _check_bracketed_host(hostname): # typed=True avoids BytesWarnings being emitted during cache key # comparison since this API supports both bytes and str input. @functools.lru_cache(typed=True) -def urlsplit(url, scheme='', allow_fragments=True): +def urlsplit(url, scheme=None, allow_fragments=True, *, missing_as_none=_MISSING_AS_NONE_DEFAULT): """Parse a URL into 5 components: :///?# @@ -494,9 +577,18 @@ def urlsplit(url, scheme='', allow_fragments=True): """ url, scheme, _coerce_result = _coerce_args(url, scheme) + if url is None: + url = '' scheme, netloc, url, query, fragment = _urlsplit(url, scheme, allow_fragments) - v = SplitResult(scheme or '', netloc or '', url, query or '', fragment or '') - return _coerce_result(v) + if not missing_as_none: + if scheme is None: scheme = '' + if netloc is None: netloc = '' + if query is None: query = '' + if fragment is None: fragment = '' + result = SplitResult(scheme, netloc, url, query, fragment) + result = _coerce_result(result) + result._keep_empty = missing_as_none + return result def _urlsplit(url, scheme=None, allow_fragments=True): # Only lstrip url as some applications rely on preserving trailing space. @@ -532,38 +624,61 @@ def _urlsplit(url, scheme=None, allow_fragments=True): _checknetloc(netloc) return (scheme, netloc, url, query, fragment) -def urlunparse(components): +def urlunparse(components, *, keep_empty=_UNSPECIFIED): """Put a parsed URL back together again. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had redundant delimiters, e.g. a ? with an empty query - (the draft states that these are equivalent).""" + (the draft states that these are equivalent) and keep_empty is false + or components is the result of the urlparse() call with + missing_as_none=False.""" scheme, netloc, url, params, query, fragment, _coerce_result = ( _coerce_args(*components)) - if not netloc: - if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): - netloc = '' - else: - netloc = None - if params: + if keep_empty is _UNSPECIFIED: + keep_empty = getattr(components, '_keep_empty', _MISSING_AS_NONE_DEFAULT) + elif keep_empty and not getattr(components, '_keep_empty', True): + raise ValueError('Cannot distinguish between empty and not defined ' + 'URI components in the result of parsing URL with ' + 'missing_as_none=False') + if not keep_empty: + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + if not scheme: scheme = None + if not params: params = None + if not query: query = None + if not fragment: fragment = None + if params is not None: url = "%s;%s" % (url, params) - return _coerce_result(_urlunsplit(scheme or None, netloc, url, - query or None, fragment or None)) + return _coerce_result(_urlunsplit(scheme, netloc, url, query, fragment)) -def urlunsplit(components): +def urlunsplit(components, *, keep_empty=_UNSPECIFIED): """Combine the elements of a tuple as returned by urlsplit() into a complete URL as a string. The data argument can be any five-item iterable. This may result in a slightly different, but equivalent URL, if the URL that was parsed originally had unnecessary delimiters (for example, a ? with an - empty query; the RFC states that these are equivalent).""" + empty query; the RFC states that these are equivalent) and keep_empty + is false or components is the result of the urlsplit() call with + missing_as_none=False.""" scheme, netloc, url, query, fragment, _coerce_result = ( _coerce_args(*components)) - if not netloc: - if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): - netloc = '' - else: - netloc = None - return _coerce_result(_urlunsplit(scheme or None, netloc, url, - query or None, fragment or None)) + if keep_empty is _UNSPECIFIED: + keep_empty = getattr(components, '_keep_empty', _MISSING_AS_NONE_DEFAULT) + elif keep_empty and not getattr(components, '_keep_empty', True): + raise ValueError('Cannot distinguish between empty and not defined ' + 'URI components in the result of parsing URL with ' + 'missing_as_none=False') + if not keep_empty: + if not netloc: + if scheme and scheme in uses_netloc and (not url or url[:1] == '/'): + netloc = '' + else: + netloc = None + if not scheme: scheme = None + if not query: query = None + if not fragment: fragment = None + return _coerce_result(_urlunsplit(scheme, netloc, url, query, fragment)) def _urlunsplit(scheme, netloc, url, query, fragment): if netloc is not None: @@ -651,21 +766,24 @@ def urljoin(base, url, allow_fragments=True): resolved_path) or '/', query, fragment)) -def urldefrag(url): +def urldefrag(url, *, missing_as_none=_MISSING_AS_NONE_DEFAULT): """Removes any existing fragment from URL. Returns a tuple of the defragmented URL and the fragment. If the URL contained no fragments, the second element is the - empty string. + empty string or None if missing_as_none is True. """ url, _coerce_result = _coerce_args(url) if '#' in url: s, n, p, q, frag = _urlsplit(url) defrag = _urlunsplit(s, n, p, q, None) else: - frag = '' + frag = None defrag = url - return _coerce_result(DefragResult(defrag, frag or '')) + if not missing_as_none and frag is None: frag = '' + result = _coerce_result(DefragResult(defrag, frag)) + result._keep_empty = missing_as_none + return result _hexdig = '0123456789ABCDEFabcdef' _hextobyte = None @@ -1066,7 +1184,7 @@ def urlencode(query, doseq=False, safe='', encoding=None, errors=None, else: try: # Is this a sufficient test for sequence-ness? - x = len(v) + len(v) except TypeError: # not a sequence v = quote_via(str(v), safe, encoding, errors) @@ -1117,7 +1235,7 @@ def unwrap(url): def splittype(url): warnings.warn("urllib.parse.splittype() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splittype(url) @@ -1138,7 +1256,7 @@ def _splittype(url): def splithost(url): warnings.warn("urllib.parse.splithost() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splithost(url) @@ -1161,7 +1279,7 @@ def _splithost(url): def splituser(host): warnings.warn("urllib.parse.splituser() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splituser(host) @@ -1174,7 +1292,7 @@ def _splituser(host): def splitpasswd(user): warnings.warn("urllib.parse.splitpasswd() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splitpasswd(user) @@ -1187,7 +1305,7 @@ def _splitpasswd(user): def splitport(host): warnings.warn("urllib.parse.splitport() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splitport(host) @@ -1210,7 +1328,7 @@ def _splitport(host): def splitnport(host, defport=-1): warnings.warn("urllib.parse.splitnport() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splitnport(host, defport) @@ -1234,7 +1352,7 @@ def _splitnport(host, defport=-1): def splitquery(url): warnings.warn("urllib.parse.splitquery() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splitquery(url) @@ -1249,7 +1367,7 @@ def _splitquery(url): def splittag(url): warnings.warn("urllib.parse.splittag() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splittag(url) @@ -1264,7 +1382,7 @@ def _splittag(url): def splitattr(url): warnings.warn("urllib.parse.splitattr() is deprecated as of 3.8, " - "use urllib.parse.urlparse() instead", + "use urllib.parse.urlsplit() instead", DeprecationWarning, stacklevel=2) return _splitattr(url) diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index a8f40100b7b..9fa92659a25 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -415,6 +415,8 @@ def add_handler(self, handler): continue i = meth.find("_") + if i < 1: + continue protocol = meth[:i] condition = meth[i+1:] @@ -813,16 +815,17 @@ def add_password(self, realm, uri, user, passwd): self.passwd[realm] = {} for default_port in True, False: reduced_uri = tuple( - self.reduce_uri(u, default_port) for u in uri) + self._reduce_uri_with_scheme(u, default_port) for u in uri) self.passwd[realm][reduced_uri] = (user, passwd) def find_user_password(self, realm, authuri): domains = self.passwd.get(realm, {}) for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uris, authinfo in domains.items(): for uri in uris: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return authinfo return None, None @@ -849,6 +852,17 @@ def reduce_uri(self, uri, default_port=True): authority = "%s:%d" % (host, dport) return authority, path + def _reduce_uri_with_scheme(self, uri, default_port=True): + parts = urlsplit(uri) + scheme = parts[0] if parts[1] else None + return (scheme or None, *self.reduce_uri(uri, default_port)) + + def _is_suburi_with_scheme(self, base, test): + if (base[0] is not None and test[0] is not None and + base[0] != test[0]): + return False + return self.is_suburi(base[1:], test[1:]) + def is_suburi(self, base, test): """Check if test is below base in a URI tree @@ -894,14 +908,15 @@ def update_authenticated(self, uri, is_authenticated=False): for default_port in True, False: for u in uri: - reduced_uri = self.reduce_uri(u, default_port) + reduced_uri = self._reduce_uri_with_scheme(u, default_port) self.authenticated[reduced_uri] = is_authenticated def is_authenticated(self, authuri): for default_port in True, False: - reduced_authuri = self.reduce_uri(authuri, default_port) + reduced_authuri = self._reduce_uri_with_scheme( + authuri, default_port) for uri in self.authenticated: - if self.is_suburi(uri, reduced_authuri): + if self._is_suburi_with_scheme(uri, reduced_authuri): return self.authenticated[uri] diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index 97aad6eea50..ec8b544a6b0 100644 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -1,7 +1,8 @@ """Interfaces for launching and remotely controlling web browsers.""" -# Maintained by Georg Brandl. +import builtins # because we override open import os +lazy import plistlib import shlex import shutil import sys @@ -492,10 +493,15 @@ def register_standard_browsers(): _tryorder = [] if sys.platform == 'darwin': - register("MacOSX", None, MacOSXOSAScript('default')) - register("chrome", None, MacOSXOSAScript('google chrome')) - register("firefox", None, MacOSXOSAScript('firefox')) - register("safari", None, MacOSXOSAScript('safari')) + register("MacOS", None, MacOS('default')) + register("MacOSX", None, MacOS('default')) # backward compat alias + register("chrome", None, MacOS('google chrome')) + register("chromium", None, MacOS('chromium')) + register("firefox", None, MacOS('firefox')) + register("safari", None, MacOS('safari')) + register("opera", None, MacOS('opera')) + register("microsoft-edge", None, MacOS('microsoft edge')) + register("brave", None, MacOS('brave browser')) # macOS can use below Unix support (but we prefer using the macOS # specific stuff) @@ -614,8 +620,80 @@ def open(self, url, new=0, autoraise=True): # if sys.platform == 'darwin': + def _macos_default_browser_bundle_id(): + """Return the bundle ID of the default web browser. + + Reads the LaunchServices preferences file that macOS maintains + when the user sets a default browser. Returns 'com.apple.Safari' + if the file is absent or no https handler is recorded, because on + a fresh macOS installation Safari is the default browser and the + LaunchServices plist is not written until the user explicitly + changes their default browser. + """ + plist = os.path.expanduser( + '~/Library/Preferences/com.apple.LaunchServices/' + 'com.apple.launchservices.secure.plist' + ) + try: + with builtins.open(plist, 'rb') as f: + data = plistlib.load(f) + for handler in data.get('LSHandlers', []): + if handler.get('LSHandlerURLScheme') == 'https': + return (handler.get('LSHandlerRoleAll') + or handler.get('LSHandlerRoleViewer')) + except (OSError, KeyError, ValueError): + pass + return 'com.apple.Safari' + + class MacOS(BaseBrowser): + """Launcher class for macOS browsers, using /usr/bin/open. + + For http/https URLs with the default browser, /usr/bin/open is called + directly; macOS routes these to the registered browser. + + For all other URL schemes (e.g. file://) and for named browsers, + /usr/bin/open -b is used so that the URL is always passed + to a browser application rather than dispatched by the OS file handler. + This prevents file injection attacks where a file:// URL pointing to an + executable bundle could otherwise be launched by the OS. + + Named browsers with known bundle IDs use -b; unknown names fall back + to -a. + """ + + _BUNDLE_IDS = { + 'google chrome': 'com.google.Chrome', + 'firefox': 'org.mozilla.firefox', + 'safari': 'com.apple.Safari', + 'chromium': 'org.chromium.Chromium', + 'opera': 'com.operasoftware.Opera', + 'microsoft edge': 'com.microsoft.edgemac', + 'brave browser': 'com.brave.Browser', + } + + def open(self, url, new=0, autoraise=True): + sys.audit("webbrowser.open", url) + self._check_url(url) + if self.name == 'default': + proto, sep, _ = url.partition(':') + if sep and proto.lower() in {'http', 'https'}: + cmd = ['/usr/bin/open', url] + else: + bundle_id = _macos_default_browser_bundle_id() + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + bundle_id = self._BUNDLE_IDS.get(self.name.lower()) + if bundle_id: + cmd = ['/usr/bin/open', '-b', bundle_id, url] + else: + cmd = ['/usr/bin/open', '-a', self.name, url] + proc = subprocess.run(cmd, stderr=subprocess.DEVNULL) + return proc.returncode == 0 + class MacOSXOSAScript(BaseBrowser): def __init__(self, name='default'): + import warnings + warnings._deprecated("webbrowser.MacOSXOSAScript", remove=(3, 17)) super().__init__(name) def open(self, url, new=0, autoraise=True): @@ -734,7 +812,7 @@ def open(self, url, new=0, autoraise=True): def parse_args(arg_list: list[str] | None): import argparse parser = argparse.ArgumentParser( - description="Open URL in a web browser.", color=True, + description="Open URL in a web browser.", ) parser.add_argument("url", help="URL to open") diff --git a/Lib/wsgiref/simple_server.py b/Lib/wsgiref/simple_server.py index a0f2397fcf0..31efd8c9bae 100644 --- a/Lib/wsgiref/simple_server.py +++ b/Lib/wsgiref/simple_server.py @@ -16,11 +16,10 @@ from wsgiref.handlers import SimpleHandler from platform import python_implementation -__version__ = "0.2" __all__ = ['WSGIServer', 'WSGIRequestHandler', 'demo_app', 'make_server'] -server_version = "WSGIServer/" + __version__ +server_version = "WSGIServer" sys_version = python_implementation() + "/" + sys.version.split()[0] software_version = server_version + ' ' + sys_version @@ -70,7 +69,7 @@ def set_app(self,application): class WSGIRequestHandler(BaseHTTPRequestHandler): - server_version = "WSGIServer/" + __version__ + server_version = "WSGIServer" def get_environ(self): env = self.server.base_environ.copy() @@ -152,6 +151,15 @@ def make_server( return server +def __getattr__(name): + if name == "__version__": + from warnings import _deprecated + + _deprecated("__version__", remove=(3, 20)) + return "0.2" # Do not change + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + if __name__ == '__main__': with make_server('', 8000, demo_app) as httpd: sa = httpd.socket.getsockname() diff --git a/crates/vm/src/getpath.rs b/crates/vm/src/getpath.rs index 68e95583a13..6bccb30261b 100644 --- a/crates/vm/src/getpath.rs +++ b/crates/vm/src/getpath.rs @@ -203,7 +203,6 @@ pub fn init_path_config(settings: &Settings) -> Paths { /// Identify an uninstalled RustPython source library independently of the /// executable location (Cargo permits targets outside the source tree). -#[cfg(windows)] pub(crate) fn source_directory(paths: &Paths) -> Option<&Path> { let library = Path::new(paths.stdlib_dir.as_deref()?); let source = library.parent()?; diff --git a/crates/vm/src/stdlib/sys.rs b/crates/vm/src/stdlib/sys.rs index 6051723b19a..84d3a3f8a84 100644 --- a/crates/vm/src/stdlib/sys.rs +++ b/crates/vm/src/stdlib/sys.rs @@ -299,6 +299,11 @@ pub mod sys { vm.state.config.paths.stdlib_dir.clone().to_pyobject(vm) } + #[pyattr] + fn _is_python_build(vm: &VirtualMachine) -> bool { + crate::getpath::source_directory(&vm.state.config.paths).is_some() + } + // alphabetical order with segments of pyattr and others #[pyattr] diff --git a/scripts/update_lib/cmd_copy_lib.py b/scripts/update_lib/cmd_copy_lib.py index 848ee9213c2..b16d88dac65 100644 --- a/scripts/update_lib/cmd_copy_lib.py +++ b/scripts/update_lib/cmd_copy_lib.py @@ -8,6 +8,9 @@ # Directory python scripts/update_lib copy-lib cpython/Lib/json + + # Explicit, pinned Cargo-build adaptation (copies only pydoc.py) + python scripts/update_lib copy-lib cpython/Lib/pydoc.py --pydoc-build-stdlib """ import argparse @@ -57,6 +60,8 @@ def _copy_single( def copy_lib( src_path: pathlib.Path, verbose: bool = True, + *, + pydoc_build_stdlib: bool = False, ) -> None: """ Copy library file or directory from CPython. @@ -66,6 +71,7 @@ def copy_lib( Args: src_path: Source path (e.g., cpython/Lib/dataclasses.py or cpython/Lib/json) verbose: Print progress messages + pydoc_build_stdlib: Adapt only pinned pydoc.py for Cargo's stdlib layout """ from update_lib.deps import get_lib_paths from update_lib.file_utils import parse_lib_path @@ -77,6 +83,28 @@ def copy_lib( if not src_path.exists(): raise FileNotFoundError(f"Source path does not exist: {src_path}") + if pydoc_build_stdlib: + from update_lib.lib_transforms import pydoc_build_stdlib as transform + + lib_path = parse_lib_path(src_path) + if lib_path != pathlib.Path("Lib/pydoc.py") or not src_path.is_file(): + raise ValueError("--pydoc-build-stdlib requires exactly Lib/pydoc.py") + if src_path.resolve() == lib_path.resolve() or ( + lib_path.exists() and src_path.samefile(lib_path) + ): + raise ValueError("pydoc source and destination must be different files") + if lib_path.is_symlink() or (lib_path.exists() and not lib_path.is_file()): + raise ValueError("pydoc destination must be a regular file") + # Validate the complete input before writing anything. This explicit + # transform must not refresh dependencies or change unrelated files. + contents = transform(src_path.read_bytes()) + if verbose: + print(f"Adapting pydoc Cargo stdlib path: {src_path} -> {lib_path}") + lib_path.parent.mkdir(parents=True, exist_ok=True) + lib_path.write_bytes(contents) + shutil.copystat(src_path, lib_path) + return + # Data files name an exact asset, not the containing top-level module. # Expanding Lib/test/data.txt as "test" would replace the entire test tree. if src_path.is_file() and src_path.suffix != ".py": @@ -109,11 +137,16 @@ def main(argv: list[str] | None = None) -> int: type=pathlib.Path, help="Source path containing /Lib/ (e.g., cpython/Lib/dataclasses.py)", ) + parser.add_argument( + "--pydoc-build-stdlib", + action="store_true", + help="Adapt only pinned pydoc.py for RustPython's Cargo-build stdlib path", + ) args = parser.parse_args(argv) try: - copy_lib(args.path) + copy_lib(args.path, pydoc_build_stdlib=args.pydoc_build_stdlib) return 0 except ValueError as e: print(f"Error: {e}", file=sys.stderr) diff --git a/scripts/update_lib/lib_transforms.py b/scripts/update_lib/lib_transforms.py new file mode 100644 index 00000000000..23714ec15ff --- /dev/null +++ b/scripts/update_lib/lib_transforms.py @@ -0,0 +1,34 @@ +"""Explicit, pinned adaptations for RustPython's standard library layout.""" + +import hashlib + +# CPython d220409a13b77de7a6c30abe16ea90db01528edc, Lib/pydoc.py. +PYDOC_SOURCE_SHA256 = "cf007075ab2a6d7d408be847ded67efa31dd99ae535540e6d80fa510359551a4" +PYDOC_STDLIB_ANCHOR = b""" STDLIB_DIR = sysconfig.get_path('stdlib') + + def document(self, object, name=None, *args): +""" +PYDOC_STDLIB_REPLACEMENT = b""" STDLIB_DIR = sysconfig.get_path('stdlib') + # TODO: RUSTPYTHON; Cargo builds keep Lib outside sysconfig's install paths. + if (sys.implementation.name == 'rustpython' + and getattr(sys, '_is_python_build', False) + and not sysconfig.is_python_build() + and getattr(sys, '_stdlib_dir', None)): + STDLIB_DIR = sys._stdlib_dir + + def document(self, object, name=None, *args): +""" + + +def pydoc_build_stdlib(source: bytes) -> bytes: + """Adapt only the default doc lookup root in the pinned canonical pydoc. + + An explicit basedir keeps its upstream meaning. CPython and installed + RustPython layouts retain upstream behavior. Regenerate from the canonical + input; already adapted or changed inputs require a fresh review. + """ + if source.count(PYDOC_STDLIB_ANCHOR) != 1: + raise ValueError("Expected one canonical pydoc STDLIB_DIR anchor") + if hashlib.sha256(source).hexdigest() != PYDOC_SOURCE_SHA256: + raise ValueError("pydoc SHA256 does not match the pinned canonical input") + return source.replace(PYDOC_STDLIB_ANCHOR, PYDOC_STDLIB_REPLACEMENT, 1) diff --git a/scripts/update_lib/tests/test_pydoc_transform.py b/scripts/update_lib/tests/test_pydoc_transform.py new file mode 100644 index 00000000000..c7ee7a0cba5 --- /dev/null +++ b/scripts/update_lib/tests/test_pydoc_transform.py @@ -0,0 +1,353 @@ +"""Host-tooling tests for the explicitly approved pydoc layout adaptation.""" + +import ast +import contextlib +import hashlib +import inspect +import io +import os +import pathlib +import sys +import tempfile +import types +import unittest +import warnings +from unittest.mock import patch + +from update_lib.cmd_copy_lib import copy_lib, main +from update_lib.lib_transforms import ( + PYDOC_SOURCE_SHA256, + PYDOC_STDLIB_ANCHOR, + PYDOC_STDLIB_REPLACEMENT, + pydoc_build_stdlib, +) + + +def canonical_pydoc(): + # Recover the pinned input from the checked-in generated file, so these + # tests need neither an external CPython checkout nor a second stdlib copy. + path = pathlib.Path(__file__).resolve().parents[3] / "Lib/pydoc.py" + contents = path.read_bytes() + if contents.count(PYDOC_STDLIB_REPLACEMENT) == 1: + contents = contents.replace(PYDOC_STDLIB_REPLACEMENT, PYDOC_STDLIB_ANCHOR, 1) + if hashlib.sha256(contents).hexdigest() != PYDOC_SOURCE_SHA256: + raise AssertionError("Checked-in pydoc no longer matches the reviewed input") + return contents + + +class TestPydocTransform(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = canonical_pydoc() + + def test_changes_only_default_stdlib_assignment(self): + original = b" STDLIB_DIR = sysconfig.get_path('stdlib')\n" + expected = ( + original + + b""" # TODO: RUSTPYTHON; Cargo builds keep Lib outside sysconfig's install paths. + if (sys.implementation.name == 'rustpython' + and getattr(sys, '_is_python_build', False) + and not sysconfig.is_python_build() + and getattr(sys, '_stdlib_dir', None)): + STDLIB_DIR = sys._stdlib_dir +""" + ) + before, found, after = self.source.partition(original) + self.assertEqual(found, original) + self.assertEqual(pydoc_build_stdlib(self.source), before + expected + after) + + def test_checked_in_pydoc_matches_generated_bytes(self): + path = pathlib.Path(__file__).resolve().parents[3] / "Lib/pydoc.py" + self.assertEqual(path.read_bytes(), pydoc_build_stdlib(self.source)) + + def test_refuses_changed_source(self): + with self.assertRaisesRegex(ValueError, "SHA256"): + pydoc_build_stdlib(self.source + b"# unreviewed change\n") + + def test_requires_exactly_one_anchor(self): + for contents in ( + self.source.replace(PYDOC_STDLIB_ANCHOR, b""), + self.source + PYDOC_STDLIB_ANCHOR, + pydoc_build_stdlib(self.source), + ): + with self.subTest(contents_length=len(contents)): + with self.assertRaisesRegex(ValueError, "one canonical.*anchor"): + pydoc_build_stdlib(contents) + + def test_explicit_copy_is_repeatable_and_changes_only_pydoc(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.parent.mkdir(parents=True) + source.write_bytes(self.source) + dependency = source.parent / "pydoc_data/module_docs.py" + dependency.parent.mkdir() + dependency.write_bytes(b"unselected dependency") + target = root / "Lib/pydoc.py" + target.parent.mkdir() + target.write_bytes(b"old pydoc") + sibling = target.parent / "pydoc_data/module_docs.py" + sibling.parent.mkdir() + sibling.write_bytes(b"unchanged dependency") + with contextlib.chdir(root), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(main([str(source), "--pydoc-build-stdlib"]), 0) + first = target.read_bytes() + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertEqual(first, pydoc_build_stdlib(self.source)) + self.assertEqual(target.read_bytes(), first) + self.assertEqual(source.read_bytes(), self.source) + self.assertEqual(sibling.read_bytes(), b"unchanged dependency") + + def test_ordinary_copy_keeps_canonical_bytes_and_dependencies(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.parent.mkdir(parents=True) + source.write_bytes(self.source) + dependency = source.parent / "pydoc_data/module_docs.py" + dependency.parent.mkdir() + dependency.write_bytes(b"canonical dependency") + with contextlib.chdir(root): + copy_lib(source, verbose=False) + self.assertEqual((root / "Lib/pydoc.py").read_bytes(), self.source) + self.assertEqual( + (root / "Lib/pydoc_data/module_docs.py").read_bytes(), + b"canonical dependency", + ) + + def test_refuses_unrelated_paths_without_writing(self): + for relative in ("other.py", "test/pydoc.py", "package/../pydoc.py"): + with self.subTest(path=relative), tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib" / relative + source.parent.mkdir(parents=True, exist_ok=True) + source.write_bytes(self.source) + with contextlib.chdir(root): + with self.assertRaisesRegex(ValueError, "exactly Lib/pydoc.py"): + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertFalse((root / "Lib").exists()) + + def test_refuses_directory_source_without_writing(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.mkdir(parents=True) + with contextlib.chdir(root): + with self.assertRaisesRegex(ValueError, "exactly Lib/pydoc.py"): + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertFalse((root / "Lib").exists()) + + def test_invalid_input_preserves_existing_destination(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.parent.mkdir(parents=True) + source.write_bytes(self.source + b"# unreviewed change\n") + target = root / "Lib/pydoc.py" + target.parent.mkdir() + target.write_bytes(b"preserve destination") + with contextlib.chdir(root), contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(main([str(source), "--pydoc-build-stdlib"]), 1) + self.assertEqual(target.read_bytes(), b"preserve destination") + + def test_refuses_source_as_destination(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "Lib/pydoc.py" + source.parent.mkdir() + source.write_bytes(self.source) + with contextlib.chdir(root): + with self.assertRaisesRegex(ValueError, "different files"): + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertEqual(source.read_bytes(), self.source) + + def test_refuses_nonregular_destination(self): + for kind in ("directory", "symlink"): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.parent.mkdir(parents=True) + source.write_bytes(self.source) + target = root / "Lib/pydoc.py" + target.parent.mkdir() + other = root / "other.py" + other.write_bytes(b"preserve other file") + if kind == "directory": + target.mkdir() + else: + target.symlink_to(other) + with contextlib.chdir(root): + with self.assertRaisesRegex(ValueError, "regular file"): + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertEqual(other.read_bytes(), b"preserve other file") + self.assertEqual(source.read_bytes(), self.source) + + def test_refuses_hardlinked_source_and_destination(self): + with tempfile.TemporaryDirectory() as tmpdir: + root = pathlib.Path(tmpdir) + source = root / "cpython/Lib/pydoc.py" + source.parent.mkdir(parents=True) + source.write_bytes(self.source) + target = root / "Lib/pydoc.py" + target.parent.mkdir() + target.hardlink_to(source) + with contextlib.chdir(root): + with self.assertRaisesRegex(ValueError, "different files"): + copy_lib(source, verbose=False, pydoc_build_stdlib=True) + self.assertEqual(source.read_bytes(), self.source) + + +class TestPydocPathSelection(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = canonical_pydoc() + cls.generated = pydoc_build_stdlib(cls.source) + + def make_doc( + self, + *, + source=None, + implementation="rustpython", + config_build=False, + runtime_attributes=None, + ): + source = self.generated if source is None else source + tree = ast.parse(source) + doc = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "Doc" + ) + runtime = types.SimpleNamespace( + implementation=types.SimpleNamespace(name=implementation), + version_info=(3, 15), + **(runtime_attributes or {}), + ) + namespace = { + "os": os, + "sys": runtime, + "sysconfig": types.SimpleNamespace( + get_path=lambda name: os.path.abspath("installed/stdlib"), + is_python_build=lambda: config_build, + get_config_var=lambda name: os.path.abspath("source"), + ), + "inspect": inspect, + "warnings": warnings, + } + exec( + compile(ast.Module(body=[doc], type_ignores=[]), "pydoc.py", "exec"), + namespace, + ) + return namespace["Doc"]() + + def test_only_cargo_build_mismatch_changes_default(self): + runtime_dir = os.path.abspath("checkout/Lib") + for implementation in ("rustpython", "cpython", "pypy"): + for config_build in (False, True): + for attributes in ( + {}, + {"_stdlib_dir": runtime_dir}, + {"_is_python_build": False, "_stdlib_dir": runtime_dir}, + {"_is_python_build": True}, + {"_is_python_build": True, "_stdlib_dir": None}, + {"_is_python_build": True, "_stdlib_dir": ""}, + {"_is_python_build": True, "_stdlib_dir": runtime_dir}, + ): + with self.subTest( + implementation=implementation, + config_build=config_build, + attributes=attributes, + ): + options = dict( + implementation=implementation, + config_build=config_build, + runtime_attributes=attributes, + ) + doc = self.make_doc(**options) + original = self.make_doc(source=self.source, **options) + adapted = ( + implementation == "rustpython" + and not config_build + and attributes.get("_is_python_build") + and attributes.get("_stdlib_dir") + ) + self.assertEqual( + doc.STDLIB_DIR, + runtime_dir if adapted else original.STDLIB_DIR, + ) + module = types.ModuleType("example") + for directory in ( + runtime_dir, + original.STDLIB_DIR, + os.path.abspath("source/Lib"), + ): + module.__file__ = os.path.join(directory, "example.py") + if not adapted: + self.assertEqual( + doc._is_stdlib_module(module), + original._is_stdlib_module(module), + ) + for explicit in (runtime_dir, original.STDLIB_DIR, ""): + self.assertEqual( + doc._is_stdlib_module(module, explicit), + original._is_stdlib_module(module, explicit), + ) + + def test_runtime_stdlib_detection_and_site_packages_exclusion(self): + runtime_dir = os.path.abspath("checkout/Lib") + doc = self.make_doc( + runtime_attributes={"_is_python_build": True, "_stdlib_dir": runtime_dir} + ) + module = types.ModuleType("example") + module.__file__ = os.path.join(runtime_dir, "example.py") + self.assertTrue(doc._is_stdlib_module(module)) + self.assertFalse(doc._is_stdlib_module(module, os.path.abspath("custom"))) + module.__file__ = os.path.join(runtime_dir, "site-packages/example.py") + self.assertFalse(doc._is_stdlib_module(module)) + + def test_getdocloc_uses_runtime_default_and_preserves_explicit_basedir(self): + runtime_dir = os.path.abspath("checkout/Lib") + doc = self.make_doc( + runtime_attributes={"_is_python_build": True, "_stdlib_dir": runtime_dir} + ) + module = types.ModuleType("example") + module.__file__ = os.path.join(runtime_dir, "example.py") + data = types.ModuleType("pydoc_data") + data.module_docs = types.SimpleNamespace( + module_docs={"example": "example.html"} + ) + with ( + patch.dict(sys.modules, {"pydoc_data": data}), + patch.dict(os.environ, {"PYTHONDOCS": "https://docs.example/library"}), + ): + self.assertEqual( + doc.getdocloc(module), "https://docs.example/library/example.html" + ) + self.assertIsNone(doc.getdocloc(module, os.path.abspath("custom"))) + + def test_version_warning_suppression_uses_runtime_stdlib(self): + class DeprecatedModule(types.ModuleType): + def __getattr__(self, name): + if name == "__version__": + warnings.warn("deprecated version", DeprecationWarning) + return "1.0" + raise AttributeError(name) + + runtime_dir = os.path.abspath("checkout/Lib") + doc = self.make_doc( + runtime_attributes={"_is_python_build": True, "_stdlib_dir": runtime_dir} + ) + module = DeprecatedModule("example") + for relative, count in (("example.py", 0), ("site-packages/example.py", 1)): + with ( + self.subTest(path=relative), + warnings.catch_warnings(record=True) as caught, + ): + warnings.simplefilter("always") + module.__file__ = os.path.join(runtime_dir, relative) + self.assertEqual(doc._get_version(module), "1.0") + self.assertEqual(len(caught), count) + + +if __name__ == "__main__": + unittest.main()