(reinterpret_cast(proc));
return true;
}
return false;
}
+[[nodiscard]] static UINT WINAPI DummyGetDpiForSystem()
+{
+ UINT dpi = USER_DEFAULT_SCREEN_DPI;
+ if (HDC hdc = ::GetDC(nullptr); hdc != nullptr)
+ {
+ dpi = static_cast(::GetDeviceCaps(hdc, LOGPIXELSX));
+ ::ReleaseDC(nullptr, hdc);
+ }
+ return dpi;
+}
+
+[[nodiscard]] static UINT WINAPI DummyGetDpiForWindow([[maybe_unused]] HWND hwnd)
+{
+ return DummyGetDpiForSystem();
+}
+
+[[nodiscard]] static int WINAPI DummyGetSystemMetricsForDpi(int nIndex, UINT dpi)
+{
+ return DPIManagerV2::scale(::GetSystemMetrics(nIndex), dpi);
+}
+
+[[nodiscard]] static BOOL WINAPI DummySystemParametersInfoForDpi(UINT uiAction, UINT uiParam, PVOID pvParam, UINT fWinIni, [[maybe_unused]] UINT dpi)
+{
+ return ::SystemParametersInfoW(uiAction, uiParam, pvParam, fWinIni);
+}
+
+[[nodiscard]] static BOOL WINAPI DummyIsValidDpiAwarenessContext([[maybe_unused]] DPI_AWARENESS_CONTEXT value)
+{
+ return FALSE;
+}
+
+[[nodiscard]] static DPI_AWARENESS_CONTEXT WINAPI DummySetThreadDpiAwarenessContext([[maybe_unused]] DPI_AWARENESS_CONTEXT dpiContext)
+{
+ return nullptr;
+}
+
+static BOOL WINAPI DummyAdjustWindowRectExForDpi(
+ [[maybe_unused]] LPRECT lpRect,
+ [[maybe_unused]] DWORD dwStyle,
+ [[maybe_unused]] BOOL bMenu,
+ [[maybe_unused]] DWORD dwExStyle,
+ [[maybe_unused]] UINT dpi
+)
+{
+ return FALSE;
+}
+
using fnGetDpiForSystem = UINT (WINAPI*)(VOID);
using fnGetDpiForWindow = UINT (WINAPI*)(HWND hwnd);
using fnGetSystemMetricsForDpi = int (WINAPI*)(int nIndex, UINT dpi);
using fnSystemParametersInfoForDpi = BOOL (WINAPI*)(UINT uiAction, UINT uiParam, PVOID pvParam, UINT fWinIni, UINT dpi);
+using fnIsValidDpiAwarenessContext = BOOL (WINAPI*)(DPI_AWARENESS_CONTEXT value);
using fnSetThreadDpiAwarenessContext = DPI_AWARENESS_CONTEXT (WINAPI*)(DPI_AWARENESS_CONTEXT dpiContext);
using fnAdjustWindowRectExForDpi = BOOL (WINAPI*)(LPRECT lpRect, DWORD dwStyle, BOOL bMenu, DWORD dwExStyle, UINT dpi);
-
-fnGetDpiForSystem _fnGetDpiForSystem = nullptr;
-fnGetDpiForWindow _fnGetDpiForWindow = nullptr;
-fnGetSystemMetricsForDpi _fnGetSystemMetricsForDpi = nullptr;
-fnSystemParametersInfoForDpi _fnSystemParametersInfoForDpi = nullptr;
-fnSetThreadDpiAwarenessContext _fnSetThreadDpiAwarenessContext = nullptr;
-fnAdjustWindowRectExForDpi _fnAdjustWindowRectExForDpi = nullptr;
-
+static fnGetDpiForSystem _fnGetDpiForSystem = DummyGetDpiForSystem;
+static fnGetDpiForWindow _fnGetDpiForWindow = DummyGetDpiForWindow;
+static fnGetSystemMetricsForDpi _fnGetSystemMetricsForDpi = DummyGetSystemMetricsForDpi;
+static fnSystemParametersInfoForDpi _fnSystemParametersInfoForDpi = DummySystemParametersInfoForDpi;
+static fnIsValidDpiAwarenessContext _fnIsValidDpiAwarenessContext = DummyIsValidDpiAwarenessContext;
+static fnSetThreadDpiAwarenessContext _fnSetThreadDpiAwarenessContext = DummySetThreadDpiAwarenessContext;
+static fnAdjustWindowRectExForDpi _fnAdjustWindowRectExForDpi = DummyAdjustWindowRectExForDpi;
void DPIManagerV2::initDpiAPI()
{
@@ -63,12 +107,13 @@ void DPIManagerV2::initDpiAPI()
HMODULE hUser32 = ::GetModuleHandleW(L"user32.dll");
if (hUser32 != nullptr)
{
- ptrFn(hUser32, _fnGetDpiForSystem, "GetDpiForSystem");
- ptrFn(hUser32, _fnGetDpiForWindow, "GetDpiForWindow");
- ptrFn(hUser32, _fnGetSystemMetricsForDpi, "GetSystemMetricsForDpi");
- ptrFn(hUser32, _fnSystemParametersInfoForDpi, "SystemParametersInfoForDpi");
- ptrFn(hUser32, _fnSetThreadDpiAwarenessContext, "SetThreadDpiAwarenessContext");
- ptrFn(hUser32, _fnAdjustWindowRectExForDpi, "AdjustWindowRectExForDpi");
+ LoadFn(hUser32, _fnGetDpiForSystem, "GetDpiForSystem");
+ LoadFn(hUser32, _fnGetDpiForWindow, "GetDpiForWindow");
+ LoadFn(hUser32, _fnGetSystemMetricsForDpi, "GetSystemMetricsForDpi");
+ LoadFn(hUser32, _fnSystemParametersInfoForDpi, "SystemParametersInfoForDpi");
+ LoadFn(hUser32, _fnIsValidDpiAwarenessContext, "IsValidDpiAwarenessContext");
+ LoadFn(hUser32, _fnSetThreadDpiAwarenessContext, "SetThreadDpiAwarenessContext");
+ LoadFn(hUser32, _fnAdjustWindowRectExForDpi, "AdjustWindowRectExForDpi");
}
}
@@ -76,59 +121,43 @@ void DPIManagerV2::initDpiAPI()
int DPIManagerV2::getSystemMetricsForDpi(int nIndex, UINT dpi)
{
- if (_fnGetSystemMetricsForDpi != nullptr)
- {
- return _fnGetSystemMetricsForDpi(nIndex, dpi);
- }
- return DPIManagerV2::scale(::GetSystemMetrics(nIndex), dpi);
+ return _fnGetSystemMetricsForDpi(nIndex, dpi);
+}
+
+bool DPIManagerV2::isValidDpiAwarenessContext(DPI_AWARENESS_CONTEXT value)
+{
+ return _fnIsValidDpiAwarenessContext(value) == TRUE;
}
DPI_AWARENESS_CONTEXT DPIManagerV2::setThreadDpiAwarenessContext(DPI_AWARENESS_CONTEXT dpiContext)
{
- if (_fnSetThreadDpiAwarenessContext != nullptr)
+ if (DPIManagerV2::isValidDpiAwarenessContext(dpiContext))
{
return _fnSetThreadDpiAwarenessContext(dpiContext);
}
- return NULL;
+ return nullptr;
}
-BOOL DPIManagerV2::adjustWindowRectExForDpi(LPRECT lpRect, DWORD dwStyle, BOOL bMenu, DWORD dwExStyle, UINT dpi)
+bool DPIManagerV2::adjustWindowRectExForDpi(LPRECT lpRect, DWORD dwStyle, BOOL bMenu, DWORD dwExStyle, UINT dpi)
{
- if (_fnAdjustWindowRectExForDpi != nullptr)
- {
- return _fnAdjustWindowRectExForDpi(lpRect, dwStyle, bMenu, dwExStyle, dpi);
- }
- return FALSE;
+ return _fnAdjustWindowRectExForDpi(lpRect, dwStyle, bMenu, dwExStyle, dpi) == TRUE;
}
UINT DPIManagerV2::getDpiForSystem()
{
- if (_fnGetDpiForSystem != nullptr)
- {
- return _fnGetDpiForSystem();
- }
-
- UINT dpi = USER_DEFAULT_SCREEN_DPI;
- HDC hdc = ::GetDC(nullptr);
- if (hdc != nullptr)
- {
- dpi = ::GetDeviceCaps(hdc, LOGPIXELSX);
- ::ReleaseDC(nullptr, hdc);
- }
- return dpi;
+ return _fnGetDpiForSystem();
}
UINT DPIManagerV2::getDpiForWindow(HWND hWnd)
{
- if (_fnGetDpiForWindow != nullptr)
+ if (hWnd != nullptr)
{
- const auto dpi = _fnGetDpiForWindow(hWnd);
- if (dpi > 0)
+ if (const auto dpi = _fnGetDpiForWindow(hWnd); dpi > 0)
{
return dpi;
}
}
- return getDpiForSystem();
+ return DPIManagerV2::getDpiForSystem();
}
void DPIManagerV2::setPositionDpi(LPARAM lParam, HWND hWnd, UINT flags)
@@ -146,21 +175,11 @@ void DPIManagerV2::setPositionDpi(LPARAM lParam, HWND hWnd, UINT flags)
LOGFONT DPIManagerV2::getDefaultGUIFontForDpi(UINT dpi, FontType type)
{
- int result = 0;
LOGFONT lf{};
NONCLIENTMETRICS ncm{};
ncm.cbSize = sizeof(NONCLIENTMETRICS);
- if (_fnSystemParametersInfoForDpi != nullptr
- && (_fnSystemParametersInfoForDpi(SPI_GETNONCLIENTMETRICS, sizeof(NONCLIENTMETRICS), &ncm, 0, dpi) != FALSE))
- {
- result = 2;
- }
- else if (::SystemParametersInfo(SPI_GETNONCLIENTMETRICS, sizeof(NONCLIENTMETRICS), &ncm, 0) != FALSE)
- {
- result = 1;
- }
- if (result > 0)
+ if (_fnSystemParametersInfoForDpi(SPI_GETNONCLIENTMETRICS, sizeof(NONCLIENTMETRICS), &ncm, 0, dpi) == TRUE)
{
switch (type)
{
@@ -176,6 +195,12 @@ LOGFONT DPIManagerV2::getDefaultGUIFontForDpi(UINT dpi, FontType type)
break;
}
+ case FontType::message:
+ {
+ lf = ncm.lfMessageFont;
+ break;
+ }
+
case FontType::caption:
{
lf = ncm.lfCaptionFont;
@@ -187,22 +212,12 @@ LOGFONT DPIManagerV2::getDefaultGUIFontForDpi(UINT dpi, FontType type)
lf = ncm.lfSmCaptionFont;
break;
}
- //case FontType::message:
- default:
- {
- lf = ncm.lfMessageFont;
- break;
- }
}
}
else // should not happen, fallback
{
- auto hf = static_cast(::GetStockObject(DEFAULT_GUI_FONT));
- ::GetObject(hf, sizeof(LOGFONT), &lf);
- }
-
- if (result < 2)
- {
+ auto* hf = static_cast(::GetStockObject(DEFAULT_GUI_FONT));
+ ::GetObjectW(hf, sizeof(LOGFONT), &lf);
lf.lfHeight = scaleFont(lf.lfHeight, dpi);
}
@@ -216,3 +231,18 @@ void DPIManagerV2::loadIcon(HINSTANCE hinst, const wchar_t* pszName, int cx, int
*phico = static_cast(::LoadImage(hinst, pszName, IMAGE_ICON, cx, cy, fuLoad));
}
}
+
+DWORD DPIManagerV2::getTextScaleFactor()
+{
+ static constexpr DWORD defaultVal = 100;
+ DWORD data = defaultVal;
+ DWORD dwBufSize = sizeof(data);
+ static constexpr LPCWSTR lpSubKey = L"Software\\Microsoft\\Accessibility";
+ static constexpr LPCWSTR lpValue = L"TextScaleFactor";
+
+ if (::RegGetValueW(HKEY_CURRENT_USER, lpSubKey, lpValue, RRF_RT_REG_DWORD, nullptr, &data, &dwBufSize) == ERROR_SUCCESS)
+ {
+ return data;
+ }
+ return defaultVal;
+}
diff --git a/PythonLib/full/_android_support.py b/PythonLib/full/_android_support.py
new file mode 100644
index 000000000..320dab52a
--- /dev/null
+++ b/PythonLib/full/_android_support.py
@@ -0,0 +1,192 @@
+import io
+import sys
+from threading import RLock
+from time import sleep, time
+
+# The maximum length of a log message in bytes, including the level marker and
+# tag, is defined as LOGGER_ENTRY_MAX_PAYLOAD at
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log.h;l=71.
+# Messages longer than this will be truncated by logcat. This limit has already
+# been reduced at least once in the history of Android (from 4076 to 4068 between
+# API level 23 and 26), so leave some headroom.
+MAX_BYTES_PER_WRITE = 4000
+
+# UTF-8 uses a maximum of 4 bytes per character, so limiting text writes to this
+# size ensures that we can always avoid exceeding MAX_BYTES_PER_WRITE.
+# However, if the actual number of bytes per character is smaller than that,
+# then we may still join multiple consecutive text writes into binary
+# writes containing a larger number of characters.
+MAX_CHARS_PER_WRITE = MAX_BYTES_PER_WRITE // 4
+
+
+# When embedded in an app on current versions of Android, there's no easy way to
+# monitor the C-level stdout and stderr. The testbed comes with a .c file to
+# redirect them to the system log using a pipe, but that wouldn't be convenient
+# or appropriate for all apps. So we redirect at the Python level instead.
+def init_streams(android_log_write, stdout_prio, stderr_prio):
+ if sys.executable:
+ return # Not embedded in an app.
+
+ global logcat
+ logcat = Logcat(android_log_write)
+ sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout)
+ sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr)
+
+
+class TextLogStream(io.TextIOWrapper):
+ def __init__(self, prio, tag, original=None, **kwargs):
+ # Respect the -u option.
+ if original:
+ kwargs.setdefault("write_through", original.write_through)
+ fileno = original.fileno()
+ else:
+ fileno = None
+
+ # The default is surrogateescape for stdout and backslashreplace for
+ # stderr, but in the context of an Android log, readability is more
+ # important than reversibility.
+ kwargs.setdefault("encoding", "UTF-8")
+ kwargs.setdefault("errors", "backslashreplace")
+
+ super().__init__(BinaryLogStream(prio, tag, fileno), **kwargs)
+ self._lock = RLock()
+ self._pending_bytes = []
+ self._pending_bytes_count = 0
+
+ def __repr__(self):
+ return f""
+
+ def write(self, s):
+ if not isinstance(s, str):
+ raise TypeError(
+ f"write() argument must be str, not {type(s).__name__}")
+
+ # In case `s` is a str subclass that writes itself to stdout or stderr
+ # when we call its methods, convert it to an actual str.
+ s = str.__str__(s)
+
+ # We want to emit one log message per line wherever possible, so split
+ # the string into lines first. Note that "".splitlines() == [], so
+ # nothing will be logged for an empty string.
+ with self._lock:
+ for line in s.splitlines(keepends=True):
+ while line:
+ chunk = line[:MAX_CHARS_PER_WRITE]
+ line = line[MAX_CHARS_PER_WRITE:]
+ self._write_chunk(chunk)
+
+ return len(s)
+
+ # The size and behavior of TextIOWrapper's buffer is not part of its public
+ # API, so we handle buffering ourselves to avoid truncation.
+ def _write_chunk(self, s):
+ b = s.encode(self.encoding, self.errors)
+ if self._pending_bytes_count + len(b) > MAX_BYTES_PER_WRITE:
+ self.flush()
+
+ self._pending_bytes.append(b)
+ self._pending_bytes_count += len(b)
+ if (
+ self.write_through
+ or b.endswith(b"\n")
+ or self._pending_bytes_count > MAX_BYTES_PER_WRITE
+ ):
+ self.flush()
+
+ def flush(self):
+ with self._lock:
+ self.buffer.write(b"".join(self._pending_bytes))
+ self._pending_bytes.clear()
+ self._pending_bytes_count = 0
+
+ # Since this is a line-based logging system, line buffering cannot be turned
+ # off, i.e. a newline always causes a flush.
+ @property
+ def line_buffering(self):
+ return True
+
+
+class BinaryLogStream(io.RawIOBase):
+ def __init__(self, prio, tag, fileno=None):
+ self.prio = prio
+ self.tag = tag
+ self._fileno = fileno
+
+ def __repr__(self):
+ return f""
+
+ def writable(self):
+ return True
+
+ def write(self, b):
+ if type(b) is not bytes:
+ try:
+ b = bytes(memoryview(b))
+ except TypeError:
+ raise TypeError(
+ f"write() argument must be bytes-like, not {type(b).__name__}"
+ ) from None
+
+ # Writing an empty string to the stream should have no effect.
+ if b:
+ logcat.write(self.prio, self.tag, b)
+ return len(b)
+
+ # This is needed by the test suite --timeout option, which uses faulthandler.
+ def fileno(self):
+ if self._fileno is None:
+ raise io.UnsupportedOperation("fileno")
+ return self._fileno
+
+
+# When a large volume of data is written to logcat at once, e.g. when a test
+# module fails in --verbose3 mode, there's a risk of overflowing logcat's own
+# buffer and losing messages. We avoid this by imposing a rate limit using the
+# token bucket algorithm, based on a conservative estimate of how fast `adb
+# logcat` can consume data.
+MAX_BYTES_PER_SECOND = 1024 * 1024
+
+# The logcat buffer size of a device can be determined by running `logcat -g`.
+# We set the token bucket size to half of the buffer size of our current minimum
+# API level, because other things on the system will be producing messages as
+# well.
+BUCKET_SIZE = 128 * 1024
+
+# https://cs.android.com/android/platform/superproject/+/android-14.0.0_r1:system/logging/liblog/include/log/log_read.h;l=39
+PER_MESSAGE_OVERHEAD = 28
+
+
+class Logcat:
+ def __init__(self, android_log_write):
+ self.android_log_write = android_log_write
+ self._lock = RLock()
+ self._bucket_level = 0
+ self._prev_write_time = time()
+
+ def write(self, prio, tag, message):
+ # Encode null bytes using "modified UTF-8" to avoid them truncating the
+ # message.
+ message = message.replace(b"\x00", b"\xc0\x80")
+
+ # On API level 30 and higher, Logcat will strip any number of leading
+ # newlines. This is visible in all `logcat` modes, even --binary. Work
+ # around this by adding a leading space, which shouldn't make any
+ # difference to the log's usability.
+ if message.startswith(b"\n"):
+ message = b" " + message
+
+ with self._lock:
+ now = time()
+ self._bucket_level += (
+ (now - self._prev_write_time) * MAX_BYTES_PER_SECOND)
+
+ # If the bucket level is still below zero, the clock must have gone
+ # backwards, so reset it to zero and continue.
+ self._bucket_level = max(0, min(self._bucket_level, BUCKET_SIZE))
+ self._prev_write_time = now
+
+ self._bucket_level -= PER_MESSAGE_OVERHEAD + len(tag) + len(message)
+ if self._bucket_level < 0:
+ sleep(-self._bucket_level / MAX_BYTES_PER_SECOND)
+
+ self.android_log_write(prio, tag, message)
diff --git a/PythonLib/full/_apple_support.py b/PythonLib/full/_apple_support.py
new file mode 100644
index 000000000..92febdcf5
--- /dev/null
+++ b/PythonLib/full/_apple_support.py
@@ -0,0 +1,66 @@
+import io
+import sys
+
+
+def init_streams(log_write, stdout_level, stderr_level):
+ # Redirect stdout and stderr to the Apple system log. This method is
+ # invoked by init_apple_streams() (initconfig.c) if config->use_system_logger
+ # is enabled.
+ sys.stdout = SystemLog(log_write, stdout_level, errors=sys.stderr.errors)
+ sys.stderr = SystemLog(log_write, stderr_level, errors=sys.stderr.errors)
+
+
+class SystemLog(io.TextIOWrapper):
+ def __init__(self, log_write, level, **kwargs):
+ kwargs.setdefault("encoding", "UTF-8")
+ kwargs.setdefault("line_buffering", True)
+ super().__init__(LogStream(log_write, level), **kwargs)
+
+ def __repr__(self):
+ return f""
+
+ def write(self, s):
+ if not isinstance(s, str):
+ raise TypeError(
+ f"write() argument must be str, not {type(s).__name__}")
+
+ # In case `s` is a str subclass that writes itself to stdout or stderr
+ # when we call its methods, convert it to an actual str.
+ s = str.__str__(s)
+
+ # We want to emit one log message per line, so split
+ # the string before sending it to the superclass.
+ for line in s.splitlines(keepends=True):
+ super().write(line)
+
+ return len(s)
+
+
+class LogStream(io.RawIOBase):
+ def __init__(self, log_write, level):
+ self.log_write = log_write
+ self.level = level
+
+ def __repr__(self):
+ return f""
+
+ def writable(self):
+ return True
+
+ def write(self, b):
+ if type(b) is not bytes:
+ try:
+ b = bytes(memoryview(b))
+ except TypeError:
+ raise TypeError(
+ f"write() argument must be bytes-like, not {type(b).__name__}"
+ ) from None
+
+ # Writing an empty string to the stream should have no effect.
+ if b:
+ # Encode null bytes using "modified UTF-8" to avoid truncating the
+ # message. This should not affect the return value, as the caller
+ # may be expecting it to match the length of the input.
+ self.log_write(self.level, b.replace(b"\x00", b"\xc0\x80"))
+
+ return len(b)
diff --git a/PythonLib/full/_ast_unparse.py b/PythonLib/full/_ast_unparse.py
new file mode 100644
index 000000000..1c8741b5a
--- /dev/null
+++ b/PythonLib/full/_ast_unparse.py
@@ -0,0 +1,1161 @@
+# This module contains ``ast.unparse()``, defined here
+# to improve the import time for the ``ast`` module.
+import sys
+from _ast import *
+from ast import NodeVisitor
+from contextlib import contextmanager, nullcontext
+from enum import IntEnum, auto, _simple_enum
+
+# Large float and imaginary literals get turned into infinities in the AST.
+# We unparse those infinities to INFSTR.
+_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
+
+@_simple_enum(IntEnum)
+class _Precedence:
+ """Precedence table that originated from python grammar."""
+
+ NAMED_EXPR = auto() # :=
+ TUPLE = auto() # ,
+ YIELD = auto() # 'yield', 'yield from'
+ TEST = auto() # 'if'-'else', 'lambda'
+ OR = auto() # 'or'
+ AND = auto() # 'and'
+ NOT = auto() # 'not'
+ CMP = auto() # '<', '>', '==', '>=', '<=', '!=',
+ # 'in', 'not in', 'is', 'is not'
+ EXPR = auto()
+ BOR = EXPR # '|'
+ BXOR = auto() # '^'
+ BAND = auto() # '&'
+ SHIFT = auto() # '<<', '>>'
+ ARITH = auto() # '+', '-'
+ TERM = auto() # '*', '@', '/', '%', '//'
+ FACTOR = auto() # unary '+', '-', '~'
+ POWER = auto() # '**'
+ AWAIT = auto() # 'await'
+ ATOM = auto()
+
+ def next(self):
+ try:
+ return self.__class__(self + 1)
+ except ValueError:
+ return self
+
+
+_SINGLE_QUOTES = ("'", '"')
+_MULTI_QUOTES = ('"""', "'''")
+_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES)
+
+class Unparser(NodeVisitor):
+ """Methods in this class recursively traverse an AST and
+ output source code for the abstract syntax; original formatting
+ is disregarded."""
+
+ def __init__(self):
+ self._source = []
+ self._precedences = {}
+ self._type_ignores = {}
+ self._indent = 0
+ self._in_try_star = False
+ self._in_interactive = False
+
+ def interleave(self, inter, f, seq):
+ """Call f on each item in seq, calling inter() in between."""
+ seq = iter(seq)
+ try:
+ f(next(seq))
+ except StopIteration:
+ pass
+ else:
+ for x in seq:
+ inter()
+ f(x)
+
+ def items_view(self, traverser, items):
+ """Traverse and separate the given *items* with a comma and append it to
+ the buffer. If *items* is a single item sequence, a trailing comma
+ will be added."""
+ if len(items) == 1:
+ traverser(items[0])
+ self.write(",")
+ else:
+ self.interleave(lambda: self.write(", "), traverser, items)
+
+ def maybe_newline(self):
+ """Adds a newline if it isn't the start of generated source"""
+ if self._source:
+ self.write("\n")
+
+ def maybe_semicolon(self):
+ """Adds a "; " delimiter if it isn't the start of generated source"""
+ if self._source:
+ self.write("; ")
+
+ def fill(self, text="", *, allow_semicolon=True):
+ """Indent a piece of text and append it, according to the current
+ indentation level, or only delineate with semicolon if applicable"""
+ if self._in_interactive and not self._indent and allow_semicolon:
+ self.maybe_semicolon()
+ self.write(text)
+ else:
+ self.maybe_newline()
+ self.write(" " * self._indent + text)
+
+ def write(self, *text):
+ """Add new source parts"""
+ self._source.extend(text)
+
+ @contextmanager
+ def buffered(self, buffer = None):
+ if buffer is None:
+ buffer = []
+
+ original_source = self._source
+ self._source = buffer
+ yield buffer
+ self._source = original_source
+
+ @contextmanager
+ def block(self, *, extra = None):
+ """A context manager for preparing the source for blocks. It adds
+ the character':', increases the indentation on enter and decreases
+ the indentation on exit. If *extra* is given, it will be directly
+ appended after the colon character.
+ """
+ self.write(":")
+ if extra:
+ self.write(extra)
+ self._indent += 1
+ yield
+ self._indent -= 1
+
+ @contextmanager
+ def delimit(self, start, end):
+ """A context manager for preparing the source for expressions. It adds
+ *start* to the buffer and enters, after exit it adds *end*."""
+
+ self.write(start)
+ yield
+ self.write(end)
+
+ def delimit_if(self, start, end, condition):
+ if condition:
+ return self.delimit(start, end)
+ else:
+ return nullcontext()
+
+ def require_parens(self, precedence, node):
+ """Shortcut to adding precedence related parens"""
+ return self.delimit_if("(", ")", self.get_precedence(node) > precedence)
+
+ def get_precedence(self, node):
+ return self._precedences.get(node, _Precedence.TEST)
+
+ def set_precedence(self, precedence, *nodes):
+ for node in nodes:
+ self._precedences[node] = precedence
+
+ def get_raw_docstring(self, node):
+ """If a docstring node is found in the body of the *node* parameter,
+ return that docstring node, None otherwise.
+
+ Logic mirrored from ``_PyAST_GetDocString``."""
+ if not isinstance(
+ node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)
+ ) or len(node.body) < 1:
+ return None
+ node = node.body[0]
+ if not isinstance(node, Expr):
+ return None
+ node = node.value
+ if isinstance(node, Constant) and isinstance(node.value, str):
+ return node
+
+ def get_type_comment(self, node):
+ comment = self._type_ignores.get(node.lineno) or node.type_comment
+ if comment is not None:
+ return f" # type: {comment}"
+
+ def traverse(self, node):
+ if isinstance(node, list):
+ for item in node:
+ self.traverse(item)
+ else:
+ super().visit(node)
+
+ # Note: as visit() resets the output text, do NOT rely on
+ # NodeVisitor.generic_visit to handle any nodes (as it calls back in to
+ # the subclass visit() method, which resets self._source to an empty list)
+ def visit(self, node):
+ """Outputs a source code string that, if converted back to an ast
+ (using ast.parse) will generate an AST equivalent to *node*"""
+ self._source = []
+ self.traverse(node)
+ return "".join(self._source)
+
+ def _write_docstring_and_traverse_body(self, node):
+ if (docstring := self.get_raw_docstring(node)):
+ self._write_docstring(docstring)
+ self.traverse(node.body[1:])
+ else:
+ self.traverse(node.body)
+
+ def visit_Module(self, node):
+ self._type_ignores = {
+ ignore.lineno: f"ignore{ignore.tag}"
+ for ignore in node.type_ignores
+ }
+ try:
+ self._write_docstring_and_traverse_body(node)
+ finally:
+ self._type_ignores.clear()
+
+ def visit_Interactive(self, node):
+ self._in_interactive = True
+ try:
+ self._write_docstring_and_traverse_body(node)
+ finally:
+ self._in_interactive = False
+
+ def visit_FunctionType(self, node):
+ with self.delimit("(", ")"):
+ self.interleave(
+ lambda: self.write(", "), self.traverse, node.argtypes
+ )
+
+ self.write(" -> ")
+ self.traverse(node.returns)
+
+ def visit_Expr(self, node):
+ self.fill()
+ self.set_precedence(_Precedence.YIELD, node.value)
+ self.traverse(node.value)
+
+ def visit_NamedExpr(self, node):
+ with self.require_parens(_Precedence.NAMED_EXPR, node):
+ self.set_precedence(_Precedence.ATOM, node.target, node.value)
+ self.traverse(node.target)
+ self.write(" := ")
+ self.traverse(node.value)
+
+ def visit_Import(self, node):
+ self.fill("import ")
+ self.interleave(lambda: self.write(", "), self.traverse, node.names)
+
+ def visit_ImportFrom(self, node):
+ self.fill("from ")
+ self.write("." * (node.level or 0))
+ if node.module:
+ self.write(node.module)
+ self.write(" import ")
+ self.interleave(lambda: self.write(", "), self.traverse, node.names)
+
+ def visit_Assign(self, node):
+ self.fill()
+ for target in node.targets:
+ self.set_precedence(_Precedence.TUPLE, target)
+ self.traverse(target)
+ self.write(" = ")
+ self.traverse(node.value)
+ if type_comment := self.get_type_comment(node):
+ self.write(type_comment)
+
+ def visit_AugAssign(self, node):
+ self.fill()
+ self.traverse(node.target)
+ self.write(" " + self.binop[node.op.__class__.__name__] + "= ")
+ self.traverse(node.value)
+
+ def visit_AnnAssign(self, node):
+ self.fill()
+ with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)):
+ self.traverse(node.target)
+ self.write(": ")
+ self.traverse(node.annotation)
+ if node.value:
+ self.write(" = ")
+ self.traverse(node.value)
+
+ def visit_Return(self, node):
+ self.fill("return")
+ if node.value:
+ self.write(" ")
+ self.traverse(node.value)
+
+ def visit_Pass(self, node):
+ self.fill("pass")
+
+ def visit_Break(self, node):
+ self.fill("break")
+
+ def visit_Continue(self, node):
+ self.fill("continue")
+
+ def visit_Delete(self, node):
+ self.fill("del ")
+ self.interleave(lambda: self.write(", "), self.traverse, node.targets)
+
+ def visit_Assert(self, node):
+ self.fill("assert ")
+ self.traverse(node.test)
+ if node.msg:
+ self.write(", ")
+ self.traverse(node.msg)
+
+ def visit_Global(self, node):
+ self.fill("global ")
+ self.interleave(lambda: self.write(", "), self.write, node.names)
+
+ def visit_Nonlocal(self, node):
+ self.fill("nonlocal ")
+ self.interleave(lambda: self.write(", "), self.write, node.names)
+
+ def visit_Await(self, node):
+ with self.require_parens(_Precedence.AWAIT, node):
+ self.write("await")
+ if node.value:
+ self.write(" ")
+ self.set_precedence(_Precedence.ATOM, node.value)
+ self.traverse(node.value)
+
+ def visit_Yield(self, node):
+ with self.require_parens(_Precedence.YIELD, node):
+ self.write("yield")
+ if node.value:
+ self.write(" ")
+ self.set_precedence(_Precedence.ATOM, node.value)
+ self.traverse(node.value)
+
+ def visit_YieldFrom(self, node):
+ with self.require_parens(_Precedence.YIELD, node):
+ self.write("yield from ")
+ if not node.value:
+ raise ValueError("Node can't be used without a value attribute.")
+ self.set_precedence(_Precedence.ATOM, node.value)
+ self.traverse(node.value)
+
+ def visit_Raise(self, node):
+ self.fill("raise")
+ if not node.exc:
+ if node.cause:
+ raise ValueError(f"Node can't use cause without an exception.")
+ return
+ self.write(" ")
+ self.traverse(node.exc)
+ if node.cause:
+ self.write(" from ")
+ self.traverse(node.cause)
+
+ def do_visit_try(self, node):
+ self.fill("try", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.body)
+ for ex in node.handlers:
+ self.traverse(ex)
+ if node.orelse:
+ self.fill("else", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.orelse)
+ if node.finalbody:
+ self.fill("finally", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.finalbody)
+
+ def visit_Try(self, node):
+ prev_in_try_star = self._in_try_star
+ try:
+ self._in_try_star = False
+ self.do_visit_try(node)
+ finally:
+ self._in_try_star = prev_in_try_star
+
+ def visit_TryStar(self, node):
+ prev_in_try_star = self._in_try_star
+ try:
+ self._in_try_star = True
+ self.do_visit_try(node)
+ finally:
+ self._in_try_star = prev_in_try_star
+
+ def visit_ExceptHandler(self, node):
+ self.fill("except*" if self._in_try_star else "except", allow_semicolon=False)
+ if node.type:
+ self.write(" ")
+ self.traverse(node.type)
+ if node.name:
+ self.write(" as ")
+ self.write(node.name)
+ with self.block():
+ self.traverse(node.body)
+
+ def visit_ClassDef(self, node):
+ self.maybe_newline()
+ for deco in node.decorator_list:
+ self.fill("@", allow_semicolon=False)
+ self.traverse(deco)
+ self.fill("class " + node.name, allow_semicolon=False)
+ if hasattr(node, "type_params"):
+ self._type_params_helper(node.type_params)
+ with self.delimit_if("(", ")", condition = node.bases or node.keywords):
+ comma = False
+ for e in node.bases:
+ if comma:
+ self.write(", ")
+ else:
+ comma = True
+ self.traverse(e)
+ for e in node.keywords:
+ if comma:
+ self.write(", ")
+ else:
+ comma = True
+ self.traverse(e)
+
+ with self.block():
+ self._write_docstring_and_traverse_body(node)
+
+ def visit_FunctionDef(self, node):
+ self._function_helper(node, "def")
+
+ def visit_AsyncFunctionDef(self, node):
+ self._function_helper(node, "async def")
+
+ def _function_helper(self, node, fill_suffix):
+ self.maybe_newline()
+ for deco in node.decorator_list:
+ self.fill("@", allow_semicolon=False)
+ self.traverse(deco)
+ def_str = fill_suffix + " " + node.name
+ self.fill(def_str, allow_semicolon=False)
+ if hasattr(node, "type_params"):
+ self._type_params_helper(node.type_params)
+ with self.delimit("(", ")"):
+ self.traverse(node.args)
+ if node.returns:
+ self.write(" -> ")
+ self.traverse(node.returns)
+ with self.block(extra=self.get_type_comment(node)):
+ self._write_docstring_and_traverse_body(node)
+
+ def _type_params_helper(self, type_params):
+ if type_params is not None and len(type_params) > 0:
+ with self.delimit("[", "]"):
+ self.interleave(lambda: self.write(", "), self.traverse, type_params)
+
+ def visit_TypeVar(self, node):
+ self.write(node.name)
+ if node.bound:
+ self.write(": ")
+ self.traverse(node.bound)
+ if node.default_value:
+ self.write(" = ")
+ self.traverse(node.default_value)
+
+ def visit_TypeVarTuple(self, node):
+ self.write("*" + node.name)
+ if node.default_value:
+ self.write(" = ")
+ self.traverse(node.default_value)
+
+ def visit_ParamSpec(self, node):
+ self.write("**" + node.name)
+ if node.default_value:
+ self.write(" = ")
+ self.traverse(node.default_value)
+
+ def visit_TypeAlias(self, node):
+ self.fill("type ")
+ self.traverse(node.name)
+ self._type_params_helper(node.type_params)
+ self.write(" = ")
+ self.traverse(node.value)
+
+ def visit_For(self, node):
+ self._for_helper("for ", node)
+
+ def visit_AsyncFor(self, node):
+ self._for_helper("async for ", node)
+
+ def _for_helper(self, fill, node):
+ self.fill(fill, allow_semicolon=False)
+ self.set_precedence(_Precedence.TUPLE, node.target)
+ self.traverse(node.target)
+ self.write(" in ")
+ self.traverse(node.iter)
+ with self.block(extra=self.get_type_comment(node)):
+ self.traverse(node.body)
+ if node.orelse:
+ self.fill("else", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.orelse)
+
+ def visit_If(self, node):
+ self.fill("if ", allow_semicolon=False)
+ self.traverse(node.test)
+ with self.block():
+ self.traverse(node.body)
+ # collapse nested ifs into equivalent elifs.
+ while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If):
+ node = node.orelse[0]
+ self.fill("elif ", allow_semicolon=False)
+ self.traverse(node.test)
+ with self.block():
+ self.traverse(node.body)
+ # final else
+ if node.orelse:
+ self.fill("else", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.orelse)
+
+ def visit_While(self, node):
+ self.fill("while ", allow_semicolon=False)
+ self.traverse(node.test)
+ with self.block():
+ self.traverse(node.body)
+ if node.orelse:
+ self.fill("else", allow_semicolon=False)
+ with self.block():
+ self.traverse(node.orelse)
+
+ def visit_With(self, node):
+ self.fill("with ", allow_semicolon=False)
+ self.interleave(lambda: self.write(", "), self.traverse, node.items)
+ with self.block(extra=self.get_type_comment(node)):
+ self.traverse(node.body)
+
+ def visit_AsyncWith(self, node):
+ self.fill("async with ", allow_semicolon=False)
+ self.interleave(lambda: self.write(", "), self.traverse, node.items)
+ with self.block(extra=self.get_type_comment(node)):
+ self.traverse(node.body)
+
+ def _str_literal_helper(
+ self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False
+ ):
+ """Helper for writing string literals, minimizing escapes.
+ Returns the tuple (string literal to write, possible quote types).
+ """
+ def escape_char(c):
+ # \n and \t are non-printable, but we only escape them if
+ # escape_special_whitespace is True
+ if not escape_special_whitespace and c in "\n\t":
+ return c
+ # Always escape backslashes and other non-printable characters
+ if c == "\\" or not c.isprintable():
+ return c.encode("unicode_escape").decode("ascii")
+ return c
+
+ escaped_string = "".join(map(escape_char, string))
+ possible_quotes = quote_types
+ if "\n" in escaped_string:
+ possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES]
+ possible_quotes = [q for q in possible_quotes if q not in escaped_string]
+ if not possible_quotes:
+ # If there aren't any possible_quotes, fallback to using repr
+ # on the original string. Try to use a quote from quote_types,
+ # e.g., so that we use triple quotes for docstrings.
+ string = repr(string)
+ quote = next((q for q in quote_types if string[0] in q), string[0])
+ return string[1:-1], [quote]
+ if escaped_string:
+ # Sort so that we prefer '''"''' over """\""""
+ possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1])
+ # If we're using triple quotes and we'd need to escape a final
+ # quote, escape it
+ if possible_quotes[0][0] == escaped_string[-1]:
+ assert len(possible_quotes[0]) == 3
+ escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1]
+ return escaped_string, possible_quotes
+
+ def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES):
+ """Write string literal value with a best effort attempt to avoid backslashes."""
+ string, quote_types = self._str_literal_helper(string, quote_types=quote_types)
+ quote_type = quote_types[0]
+ self.write(f"{quote_type}{string}{quote_type}")
+
+ def _ftstring_helper(self, parts):
+ new_parts = []
+ quote_types = list(_ALL_QUOTES)
+ fallback_to_repr = False
+ for value, is_constant in parts:
+ if is_constant:
+ value, new_quote_types = self._str_literal_helper(
+ value,
+ quote_types=quote_types,
+ escape_special_whitespace=True,
+ )
+ if set(new_quote_types).isdisjoint(quote_types):
+ fallback_to_repr = True
+ break
+ quote_types = new_quote_types
+ else:
+ if "\n" in value:
+ quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
+ assert quote_types
+
+ new_quote_types = [q for q in quote_types if q not in value]
+ if new_quote_types:
+ quote_types = new_quote_types
+ new_parts.append(value)
+
+ if fallback_to_repr:
+ # If we weren't able to find a quote type that works for all parts
+ # of the JoinedStr, fallback to using repr and triple single quotes.
+ quote_types = ["'''"]
+ new_parts.clear()
+ for value, is_constant in parts:
+ if is_constant:
+ value = repr('"' + value) # force repr to use single quotes
+ expected_prefix = "'\""
+ assert value.startswith(expected_prefix), repr(value)
+ value = value[len(expected_prefix):-1]
+ new_parts.append(value)
+
+ value = "".join(new_parts)
+ quote_type = quote_types[0]
+ self.write(f"{quote_type}{value}{quote_type}")
+
+ def _write_ftstring(self, values, prefix):
+ self.write(prefix)
+ fstring_parts = []
+ for value in values:
+ with self.buffered() as buffer:
+ self._write_ftstring_inner(value)
+ fstring_parts.append(
+ ("".join(buffer), isinstance(value, Constant))
+ )
+ self._ftstring_helper(fstring_parts)
+
+ def visit_JoinedStr(self, node):
+ self._write_ftstring(node.values, "f")
+
+ def visit_TemplateStr(self, node):
+ self._write_ftstring(node.values, "t")
+
+ def _write_ftstring_inner(self, node, is_format_spec=False):
+ if isinstance(node, JoinedStr):
+ # for both the f-string itself, and format_spec
+ for value in node.values:
+ self._write_ftstring_inner(value, is_format_spec=is_format_spec)
+ elif isinstance(node, Constant) and isinstance(node.value, str):
+ value = node.value.replace("{", "{{").replace("}", "}}")
+
+ if is_format_spec:
+ value = value.replace("\\", "\\\\")
+ value = value.replace("'", "\\'")
+ value = value.replace('"', '\\"')
+ value = value.replace("\n", "\\n")
+ self.write(value)
+ elif isinstance(node, FormattedValue):
+ self.visit_FormattedValue(node)
+ elif isinstance(node, Interpolation):
+ self.visit_Interpolation(node)
+ else:
+ raise ValueError(f"Unexpected node inside JoinedStr, {node!r}")
+
+ def _unparse_interpolation_value(self, inner):
+ unparser = type(self)()
+ unparser.set_precedence(_Precedence.TEST.next(), inner)
+ return unparser.visit(inner)
+
+ def _write_interpolation(self, node, use_str_attr=False):
+ with self.delimit("{", "}"):
+ if use_str_attr:
+ expr = node.str
+ else:
+ expr = self._unparse_interpolation_value(node.value)
+ if expr.startswith("{"):
+ # Separate pair of opening brackets as "{ {"
+ self.write(" ")
+ self.write(expr)
+ if node.conversion != -1:
+ self.write(f"!{chr(node.conversion)}")
+ if node.format_spec:
+ self.write(":")
+ self._write_ftstring_inner(node.format_spec, is_format_spec=True)
+
+ def visit_FormattedValue(self, node):
+ self._write_interpolation(node)
+
+ def visit_Interpolation(self, node):
+ # If `str` is set to `None`, use the `value` to generate the source code.
+ self._write_interpolation(node, use_str_attr=node.str is not None)
+
+ def visit_Name(self, node):
+ self.write(node.id)
+
+ def _write_docstring(self, node):
+ self.fill(allow_semicolon=False)
+ if node.kind == "u":
+ self.write("u")
+ self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES)
+
+ def _write_constant(self, value):
+ if isinstance(value, (float, complex)):
+ # Substitute overflowing decimal literal for AST infinities,
+ # and inf - inf for NaNs.
+ self.write(
+ repr(value)
+ .replace("inf", _INFSTR)
+ .replace("nan", f"({_INFSTR}-{_INFSTR})")
+ )
+ else:
+ self.write(repr(value))
+
+ def visit_Constant(self, node):
+ value = node.value
+ if isinstance(value, tuple):
+ with self.delimit("(", ")"):
+ self.items_view(self._write_constant, value)
+ elif value is ...:
+ self.write("...")
+ else:
+ if node.kind == "u":
+ self.write("u")
+ self._write_constant(node.value)
+
+ def visit_List(self, node):
+ with self.delimit("[", "]"):
+ self.interleave(lambda: self.write(", "), self.traverse, node.elts)
+
+ def visit_ListComp(self, node):
+ with self.delimit("[", "]"):
+ self.traverse(node.elt)
+ for gen in node.generators:
+ self.traverse(gen)
+
+ def visit_GeneratorExp(self, node):
+ with self.delimit("(", ")"):
+ self.traverse(node.elt)
+ for gen in node.generators:
+ self.traverse(gen)
+
+ def visit_SetComp(self, node):
+ with self.delimit("{", "}"):
+ self.traverse(node.elt)
+ for gen in node.generators:
+ self.traverse(gen)
+
+ def visit_DictComp(self, node):
+ with self.delimit("{", "}"):
+ self.traverse(node.key)
+ self.write(": ")
+ self.traverse(node.value)
+ for gen in node.generators:
+ self.traverse(gen)
+
+ def visit_comprehension(self, node):
+ if node.is_async:
+ self.write(" async for ")
+ else:
+ self.write(" for ")
+ self.set_precedence(_Precedence.TUPLE, node.target)
+ self.traverse(node.target)
+ self.write(" in ")
+ self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs)
+ self.traverse(node.iter)
+ for if_clause in node.ifs:
+ self.write(" if ")
+ self.traverse(if_clause)
+
+ def visit_IfExp(self, node):
+ with self.require_parens(_Precedence.TEST, node):
+ self.set_precedence(_Precedence.TEST.next(), node.body, node.test)
+ self.traverse(node.body)
+ self.write(" if ")
+ self.traverse(node.test)
+ self.write(" else ")
+ self.set_precedence(_Precedence.TEST, node.orelse)
+ self.traverse(node.orelse)
+
+ def visit_Set(self, node):
+ if node.elts:
+ with self.delimit("{", "}"):
+ self.interleave(lambda: self.write(", "), self.traverse, node.elts)
+ else:
+ # `{}` would be interpreted as a dictionary literal, and
+ # `set` might be shadowed. Thus:
+ self.write('{*()}')
+
+ def visit_Dict(self, node):
+ def write_key_value_pair(k, v):
+ self.traverse(k)
+ self.write(": ")
+ self.traverse(v)
+
+ def write_item(item):
+ k, v = item
+ if k is None:
+ # for dictionary unpacking operator in dicts {**{'y': 2}}
+ # see PEP 448 for details
+ self.write("**")
+ self.set_precedence(_Precedence.EXPR, v)
+ self.traverse(v)
+ else:
+ write_key_value_pair(k, v)
+
+ with self.delimit("{", "}"):
+ self.interleave(
+ lambda: self.write(", "), write_item, zip(node.keys, node.values)
+ )
+
+ def visit_Tuple(self, node):
+ with self.delimit_if(
+ "(",
+ ")",
+ len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE
+ ):
+ self.items_view(self.traverse, node.elts)
+
+ unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}
+ unop_precedence = {
+ "not": _Precedence.NOT,
+ "~": _Precedence.FACTOR,
+ "+": _Precedence.FACTOR,
+ "-": _Precedence.FACTOR,
+ }
+
+ def visit_UnaryOp(self, node):
+ operator = self.unop[node.op.__class__.__name__]
+ operator_precedence = self.unop_precedence[operator]
+ with self.require_parens(operator_precedence, node):
+ self.write(operator)
+ # factor prefixes (+, -, ~) shouldn't be separated
+ # from the value they belong, (e.g: +1 instead of + 1)
+ if operator_precedence is not _Precedence.FACTOR:
+ self.write(" ")
+ self.set_precedence(operator_precedence, node.operand)
+ self.traverse(node.operand)
+
+ binop = {
+ "Add": "+",
+ "Sub": "-",
+ "Mult": "*",
+ "MatMult": "@",
+ "Div": "/",
+ "Mod": "%",
+ "LShift": "<<",
+ "RShift": ">>",
+ "BitOr": "|",
+ "BitXor": "^",
+ "BitAnd": "&",
+ "FloorDiv": "//",
+ "Pow": "**",
+ }
+
+ binop_precedence = {
+ "+": _Precedence.ARITH,
+ "-": _Precedence.ARITH,
+ "*": _Precedence.TERM,
+ "@": _Precedence.TERM,
+ "/": _Precedence.TERM,
+ "%": _Precedence.TERM,
+ "<<": _Precedence.SHIFT,
+ ">>": _Precedence.SHIFT,
+ "|": _Precedence.BOR,
+ "^": _Precedence.BXOR,
+ "&": _Precedence.BAND,
+ "//": _Precedence.TERM,
+ "**": _Precedence.POWER,
+ }
+
+ binop_rassoc = frozenset(("**",))
+ def visit_BinOp(self, node):
+ operator = self.binop[node.op.__class__.__name__]
+ operator_precedence = self.binop_precedence[operator]
+ with self.require_parens(operator_precedence, node):
+ if operator in self.binop_rassoc:
+ left_precedence = operator_precedence.next()
+ right_precedence = operator_precedence
+ else:
+ left_precedence = operator_precedence
+ right_precedence = operator_precedence.next()
+
+ self.set_precedence(left_precedence, node.left)
+ self.traverse(node.left)
+ self.write(f" {operator} ")
+ self.set_precedence(right_precedence, node.right)
+ self.traverse(node.right)
+
+ cmpops = {
+ "Eq": "==",
+ "NotEq": "!=",
+ "Lt": "<",
+ "LtE": "<=",
+ "Gt": ">",
+ "GtE": ">=",
+ "Is": "is",
+ "IsNot": "is not",
+ "In": "in",
+ "NotIn": "not in",
+ }
+
+ def visit_Compare(self, node):
+ with self.require_parens(_Precedence.CMP, node):
+ self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators)
+ self.traverse(node.left)
+ for o, e in zip(node.ops, node.comparators):
+ self.write(" " + self.cmpops[o.__class__.__name__] + " ")
+ self.traverse(e)
+
+ boolops = {"And": "and", "Or": "or"}
+ boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR}
+
+ def visit_BoolOp(self, node):
+ operator = self.boolops[node.op.__class__.__name__]
+ operator_precedence = self.boolop_precedence[operator]
+
+ def increasing_level_traverse(node):
+ nonlocal operator_precedence
+ operator_precedence = operator_precedence.next()
+ self.set_precedence(operator_precedence, node)
+ self.traverse(node)
+
+ with self.require_parens(operator_precedence, node):
+ s = f" {operator} "
+ self.interleave(lambda: self.write(s), increasing_level_traverse, node.values)
+
+ def visit_Attribute(self, node):
+ self.set_precedence(_Precedence.ATOM, node.value)
+ self.traverse(node.value)
+ # Special case: 3.__abs__() is a syntax error, so if node.value
+ # is an integer literal then we need to either parenthesize
+ # it or add an extra space to get 3 .__abs__().
+ if isinstance(node.value, Constant) and isinstance(node.value.value, int):
+ self.write(" ")
+ self.write(".")
+ self.write(node.attr)
+
+ def visit_Call(self, node):
+ self.set_precedence(_Precedence.ATOM, node.func)
+ self.traverse(node.func)
+ with self.delimit("(", ")"):
+ comma = False
+ for e in node.args:
+ if comma:
+ self.write(", ")
+ else:
+ comma = True
+ self.traverse(e)
+ for e in node.keywords:
+ if comma:
+ self.write(", ")
+ else:
+ comma = True
+ self.traverse(e)
+
+ def visit_Subscript(self, node):
+ def is_non_empty_tuple(slice_value):
+ return (
+ isinstance(slice_value, Tuple)
+ and slice_value.elts
+ )
+
+ self.set_precedence(_Precedence.ATOM, node.value)
+ self.traverse(node.value)
+ with self.delimit("[", "]"):
+ if is_non_empty_tuple(node.slice):
+ # parentheses can be omitted if the tuple isn't empty
+ self.items_view(self.traverse, node.slice.elts)
+ else:
+ self.traverse(node.slice)
+
+ def visit_Starred(self, node):
+ self.write("*")
+ self.set_precedence(_Precedence.EXPR, node.value)
+ self.traverse(node.value)
+
+ def visit_Ellipsis(self, node):
+ self.write("...")
+
+ def visit_Slice(self, node):
+ if node.lower:
+ self.traverse(node.lower)
+ self.write(":")
+ if node.upper:
+ self.traverse(node.upper)
+ if node.step:
+ self.write(":")
+ self.traverse(node.step)
+
+ def visit_Match(self, node):
+ self.fill("match ", allow_semicolon=False)
+ self.traverse(node.subject)
+ with self.block():
+ for case in node.cases:
+ self.traverse(case)
+
+ def visit_arg(self, node):
+ self.write(node.arg)
+ if node.annotation:
+ self.write(": ")
+ self.traverse(node.annotation)
+
+ def visit_arguments(self, node):
+ first = True
+ # normal arguments
+ all_args = node.posonlyargs + node.args
+ defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults
+ for index, elements in enumerate(zip(all_args, defaults), 1):
+ a, d = elements
+ if first:
+ first = False
+ else:
+ self.write(", ")
+ self.traverse(a)
+ if d:
+ self.write("=")
+ self.traverse(d)
+ if index == len(node.posonlyargs):
+ self.write(", /")
+
+ # varargs, or bare '*' if no varargs but keyword-only arguments present
+ if node.vararg or node.kwonlyargs:
+ if first:
+ first = False
+ else:
+ self.write(", ")
+ self.write("*")
+ if node.vararg:
+ self.write(node.vararg.arg)
+ if node.vararg.annotation:
+ self.write(": ")
+ self.traverse(node.vararg.annotation)
+
+ # keyword-only arguments
+ if node.kwonlyargs:
+ for a, d in zip(node.kwonlyargs, node.kw_defaults):
+ self.write(", ")
+ self.traverse(a)
+ if d:
+ self.write("=")
+ self.traverse(d)
+
+ # kwargs
+ if node.kwarg:
+ if first:
+ first = False
+ else:
+ self.write(", ")
+ self.write("**" + node.kwarg.arg)
+ if node.kwarg.annotation:
+ self.write(": ")
+ self.traverse(node.kwarg.annotation)
+
+ def visit_keyword(self, node):
+ if node.arg is None:
+ self.write("**")
+ else:
+ self.write(node.arg)
+ self.write("=")
+ self.traverse(node.value)
+
+ def visit_Lambda(self, node):
+ with self.require_parens(_Precedence.TEST, node):
+ self.write("lambda")
+ with self.buffered() as buffer:
+ self.traverse(node.args)
+ if buffer:
+ self.write(" ", *buffer)
+ self.write(": ")
+ self.set_precedence(_Precedence.TEST, node.body)
+ self.traverse(node.body)
+
+ def visit_alias(self, node):
+ self.write(node.name)
+ if node.asname:
+ self.write(" as " + node.asname)
+
+ def visit_withitem(self, node):
+ self.traverse(node.context_expr)
+ if node.optional_vars:
+ self.write(" as ")
+ self.traverse(node.optional_vars)
+
+ def visit_match_case(self, node):
+ self.fill("case ", allow_semicolon=False)
+ self.traverse(node.pattern)
+ if node.guard:
+ self.write(" if ")
+ self.traverse(node.guard)
+ with self.block():
+ self.traverse(node.body)
+
+ def visit_MatchValue(self, node):
+ self.traverse(node.value)
+
+ def visit_MatchSingleton(self, node):
+ self._write_constant(node.value)
+
+ def visit_MatchSequence(self, node):
+ with self.delimit("[", "]"):
+ self.interleave(
+ lambda: self.write(", "), self.traverse, node.patterns
+ )
+
+ def visit_MatchStar(self, node):
+ name = node.name
+ if name is None:
+ name = "_"
+ self.write(f"*{name}")
+
+ def visit_MatchMapping(self, node):
+ def write_key_pattern_pair(pair):
+ k, p = pair
+ self.traverse(k)
+ self.write(": ")
+ self.traverse(p)
+
+ with self.delimit("{", "}"):
+ keys = node.keys
+ self.interleave(
+ lambda: self.write(", "),
+ write_key_pattern_pair,
+ zip(keys, node.patterns, strict=True),
+ )
+ rest = node.rest
+ if rest is not None:
+ if keys:
+ self.write(", ")
+ self.write(f"**{rest}")
+
+ def visit_MatchClass(self, node):
+ self.set_precedence(_Precedence.ATOM, node.cls)
+ self.traverse(node.cls)
+ with self.delimit("(", ")"):
+ patterns = node.patterns
+ self.interleave(
+ lambda: self.write(", "), self.traverse, patterns
+ )
+ attrs = node.kwd_attrs
+ if attrs:
+ def write_attr_pattern(pair):
+ attr, pattern = pair
+ self.write(f"{attr}=")
+ self.traverse(pattern)
+
+ if patterns:
+ self.write(", ")
+ self.interleave(
+ lambda: self.write(", "),
+ write_attr_pattern,
+ zip(attrs, node.kwd_patterns, strict=True),
+ )
+
+ def visit_MatchAs(self, node):
+ name = node.name
+ pattern = node.pattern
+ if name is None:
+ self.write("_")
+ elif pattern is None:
+ self.write(node.name)
+ else:
+ with self.require_parens(_Precedence.TEST, node):
+ self.set_precedence(_Precedence.BOR, node.pattern)
+ self.traverse(node.pattern)
+ self.write(f" as {node.name}")
+
+ def visit_MatchOr(self, node):
+ with self.require_parens(_Precedence.BOR, node):
+ self.set_precedence(_Precedence.BOR.next(), *node.patterns)
+ self.interleave(lambda: self.write(" | "), self.traverse, node.patterns)
diff --git a/PythonLib/full/_collections_abc.py b/PythonLib/full/_collections_abc.py
index 09745658d..241d40d57 100644
--- a/PythonLib/full/_collections_abc.py
+++ b/PythonLib/full/_collections_abc.py
@@ -85,6 +85,10 @@ def _f(): pass
dict_items = type({}.items())
## misc ##
mappingproxy = type(type.__dict__)
+def _get_framelocalsproxy():
+ return type(sys._getframe().f_locals)
+framelocalsproxy = _get_framelocalsproxy()
+del _get_framelocalsproxy
generator = type((lambda: (yield))())
## coroutine ##
async def _coro(): pass
@@ -481,9 +485,10 @@ def __new__(cls, origin, args):
def __repr__(self):
if len(self.__args__) == 2 and _is_param_expr(self.__args__[0]):
return super().__repr__()
+ from annotationlib import type_repr
return (f'collections.abc.Callable'
- f'[[{", ".join([_type_repr(a) for a in self.__args__[:-1]])}], '
- f'{_type_repr(self.__args__[-1])}]')
+ f'[[{", ".join([type_repr(a) for a in self.__args__[:-1]])}], '
+ f'{type_repr(self.__args__[-1])}]')
def __reduce__(self):
args = self.__args__
@@ -520,23 +525,6 @@ def _is_param_expr(obj):
names = ('ParamSpec', '_ConcatenateGenericAlias')
return obj.__module__ == 'typing' and any(obj.__name__ == name for name in names)
-def _type_repr(obj):
- """Return the repr() of an object, special-casing types (internal helper).
-
- Copied from :mod:`typing` since collections.abc
- shouldn't depend on that module.
- (Keep this roughly in sync with the typing version.)
- """
- if isinstance(obj, type):
- if obj.__module__ == 'builtins':
- return obj.__qualname__
- return f'{obj.__module__}.{obj.__qualname__}'
- if obj is Ellipsis:
- return '...'
- if isinstance(obj, FunctionType):
- return obj.__name__
- return repr(obj)
-
class Callable(metaclass=ABCMeta):
@@ -836,6 +824,7 @@ def __eq__(self, other):
__reversed__ = None
Mapping.register(mappingproxy)
+Mapping.register(framelocalsproxy)
class MappingView(Sized):
@@ -1068,6 +1057,7 @@ def count(self, value):
Sequence.register(tuple)
Sequence.register(str)
+Sequence.register(bytes)
Sequence.register(range)
Sequence.register(memoryview)
@@ -1078,7 +1068,7 @@ def __new__(cls, name, bases, namespace, **kwargs):
warnings._deprecated(
"collections.abc.ByteString",
- remove=(3, 14),
+ remove=(3, 17),
)
return super().__new__(cls, name, bases, namespace, **kwargs)
@@ -1087,14 +1077,18 @@ def __instancecheck__(cls, instance):
warnings._deprecated(
"collections.abc.ByteString",
- remove=(3, 14),
+ remove=(3, 17),
)
return super().__instancecheck__(instance)
class ByteString(Sequence, metaclass=_DeprecateByteStringMeta):
- """This unifies bytes and bytearray.
+ """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``.
- XXX Should add all their methods.
+ This ABC is scheduled for removal in Python 3.17.
+ Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj``
+ implements the buffer protocol at runtime. For use in type annotations,
+ either use ``Buffer`` or a union that explicitly specifies the types your
+ code supports (e.g., ``bytes | bytearray | memoryview``).
"""
__slots__ = ()
@@ -1170,4 +1164,4 @@ def __iadd__(self, values):
MutableSequence.register(list)
-MutableSequence.register(bytearray) # Multiply inheriting, see ByteString
+MutableSequence.register(bytearray)
diff --git a/PythonLib/full/_colorize.py b/PythonLib/full/_colorize.py
new file mode 100644
index 000000000..d6673f669
--- /dev/null
+++ b/PythonLib/full/_colorize.py
@@ -0,0 +1,355 @@
+import os
+import sys
+
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass, field, Field
+
+COLORIZE = True
+
+
+# types
+if False:
+ from typing import IO, Self, ClassVar
+ _theme: Theme
+
+
+class ANSIColors:
+ RESET = "\x1b[0m"
+
+ BLACK = "\x1b[30m"
+ BLUE = "\x1b[34m"
+ CYAN = "\x1b[36m"
+ GREEN = "\x1b[32m"
+ GREY = "\x1b[90m"
+ MAGENTA = "\x1b[35m"
+ RED = "\x1b[31m"
+ WHITE = "\x1b[37m" # more like LIGHT GRAY
+ YELLOW = "\x1b[33m"
+
+ BOLD = "\x1b[1m"
+ BOLD_BLACK = "\x1b[1;30m" # DARK GRAY
+ BOLD_BLUE = "\x1b[1;34m"
+ BOLD_CYAN = "\x1b[1;36m"
+ BOLD_GREEN = "\x1b[1;32m"
+ BOLD_MAGENTA = "\x1b[1;35m"
+ BOLD_RED = "\x1b[1;31m"
+ BOLD_WHITE = "\x1b[1;37m" # actual WHITE
+ BOLD_YELLOW = "\x1b[1;33m"
+
+ # intense = like bold but without being bold
+ INTENSE_BLACK = "\x1b[90m"
+ INTENSE_BLUE = "\x1b[94m"
+ INTENSE_CYAN = "\x1b[96m"
+ INTENSE_GREEN = "\x1b[92m"
+ INTENSE_MAGENTA = "\x1b[95m"
+ INTENSE_RED = "\x1b[91m"
+ INTENSE_WHITE = "\x1b[97m"
+ INTENSE_YELLOW = "\x1b[93m"
+
+ BACKGROUND_BLACK = "\x1b[40m"
+ BACKGROUND_BLUE = "\x1b[44m"
+ BACKGROUND_CYAN = "\x1b[46m"
+ BACKGROUND_GREEN = "\x1b[42m"
+ BACKGROUND_MAGENTA = "\x1b[45m"
+ BACKGROUND_RED = "\x1b[41m"
+ BACKGROUND_WHITE = "\x1b[47m"
+ BACKGROUND_YELLOW = "\x1b[43m"
+
+ INTENSE_BACKGROUND_BLACK = "\x1b[100m"
+ INTENSE_BACKGROUND_BLUE = "\x1b[104m"
+ INTENSE_BACKGROUND_CYAN = "\x1b[106m"
+ INTENSE_BACKGROUND_GREEN = "\x1b[102m"
+ INTENSE_BACKGROUND_MAGENTA = "\x1b[105m"
+ INTENSE_BACKGROUND_RED = "\x1b[101m"
+ INTENSE_BACKGROUND_WHITE = "\x1b[107m"
+ INTENSE_BACKGROUND_YELLOW = "\x1b[103m"
+
+
+ColorCodes = set()
+NoColors = ANSIColors()
+
+for attr, code in ANSIColors.__dict__.items():
+ if not attr.startswith("__"):
+ ColorCodes.add(code)
+ setattr(NoColors, attr, "")
+
+
+#
+# Experimental theming support (see gh-133346)
+#
+
+# - Create a theme by copying an existing `Theme` with one or more sections
+# replaced, using `default_theme.copy_with()`;
+# - create a theme section by copying an existing `ThemeSection` with one or
+# more colors replaced, using for example `default_theme.syntax.copy_with()`;
+# - create a theme from scratch by instantiating a `Theme` data class with
+# the required sections (which are also dataclass instances).
+#
+# Then call `_colorize.set_theme(your_theme)` to set it.
+#
+# Put your theme configuration in $PYTHONSTARTUP for the interactive shell,
+# or sitecustomize.py in your virtual environment or Python installation for
+# other uses. Your applications can call `_colorize.set_theme()` too.
+#
+# Note that thanks to the dataclasses providing default values for all fields,
+# creating a new theme or theme section from scratch is possible without
+# specifying all keys.
+#
+# For example, here's a theme that makes punctuation and operators less prominent:
+#
+# try:
+# from _colorize import set_theme, default_theme, Syntax, ANSIColors
+# except ImportError:
+# pass
+# else:
+# theme_with_dim_operators = default_theme.copy_with(
+# syntax=Syntax(op=ANSIColors.INTENSE_BLACK),
+# )
+# set_theme(theme_with_dim_operators)
+# del set_theme, default_theme, Syntax, ANSIColors, theme_with_dim_operators
+#
+# Guarding the import ensures that your .pythonstartup file will still work in
+# Python 3.13 and older. Deleting the variables ensures they don't remain in your
+# interactive shell's global scope.
+
+class ThemeSection(Mapping[str, str]):
+ """A mixin/base class for theme sections.
+
+ It enables dictionary access to a section, as well as implements convenience
+ methods.
+ """
+
+ # The two types below are just that: types to inform the type checker that the
+ # mixin will work in context of those fields existing
+ __dataclass_fields__: ClassVar[dict[str, Field[str]]]
+ _name_to_value: Callable[[str], str]
+
+ def __post_init__(self) -> None:
+ name_to_value = {}
+ for color_name in self.__dataclass_fields__:
+ name_to_value[color_name] = getattr(self, color_name)
+ super().__setattr__('_name_to_value', name_to_value.__getitem__)
+
+ def copy_with(self, **kwargs: str) -> Self:
+ color_state: dict[str, str] = {}
+ for color_name in self.__dataclass_fields__:
+ color_state[color_name] = getattr(self, color_name)
+ color_state.update(kwargs)
+ return type(self)(**color_state)
+
+ @classmethod
+ def no_colors(cls) -> Self:
+ color_state: dict[str, str] = {}
+ for color_name in cls.__dataclass_fields__:
+ color_state[color_name] = ""
+ return cls(**color_state)
+
+ def __getitem__(self, key: str) -> str:
+ return self._name_to_value(key)
+
+ def __len__(self) -> int:
+ return len(self.__dataclass_fields__)
+
+ def __iter__(self) -> Iterator[str]:
+ return iter(self.__dataclass_fields__)
+
+
+@dataclass(frozen=True, kw_only=True)
+class Argparse(ThemeSection):
+ usage: str = ANSIColors.BOLD_BLUE
+ prog: str = ANSIColors.BOLD_MAGENTA
+ prog_extra: str = ANSIColors.MAGENTA
+ heading: str = ANSIColors.BOLD_BLUE
+ summary_long_option: str = ANSIColors.CYAN
+ summary_short_option: str = ANSIColors.GREEN
+ summary_label: str = ANSIColors.YELLOW
+ summary_action: str = ANSIColors.GREEN
+ long_option: str = ANSIColors.BOLD_CYAN
+ short_option: str = ANSIColors.BOLD_GREEN
+ label: str = ANSIColors.BOLD_YELLOW
+ action: str = ANSIColors.BOLD_GREEN
+ reset: str = ANSIColors.RESET
+
+
+@dataclass(frozen=True)
+class Syntax(ThemeSection):
+ prompt: str = ANSIColors.BOLD_MAGENTA
+ keyword: str = ANSIColors.BOLD_BLUE
+ keyword_constant: str = ANSIColors.BOLD_BLUE
+ builtin: str = ANSIColors.CYAN
+ comment: str = ANSIColors.RED
+ string: str = ANSIColors.GREEN
+ number: str = ANSIColors.YELLOW
+ op: str = ANSIColors.RESET
+ definition: str = ANSIColors.BOLD
+ soft_keyword: str = ANSIColors.BOLD_BLUE
+ reset: str = ANSIColors.RESET
+
+
+@dataclass(frozen=True)
+class Traceback(ThemeSection):
+ type: str = ANSIColors.BOLD_MAGENTA
+ message: str = ANSIColors.MAGENTA
+ filename: str = ANSIColors.MAGENTA
+ line_no: str = ANSIColors.MAGENTA
+ frame: str = ANSIColors.MAGENTA
+ error_highlight: str = ANSIColors.BOLD_RED
+ error_range: str = ANSIColors.RED
+ reset: str = ANSIColors.RESET
+
+
+@dataclass(frozen=True)
+class Unittest(ThemeSection):
+ passed: str = ANSIColors.GREEN
+ warn: str = ANSIColors.YELLOW
+ fail: str = ANSIColors.RED
+ fail_info: str = ANSIColors.BOLD_RED
+ reset: str = ANSIColors.RESET
+
+
+@dataclass(frozen=True)
+class Theme:
+ """A suite of themes for all sections of Python.
+
+ When adding a new one, remember to also modify `copy_with` and `no_colors`
+ below.
+ """
+ argparse: Argparse = field(default_factory=Argparse)
+ syntax: Syntax = field(default_factory=Syntax)
+ traceback: Traceback = field(default_factory=Traceback)
+ unittest: Unittest = field(default_factory=Unittest)
+
+ def copy_with(
+ self,
+ *,
+ argparse: Argparse | None = None,
+ syntax: Syntax | None = None,
+ traceback: Traceback | None = None,
+ unittest: Unittest | None = None,
+ ) -> Self:
+ """Return a new Theme based on this instance with some sections replaced.
+
+ Themes are immutable to protect against accidental modifications that
+ could lead to invalid terminal states.
+ """
+ return type(self)(
+ argparse=argparse or self.argparse,
+ syntax=syntax or self.syntax,
+ traceback=traceback or self.traceback,
+ unittest=unittest or self.unittest,
+ )
+
+ @classmethod
+ def no_colors(cls) -> Self:
+ """Return a new Theme where colors in all sections are empty strings.
+
+ This allows writing user code as if colors are always used. The color
+ fields will be ANSI color code strings when colorization is desired
+ and possible, and empty strings otherwise.
+ """
+ return cls(
+ argparse=Argparse.no_colors(),
+ syntax=Syntax.no_colors(),
+ traceback=Traceback.no_colors(),
+ unittest=Unittest.no_colors(),
+ )
+
+
+def get_colors(
+ colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None
+) -> ANSIColors:
+ if colorize or can_colorize(file=file):
+ return ANSIColors()
+ else:
+ return NoColors
+
+
+def decolor(text: str) -> str:
+ """Remove ANSI color codes from a string."""
+ for code in ColorCodes:
+ text = text.replace(code, "")
+ return text
+
+
+def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool:
+
+ def _safe_getenv(k: str, fallback: str | None = None) -> str | None:
+ """Exception-safe environment retrieval. See gh-128636."""
+ try:
+ return os.environ.get(k, fallback)
+ except Exception:
+ return fallback
+
+ if file is None:
+ file = sys.stdout
+
+ if not sys.flags.ignore_environment:
+ if _safe_getenv("PYTHON_COLORS") == "0":
+ return False
+ if _safe_getenv("PYTHON_COLORS") == "1":
+ return True
+ if _safe_getenv("NO_COLOR"):
+ return False
+ if not COLORIZE:
+ return False
+ if _safe_getenv("FORCE_COLOR"):
+ return True
+ if _safe_getenv("TERM") == "dumb":
+ return False
+
+ if not hasattr(file, "fileno"):
+ return False
+
+ if sys.platform == "win32":
+ try:
+ import nt
+
+ if not nt._supports_virtual_terminal():
+ return False
+ except (ImportError, AttributeError):
+ return False
+
+ try:
+ return os.isatty(file.fileno())
+ except OSError:
+ return hasattr(file, "isatty") and file.isatty()
+
+
+default_theme = Theme()
+theme_no_color = default_theme.no_colors()
+
+
+def get_theme(
+ *,
+ tty_file: IO[str] | IO[bytes] | None = None,
+ force_color: bool = False,
+ force_no_color: bool = False,
+) -> Theme:
+ """Returns the currently set theme, potentially in a zero-color variant.
+
+ In cases where colorizing is not possible (see `can_colorize`), the returned
+ theme contains all empty strings in all color definitions.
+ See `Theme.no_colors()` for more information.
+
+ It is recommended not to cache the result of this function for extended
+ periods of time because the user might influence theme selection by
+ the interactive shell, a debugger, or application-specific code. The
+ environment (including environment variable state and console configuration
+ on Windows) can also change in the course of the application life cycle.
+ """
+ if force_color or (not force_no_color and
+ can_colorize(file=tty_file)):
+ return _theme
+ return theme_no_color
+
+
+def set_theme(t: Theme) -> None:
+ global _theme
+
+ if not isinstance(t, Theme):
+ raise ValueError(f"Expected Theme object, found {t}")
+
+ _theme = t
+
+
+set_theme(default_theme)
diff --git a/PythonLib/full/_compat_pickle.py b/PythonLib/full/_compat_pickle.py
index 65a94b6b1..439f8c02f 100644
--- a/PythonLib/full/_compat_pickle.py
+++ b/PythonLib/full/_compat_pickle.py
@@ -22,7 +22,6 @@
'tkMessageBox': 'tkinter.messagebox',
'ScrolledText': 'tkinter.scrolledtext',
'Tkconstants': 'tkinter.constants',
- 'Tix': 'tkinter.tix',
'ttk': 'tkinter.ttk',
'Tkinter': 'tkinter',
'markupbase': '_markupbase',
diff --git a/PythonLib/full/_compression.py b/PythonLib/full/_compression.py
deleted file mode 100644
index e8b70aa0a..000000000
--- a/PythonLib/full/_compression.py
+++ /dev/null
@@ -1,162 +0,0 @@
-"""Internal classes used by the gzip, lzma and bz2 modules"""
-
-import io
-import sys
-
-BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
-
-
-class BaseStream(io.BufferedIOBase):
- """Mode-checking helper functions."""
-
- def _check_not_closed(self):
- if self.closed:
- raise ValueError("I/O operation on closed file")
-
- def _check_can_read(self):
- if not self.readable():
- raise io.UnsupportedOperation("File not open for reading")
-
- def _check_can_write(self):
- if not self.writable():
- raise io.UnsupportedOperation("File not open for writing")
-
- def _check_can_seek(self):
- if not self.readable():
- raise io.UnsupportedOperation("Seeking is only supported "
- "on files open for reading")
- if not self.seekable():
- raise io.UnsupportedOperation("The underlying file object "
- "does not support seeking")
-
-
-class DecompressReader(io.RawIOBase):
- """Adapts the decompressor API to a RawIOBase reader API"""
-
- def readable(self):
- return True
-
- def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
- self._fp = fp
- self._eof = False
- self._pos = 0 # Current offset in decompressed stream
-
- # Set to size of decompressed stream once it is known, for SEEK_END
- self._size = -1
-
- # Save the decompressor factory and arguments.
- # If the file contains multiple compressed streams, each
- # stream will need a separate decompressor object. A new decompressor
- # object is also needed when implementing a backwards seek().
- self._decomp_factory = decomp_factory
- self._decomp_args = decomp_args
- self._decompressor = self._decomp_factory(**self._decomp_args)
-
- # Exception class to catch from decompressor signifying invalid
- # trailing data to ignore
- self._trailing_error = trailing_error
-
- def close(self):
- self._decompressor = None
- return super().close()
-
- def seekable(self):
- return self._fp.seekable()
-
- def readinto(self, b):
- with memoryview(b) as view, view.cast("B") as byte_view:
- data = self.read(len(byte_view))
- byte_view[:len(data)] = data
- return len(data)
-
- def read(self, size=-1):
- if size < 0:
- return self.readall()
-
- if not size or self._eof:
- return b""
- data = None # Default if EOF is encountered
- # Depending on the input data, our call to the decompressor may not
- # return any data. In this case, try again after reading another block.
- while True:
- if self._decompressor.eof:
- rawblock = (self._decompressor.unused_data or
- self._fp.read(BUFFER_SIZE))
- if not rawblock:
- break
- # Continue to next stream.
- self._decompressor = self._decomp_factory(
- **self._decomp_args)
- try:
- data = self._decompressor.decompress(rawblock, size)
- except self._trailing_error:
- # Trailing data isn't a valid compressed stream; ignore it.
- break
- else:
- if self._decompressor.needs_input:
- rawblock = self._fp.read(BUFFER_SIZE)
- if not rawblock:
- raise EOFError("Compressed file ended before the "
- "end-of-stream marker was reached")
- else:
- rawblock = b""
- data = self._decompressor.decompress(rawblock, size)
- if data:
- break
- if not data:
- self._eof = True
- self._size = self._pos
- return b""
- self._pos += len(data)
- return data
-
- def readall(self):
- chunks = []
- # sys.maxsize means the max length of output buffer is unlimited,
- # so that the whole input buffer can be decompressed within one
- # .decompress() call.
- while data := self.read(sys.maxsize):
- chunks.append(data)
-
- return b"".join(chunks)
-
- # Rewind the file to the beginning of the data stream.
- def _rewind(self):
- self._fp.seek(0)
- self._eof = False
- self._pos = 0
- self._decompressor = self._decomp_factory(**self._decomp_args)
-
- def seek(self, offset, whence=io.SEEK_SET):
- # Recalculate offset as an absolute file position.
- if whence == io.SEEK_SET:
- pass
- elif whence == io.SEEK_CUR:
- offset = self._pos + offset
- elif whence == io.SEEK_END:
- # Seeking relative to EOF - we need to know the file's size.
- if self._size < 0:
- while self.read(io.DEFAULT_BUFFER_SIZE):
- pass
- offset = self._size + offset
- else:
- raise ValueError("Invalid value for whence: {}".format(whence))
-
- # Make it so that offset is the number of bytes to skip forward.
- if offset < self._pos:
- self._rewind()
- else:
- offset -= self._pos
-
- # Read and discard data until we reach the desired position.
- while offset > 0:
- data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
- if not data:
- break
- offset -= len(data)
-
- return self._pos
-
- def tell(self):
- """Return the current file position."""
- return self._pos
diff --git a/PythonLib/full/_ios_support.py b/PythonLib/full/_ios_support.py
new file mode 100644
index 000000000..20467a7c2
--- /dev/null
+++ b/PythonLib/full/_ios_support.py
@@ -0,0 +1,71 @@
+import sys
+try:
+ from ctypes import cdll, c_void_p, c_char_p, util
+except ImportError:
+ # ctypes is an optional module. If it's not present, we're limited in what
+ # we can tell about the system, but we don't want to prevent the module
+ # from working.
+ print("ctypes isn't available; iOS system calls will not be available", file=sys.stderr)
+ objc = None
+else:
+ # ctypes is available. Load the ObjC library, and wrap the objc_getClass,
+ # sel_registerName methods
+ lib = util.find_library("objc")
+ if lib is None:
+ # Failed to load the objc library
+ raise ImportError("ObjC runtime library couldn't be loaded")
+
+ objc = cdll.LoadLibrary(lib)
+ objc.objc_getClass.restype = c_void_p
+ objc.objc_getClass.argtypes = [c_char_p]
+ objc.sel_registerName.restype = c_void_p
+ objc.sel_registerName.argtypes = [c_char_p]
+
+
+def get_platform_ios():
+ # Determine if this is a simulator using the multiarch value
+ is_simulator = sys.implementation._multiarch.endswith("simulator")
+
+ # We can't use ctypes; abort
+ if not objc:
+ return None
+
+ # Most of the methods return ObjC objects
+ objc.objc_msgSend.restype = c_void_p
+ # All the methods used have no arguments.
+ objc.objc_msgSend.argtypes = [c_void_p, c_void_p]
+
+ # Equivalent of:
+ # device = [UIDevice currentDevice]
+ UIDevice = objc.objc_getClass(b"UIDevice")
+ SEL_currentDevice = objc.sel_registerName(b"currentDevice")
+ device = objc.objc_msgSend(UIDevice, SEL_currentDevice)
+
+ # Equivalent of:
+ # device_systemVersion = [device systemVersion]
+ SEL_systemVersion = objc.sel_registerName(b"systemVersion")
+ device_systemVersion = objc.objc_msgSend(device, SEL_systemVersion)
+
+ # Equivalent of:
+ # device_systemName = [device systemName]
+ SEL_systemName = objc.sel_registerName(b"systemName")
+ device_systemName = objc.objc_msgSend(device, SEL_systemName)
+
+ # Equivalent of:
+ # device_model = [device model]
+ SEL_model = objc.sel_registerName(b"model")
+ device_model = objc.objc_msgSend(device, SEL_model)
+
+ # UTF8String returns a const char*;
+ SEL_UTF8String = objc.sel_registerName(b"UTF8String")
+ objc.objc_msgSend.restype = c_char_p
+
+ # Equivalent of:
+ # system = [device_systemName UTF8String]
+ # release = [device_systemVersion UTF8String]
+ # model = [device_model UTF8String]
+ system = objc.objc_msgSend(device_systemName, SEL_UTF8String).decode()
+ release = objc.objc_msgSend(device_systemVersion, SEL_UTF8String).decode()
+ model = objc.objc_msgSend(device_model, SEL_UTF8String).decode()
+
+ return system, release, model, is_simulator
diff --git a/PythonLib/full/_markupbase.py b/PythonLib/full/_markupbase.py
index 3ad7e2799..614f0cd16 100644
--- a/PythonLib/full/_markupbase.py
+++ b/PythonLib/full/_markupbase.py
@@ -13,7 +13,7 @@
_markedsectionclose = re.compile(r']\s*]\s*>')
# An analysis of the MS-Word extensions is available at
-# http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
+# http://web.archive.org/web/20060321153828/http://www.planetpublish.com/xmlarena/xap/Thursday/WordtoXML.pdf
_msmarkedsectionclose = re.compile(r']\s*>')
diff --git a/PythonLib/full/_opcode_metadata.py b/PythonLib/full/_opcode_metadata.py
new file mode 100644
index 000000000..b9304ec3c
--- /dev/null
+++ b/PythonLib/full/_opcode_metadata.py
@@ -0,0 +1,371 @@
+# This file is generated by Tools/cases_generator/py_metadata_generator.py
+# from:
+# Python/bytecodes.c
+# Do not edit!
+_specializations = {
+ "RESUME": [
+ "RESUME_CHECK",
+ ],
+ "LOAD_CONST": [
+ "LOAD_CONST_MORTAL",
+ "LOAD_CONST_IMMORTAL",
+ ],
+ "TO_BOOL": [
+ "TO_BOOL_ALWAYS_TRUE",
+ "TO_BOOL_BOOL",
+ "TO_BOOL_INT",
+ "TO_BOOL_LIST",
+ "TO_BOOL_NONE",
+ "TO_BOOL_STR",
+ ],
+ "BINARY_OP": [
+ "BINARY_OP_MULTIPLY_INT",
+ "BINARY_OP_ADD_INT",
+ "BINARY_OP_SUBTRACT_INT",
+ "BINARY_OP_MULTIPLY_FLOAT",
+ "BINARY_OP_ADD_FLOAT",
+ "BINARY_OP_SUBTRACT_FLOAT",
+ "BINARY_OP_ADD_UNICODE",
+ "BINARY_OP_SUBSCR_LIST_INT",
+ "BINARY_OP_SUBSCR_LIST_SLICE",
+ "BINARY_OP_SUBSCR_TUPLE_INT",
+ "BINARY_OP_SUBSCR_STR_INT",
+ "BINARY_OP_SUBSCR_DICT",
+ "BINARY_OP_SUBSCR_GETITEM",
+ "BINARY_OP_EXTEND",
+ "BINARY_OP_INPLACE_ADD_UNICODE",
+ ],
+ "STORE_SUBSCR": [
+ "STORE_SUBSCR_DICT",
+ "STORE_SUBSCR_LIST_INT",
+ ],
+ "SEND": [
+ "SEND_GEN",
+ ],
+ "UNPACK_SEQUENCE": [
+ "UNPACK_SEQUENCE_TWO_TUPLE",
+ "UNPACK_SEQUENCE_TUPLE",
+ "UNPACK_SEQUENCE_LIST",
+ ],
+ "STORE_ATTR": [
+ "STORE_ATTR_INSTANCE_VALUE",
+ "STORE_ATTR_SLOT",
+ "STORE_ATTR_WITH_HINT",
+ ],
+ "LOAD_GLOBAL": [
+ "LOAD_GLOBAL_MODULE",
+ "LOAD_GLOBAL_BUILTIN",
+ ],
+ "LOAD_SUPER_ATTR": [
+ "LOAD_SUPER_ATTR_ATTR",
+ "LOAD_SUPER_ATTR_METHOD",
+ ],
+ "LOAD_ATTR": [
+ "LOAD_ATTR_INSTANCE_VALUE",
+ "LOAD_ATTR_MODULE",
+ "LOAD_ATTR_WITH_HINT",
+ "LOAD_ATTR_SLOT",
+ "LOAD_ATTR_CLASS",
+ "LOAD_ATTR_CLASS_WITH_METACLASS_CHECK",
+ "LOAD_ATTR_PROPERTY",
+ "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN",
+ "LOAD_ATTR_METHOD_WITH_VALUES",
+ "LOAD_ATTR_METHOD_NO_DICT",
+ "LOAD_ATTR_METHOD_LAZY_DICT",
+ "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES",
+ "LOAD_ATTR_NONDESCRIPTOR_NO_DICT",
+ ],
+ "COMPARE_OP": [
+ "COMPARE_OP_FLOAT",
+ "COMPARE_OP_INT",
+ "COMPARE_OP_STR",
+ ],
+ "CONTAINS_OP": [
+ "CONTAINS_OP_SET",
+ "CONTAINS_OP_DICT",
+ ],
+ "JUMP_BACKWARD": [
+ "JUMP_BACKWARD_NO_JIT",
+ "JUMP_BACKWARD_JIT",
+ ],
+ "FOR_ITER": [
+ "FOR_ITER_LIST",
+ "FOR_ITER_TUPLE",
+ "FOR_ITER_RANGE",
+ "FOR_ITER_GEN",
+ ],
+ "CALL": [
+ "CALL_BOUND_METHOD_EXACT_ARGS",
+ "CALL_PY_EXACT_ARGS",
+ "CALL_TYPE_1",
+ "CALL_STR_1",
+ "CALL_TUPLE_1",
+ "CALL_BUILTIN_CLASS",
+ "CALL_BUILTIN_O",
+ "CALL_BUILTIN_FAST",
+ "CALL_BUILTIN_FAST_WITH_KEYWORDS",
+ "CALL_LEN",
+ "CALL_ISINSTANCE",
+ "CALL_LIST_APPEND",
+ "CALL_METHOD_DESCRIPTOR_O",
+ "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS",
+ "CALL_METHOD_DESCRIPTOR_NOARGS",
+ "CALL_METHOD_DESCRIPTOR_FAST",
+ "CALL_ALLOC_AND_ENTER_INIT",
+ "CALL_PY_GENERAL",
+ "CALL_BOUND_METHOD_GENERAL",
+ "CALL_NON_PY_GENERAL",
+ ],
+ "CALL_KW": [
+ "CALL_KW_BOUND_METHOD",
+ "CALL_KW_PY",
+ "CALL_KW_NON_PY",
+ ],
+}
+
+_specialized_opmap = {
+ 'BINARY_OP_ADD_FLOAT': 129,
+ 'BINARY_OP_ADD_INT': 130,
+ 'BINARY_OP_ADD_UNICODE': 131,
+ 'BINARY_OP_EXTEND': 132,
+ 'BINARY_OP_INPLACE_ADD_UNICODE': 3,
+ 'BINARY_OP_MULTIPLY_FLOAT': 133,
+ 'BINARY_OP_MULTIPLY_INT': 134,
+ 'BINARY_OP_SUBSCR_DICT': 135,
+ 'BINARY_OP_SUBSCR_GETITEM': 136,
+ 'BINARY_OP_SUBSCR_LIST_INT': 137,
+ 'BINARY_OP_SUBSCR_LIST_SLICE': 138,
+ 'BINARY_OP_SUBSCR_STR_INT': 139,
+ 'BINARY_OP_SUBSCR_TUPLE_INT': 140,
+ 'BINARY_OP_SUBTRACT_FLOAT': 141,
+ 'BINARY_OP_SUBTRACT_INT': 142,
+ 'CALL_ALLOC_AND_ENTER_INIT': 143,
+ 'CALL_BOUND_METHOD_EXACT_ARGS': 144,
+ 'CALL_BOUND_METHOD_GENERAL': 145,
+ 'CALL_BUILTIN_CLASS': 146,
+ 'CALL_BUILTIN_FAST': 147,
+ 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 148,
+ 'CALL_BUILTIN_O': 149,
+ 'CALL_ISINSTANCE': 150,
+ 'CALL_KW_BOUND_METHOD': 151,
+ 'CALL_KW_NON_PY': 152,
+ 'CALL_KW_PY': 153,
+ 'CALL_LEN': 154,
+ 'CALL_LIST_APPEND': 155,
+ 'CALL_METHOD_DESCRIPTOR_FAST': 156,
+ 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 157,
+ 'CALL_METHOD_DESCRIPTOR_NOARGS': 158,
+ 'CALL_METHOD_DESCRIPTOR_O': 159,
+ 'CALL_NON_PY_GENERAL': 160,
+ 'CALL_PY_EXACT_ARGS': 161,
+ 'CALL_PY_GENERAL': 162,
+ 'CALL_STR_1': 163,
+ 'CALL_TUPLE_1': 164,
+ 'CALL_TYPE_1': 165,
+ 'COMPARE_OP_FLOAT': 166,
+ 'COMPARE_OP_INT': 167,
+ 'COMPARE_OP_STR': 168,
+ 'CONTAINS_OP_DICT': 169,
+ 'CONTAINS_OP_SET': 170,
+ 'FOR_ITER_GEN': 171,
+ 'FOR_ITER_LIST': 172,
+ 'FOR_ITER_RANGE': 173,
+ 'FOR_ITER_TUPLE': 174,
+ 'JUMP_BACKWARD_JIT': 175,
+ 'JUMP_BACKWARD_NO_JIT': 176,
+ 'LOAD_ATTR_CLASS': 177,
+ 'LOAD_ATTR_CLASS_WITH_METACLASS_CHECK': 178,
+ 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 179,
+ 'LOAD_ATTR_INSTANCE_VALUE': 180,
+ 'LOAD_ATTR_METHOD_LAZY_DICT': 181,
+ 'LOAD_ATTR_METHOD_NO_DICT': 182,
+ 'LOAD_ATTR_METHOD_WITH_VALUES': 183,
+ 'LOAD_ATTR_MODULE': 184,
+ 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 185,
+ 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 186,
+ 'LOAD_ATTR_PROPERTY': 187,
+ 'LOAD_ATTR_SLOT': 188,
+ 'LOAD_ATTR_WITH_HINT': 189,
+ 'LOAD_CONST_IMMORTAL': 190,
+ 'LOAD_CONST_MORTAL': 191,
+ 'LOAD_GLOBAL_BUILTIN': 192,
+ 'LOAD_GLOBAL_MODULE': 193,
+ 'LOAD_SUPER_ATTR_ATTR': 194,
+ 'LOAD_SUPER_ATTR_METHOD': 195,
+ 'RESUME_CHECK': 196,
+ 'SEND_GEN': 197,
+ 'STORE_ATTR_INSTANCE_VALUE': 198,
+ 'STORE_ATTR_SLOT': 199,
+ 'STORE_ATTR_WITH_HINT': 200,
+ 'STORE_SUBSCR_DICT': 201,
+ 'STORE_SUBSCR_LIST_INT': 202,
+ 'TO_BOOL_ALWAYS_TRUE': 203,
+ 'TO_BOOL_BOOL': 204,
+ 'TO_BOOL_INT': 205,
+ 'TO_BOOL_LIST': 206,
+ 'TO_BOOL_NONE': 207,
+ 'TO_BOOL_STR': 208,
+ 'UNPACK_SEQUENCE_LIST': 209,
+ 'UNPACK_SEQUENCE_TUPLE': 210,
+ 'UNPACK_SEQUENCE_TWO_TUPLE': 211,
+}
+
+opmap = {
+ 'CACHE': 0,
+ 'RESERVED': 17,
+ 'RESUME': 128,
+ 'INSTRUMENTED_LINE': 254,
+ 'ENTER_EXECUTOR': 255,
+ 'BINARY_SLICE': 1,
+ 'BUILD_TEMPLATE': 2,
+ 'CALL_FUNCTION_EX': 4,
+ 'CHECK_EG_MATCH': 5,
+ 'CHECK_EXC_MATCH': 6,
+ 'CLEANUP_THROW': 7,
+ 'DELETE_SUBSCR': 8,
+ 'END_FOR': 9,
+ 'END_SEND': 10,
+ 'EXIT_INIT_CHECK': 11,
+ 'FORMAT_SIMPLE': 12,
+ 'FORMAT_WITH_SPEC': 13,
+ 'GET_AITER': 14,
+ 'GET_ANEXT': 15,
+ 'GET_ITER': 16,
+ 'GET_LEN': 18,
+ 'GET_YIELD_FROM_ITER': 19,
+ 'INTERPRETER_EXIT': 20,
+ 'LOAD_BUILD_CLASS': 21,
+ 'LOAD_LOCALS': 22,
+ 'MAKE_FUNCTION': 23,
+ 'MATCH_KEYS': 24,
+ 'MATCH_MAPPING': 25,
+ 'MATCH_SEQUENCE': 26,
+ 'NOP': 27,
+ 'NOT_TAKEN': 28,
+ 'POP_EXCEPT': 29,
+ 'POP_ITER': 30,
+ 'POP_TOP': 31,
+ 'PUSH_EXC_INFO': 32,
+ 'PUSH_NULL': 33,
+ 'RETURN_GENERATOR': 34,
+ 'RETURN_VALUE': 35,
+ 'SETUP_ANNOTATIONS': 36,
+ 'STORE_SLICE': 37,
+ 'STORE_SUBSCR': 38,
+ 'TO_BOOL': 39,
+ 'UNARY_INVERT': 40,
+ 'UNARY_NEGATIVE': 41,
+ 'UNARY_NOT': 42,
+ 'WITH_EXCEPT_START': 43,
+ 'BINARY_OP': 44,
+ 'BUILD_INTERPOLATION': 45,
+ 'BUILD_LIST': 46,
+ 'BUILD_MAP': 47,
+ 'BUILD_SET': 48,
+ 'BUILD_SLICE': 49,
+ 'BUILD_STRING': 50,
+ 'BUILD_TUPLE': 51,
+ 'CALL': 52,
+ 'CALL_INTRINSIC_1': 53,
+ 'CALL_INTRINSIC_2': 54,
+ 'CALL_KW': 55,
+ 'COMPARE_OP': 56,
+ 'CONTAINS_OP': 57,
+ 'CONVERT_VALUE': 58,
+ 'COPY': 59,
+ 'COPY_FREE_VARS': 60,
+ 'DELETE_ATTR': 61,
+ 'DELETE_DEREF': 62,
+ 'DELETE_FAST': 63,
+ 'DELETE_GLOBAL': 64,
+ 'DELETE_NAME': 65,
+ 'DICT_MERGE': 66,
+ 'DICT_UPDATE': 67,
+ 'END_ASYNC_FOR': 68,
+ 'EXTENDED_ARG': 69,
+ 'FOR_ITER': 70,
+ 'GET_AWAITABLE': 71,
+ 'IMPORT_FROM': 72,
+ 'IMPORT_NAME': 73,
+ 'IS_OP': 74,
+ 'JUMP_BACKWARD': 75,
+ 'JUMP_BACKWARD_NO_INTERRUPT': 76,
+ 'JUMP_FORWARD': 77,
+ 'LIST_APPEND': 78,
+ 'LIST_EXTEND': 79,
+ 'LOAD_ATTR': 80,
+ 'LOAD_COMMON_CONSTANT': 81,
+ 'LOAD_CONST': 82,
+ 'LOAD_DEREF': 83,
+ 'LOAD_FAST': 84,
+ 'LOAD_FAST_AND_CLEAR': 85,
+ 'LOAD_FAST_BORROW': 86,
+ 'LOAD_FAST_BORROW_LOAD_FAST_BORROW': 87,
+ 'LOAD_FAST_CHECK': 88,
+ 'LOAD_FAST_LOAD_FAST': 89,
+ 'LOAD_FROM_DICT_OR_DEREF': 90,
+ 'LOAD_FROM_DICT_OR_GLOBALS': 91,
+ 'LOAD_GLOBAL': 92,
+ 'LOAD_NAME': 93,
+ 'LOAD_SMALL_INT': 94,
+ 'LOAD_SPECIAL': 95,
+ 'LOAD_SUPER_ATTR': 96,
+ 'MAKE_CELL': 97,
+ 'MAP_ADD': 98,
+ 'MATCH_CLASS': 99,
+ 'POP_JUMP_IF_FALSE': 100,
+ 'POP_JUMP_IF_NONE': 101,
+ 'POP_JUMP_IF_NOT_NONE': 102,
+ 'POP_JUMP_IF_TRUE': 103,
+ 'RAISE_VARARGS': 104,
+ 'RERAISE': 105,
+ 'SEND': 106,
+ 'SET_ADD': 107,
+ 'SET_FUNCTION_ATTRIBUTE': 108,
+ 'SET_UPDATE': 109,
+ 'STORE_ATTR': 110,
+ 'STORE_DEREF': 111,
+ 'STORE_FAST': 112,
+ 'STORE_FAST_LOAD_FAST': 113,
+ 'STORE_FAST_STORE_FAST': 114,
+ 'STORE_GLOBAL': 115,
+ 'STORE_NAME': 116,
+ 'SWAP': 117,
+ 'UNPACK_EX': 118,
+ 'UNPACK_SEQUENCE': 119,
+ 'YIELD_VALUE': 120,
+ 'INSTRUMENTED_END_FOR': 234,
+ 'INSTRUMENTED_POP_ITER': 235,
+ 'INSTRUMENTED_END_SEND': 236,
+ 'INSTRUMENTED_FOR_ITER': 237,
+ 'INSTRUMENTED_INSTRUCTION': 238,
+ 'INSTRUMENTED_JUMP_FORWARD': 239,
+ 'INSTRUMENTED_NOT_TAKEN': 240,
+ 'INSTRUMENTED_POP_JUMP_IF_TRUE': 241,
+ 'INSTRUMENTED_POP_JUMP_IF_FALSE': 242,
+ 'INSTRUMENTED_POP_JUMP_IF_NONE': 243,
+ 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 244,
+ 'INSTRUMENTED_RESUME': 245,
+ 'INSTRUMENTED_RETURN_VALUE': 246,
+ 'INSTRUMENTED_YIELD_VALUE': 247,
+ 'INSTRUMENTED_END_ASYNC_FOR': 248,
+ 'INSTRUMENTED_LOAD_SUPER_ATTR': 249,
+ 'INSTRUMENTED_CALL': 250,
+ 'INSTRUMENTED_CALL_KW': 251,
+ 'INSTRUMENTED_CALL_FUNCTION_EX': 252,
+ 'INSTRUMENTED_JUMP_BACKWARD': 253,
+ 'ANNOTATIONS_PLACEHOLDER': 256,
+ 'JUMP': 257,
+ 'JUMP_IF_FALSE': 258,
+ 'JUMP_IF_TRUE': 259,
+ 'JUMP_NO_INTERRUPT': 260,
+ 'LOAD_CLOSURE': 261,
+ 'POP_BLOCK': 262,
+ 'SETUP_CLEANUP': 263,
+ 'SETUP_FINALLY': 264,
+ 'SETUP_WITH': 265,
+ 'STORE_FAST_MAYBE_NULL': 266,
+}
+
+HAVE_ARGUMENT = 43
+MIN_INSTRUMENTED_OPCODE = 234
diff --git a/PythonLib/full/_py_warnings.py b/PythonLib/full/_py_warnings.py
new file mode 100644
index 000000000..36513ba2e
--- /dev/null
+++ b/PythonLib/full/_py_warnings.py
@@ -0,0 +1,869 @@
+"""Python part of the warnings subsystem."""
+
+import sys
+import _contextvars
+import _thread
+
+
+__all__ = ["warn", "warn_explicit", "showwarning",
+ "formatwarning", "filterwarnings", "simplefilter",
+ "resetwarnings", "catch_warnings", "deprecated"]
+
+
+# Normally '_wm' is sys.modules['warnings'] but for unit tests it can be
+# a different module. User code is allowed to reassign global attributes
+# of the 'warnings' module, commonly 'filters' or 'showwarning'. So we
+# need to lookup these global attributes dynamically on the '_wm' object,
+# rather than binding them earlier. The code in this module consistently uses
+# '_wm.' rather than using the globals of this module. If the
+# '_warnings' C extension is in use, some globals are replaced by functions
+# and variables defined in that extension.
+_wm = None
+
+
+def _set_module(module):
+ global _wm
+ _wm = module
+
+
+# filters contains a sequence of filter 5-tuples
+# The components of the 5-tuple are:
+# - an action: error, ignore, always, all, default, module, or once
+# - a compiled regex that must match the warning message
+# - a class representing the warning category
+# - a compiled regex that must match the module that is being warned
+# - a line number for the line being warning, or 0 to mean any line
+# If either if the compiled regexs are None, match anything.
+filters = []
+
+
+defaultaction = "default"
+onceregistry = {}
+_lock = _thread.RLock()
+_filters_version = 1
+
+
+# If true, catch_warnings() will use a context var to hold the modified
+# filters list. Otherwise, catch_warnings() will operate on the 'filters'
+# global of the warnings module.
+_use_context = sys.flags.context_aware_warnings
+
+
+class _Context:
+ def __init__(self, filters):
+ self._filters = filters
+ self.log = None # if set to a list, logging is enabled
+
+ def copy(self):
+ context = _Context(self._filters[:])
+ if self.log is not None:
+ context.log = self.log
+ return context
+
+ def _record_warning(self, msg):
+ self.log.append(msg)
+
+
+class _GlobalContext(_Context):
+ def __init__(self):
+ self.log = None
+
+ @property
+ def _filters(self):
+ # Since there is quite a lot of code that assigns to
+ # warnings.filters, this needs to return the current value of
+ # the module global.
+ try:
+ return _wm.filters
+ except AttributeError:
+ # 'filters' global was deleted. Do we need to actually handle this case?
+ return []
+
+
+_global_context = _GlobalContext()
+
+
+_warnings_context = _contextvars.ContextVar('warnings_context')
+
+
+def _get_context():
+ if not _use_context:
+ return _global_context
+ try:
+ return _wm._warnings_context.get()
+ except LookupError:
+ return _global_context
+
+
+def _set_context(context):
+ assert _use_context
+ _wm._warnings_context.set(context)
+
+
+def _new_context():
+ assert _use_context
+ old_context = _wm._get_context()
+ new_context = old_context.copy()
+ _wm._set_context(new_context)
+ return old_context, new_context
+
+
+def _get_filters():
+ """Return the current list of filters. This is a non-public API used by
+ module functions and by the unit tests."""
+ return _wm._get_context()._filters
+
+
+def _filters_mutated_lock_held():
+ _wm._filters_version += 1
+
+
+def showwarning(message, category, filename, lineno, file=None, line=None):
+ """Hook to write a warning to a file; replace if you like."""
+ msg = _wm.WarningMessage(message, category, filename, lineno, file, line)
+ _wm._showwarnmsg_impl(msg)
+
+
+def formatwarning(message, category, filename, lineno, line=None):
+ """Function to format a warning the standard way."""
+ msg = _wm.WarningMessage(message, category, filename, lineno, None, line)
+ return _wm._formatwarnmsg_impl(msg)
+
+
+def _showwarnmsg_impl(msg):
+ context = _wm._get_context()
+ if context.log is not None:
+ context._record_warning(msg)
+ return
+ file = msg.file
+ if file is None:
+ file = sys.stderr
+ if file is None:
+ # sys.stderr is None when run with pythonw.exe:
+ # warnings get lost
+ return
+ text = _wm._formatwarnmsg(msg)
+ try:
+ file.write(text)
+ except OSError:
+ # the file (probably stderr) is invalid - this warning gets lost.
+ pass
+
+
+def _formatwarnmsg_impl(msg):
+ category = msg.category.__name__
+ s = f"{msg.filename}:{msg.lineno}: {category}: {msg.message}\n"
+
+ if msg.line is None:
+ try:
+ import linecache
+ line = linecache.getline(msg.filename, msg.lineno)
+ except Exception:
+ # When a warning is logged during Python shutdown, linecache
+ # and the import machinery don't work anymore
+ line = None
+ linecache = None
+ else:
+ line = msg.line
+ if line:
+ line = line.strip()
+ s += " %s\n" % line
+
+ if msg.source is not None:
+ try:
+ import tracemalloc
+ # Logging a warning should not raise a new exception:
+ # catch Exception, not only ImportError and RecursionError.
+ except Exception:
+ # don't suggest to enable tracemalloc if it's not available
+ suggest_tracemalloc = False
+ tb = None
+ else:
+ try:
+ suggest_tracemalloc = not tracemalloc.is_tracing()
+ tb = tracemalloc.get_object_traceback(msg.source)
+ except Exception:
+ # When a warning is logged during Python shutdown, tracemalloc
+ # and the import machinery don't work anymore
+ suggest_tracemalloc = False
+ tb = None
+
+ if tb is not None:
+ s += 'Object allocated at (most recent call last):\n'
+ for frame in tb:
+ s += (' File "%s", lineno %s\n'
+ % (frame.filename, frame.lineno))
+
+ try:
+ if linecache is not None:
+ line = linecache.getline(frame.filename, frame.lineno)
+ else:
+ line = None
+ except Exception:
+ line = None
+ if line:
+ line = line.strip()
+ s += ' %s\n' % line
+ elif suggest_tracemalloc:
+ s += (f'{category}: Enable tracemalloc to get the object '
+ f'allocation traceback\n')
+ return s
+
+
+# Keep a reference to check if the function was replaced
+_showwarning_orig = showwarning
+
+
+def _showwarnmsg(msg):
+ """Hook to write a warning to a file; replace if you like."""
+ try:
+ sw = _wm.showwarning
+ except AttributeError:
+ pass
+ else:
+ if sw is not _showwarning_orig:
+ # warnings.showwarning() was replaced
+ if not callable(sw):
+ raise TypeError("warnings.showwarning() must be set to a "
+ "function or method")
+
+ sw(msg.message, msg.category, msg.filename, msg.lineno,
+ msg.file, msg.line)
+ return
+ _wm._showwarnmsg_impl(msg)
+
+
+# Keep a reference to check if the function was replaced
+_formatwarning_orig = formatwarning
+
+
+def _formatwarnmsg(msg):
+ """Function to format a warning the standard way."""
+ try:
+ fw = _wm.formatwarning
+ except AttributeError:
+ pass
+ else:
+ if fw is not _formatwarning_orig:
+ # warnings.formatwarning() was replaced
+ return fw(msg.message, msg.category,
+ msg.filename, msg.lineno, msg.line)
+ return _wm._formatwarnmsg_impl(msg)
+
+
+def filterwarnings(action, message="", category=Warning, module="", lineno=0,
+ append=False):
+ """Insert an entry into the list of warnings filters (at the front).
+
+ 'action' -- one of "error", "ignore", "always", "all", "default", "module",
+ or "once"
+ 'message' -- a regex that the warning message must match
+ 'category' -- a class that the warning must be a subclass of
+ 'module' -- a regex that the module name must match
+ 'lineno' -- an integer line number, 0 matches all warnings
+ 'append' -- if true, append to the list of filters
+ """
+ if action not in {"error", "ignore", "always", "all", "default", "module", "once"}:
+ raise ValueError(f"invalid action: {action!r}")
+ if not isinstance(message, str):
+ raise TypeError("message must be a string")
+ if not isinstance(category, type) or not issubclass(category, Warning):
+ raise TypeError("category must be a Warning subclass")
+ if not isinstance(module, str):
+ raise TypeError("module must be a string")
+ if not isinstance(lineno, int):
+ raise TypeError("lineno must be an int")
+ if lineno < 0:
+ raise ValueError("lineno must be an int >= 0")
+
+ if message or module:
+ import re
+
+ if message:
+ message = re.compile(message, re.I)
+ else:
+ message = None
+ if module:
+ module = re.compile(module)
+ else:
+ module = None
+
+ _wm._add_filter(action, message, category, module, lineno, append=append)
+
+
+def simplefilter(action, category=Warning, lineno=0, append=False):
+ """Insert a simple entry into the list of warnings filters (at the front).
+
+ A simple filter matches all modules and messages.
+ 'action' -- one of "error", "ignore", "always", "all", "default", "module",
+ or "once"
+ 'category' -- a class that the warning must be a subclass of
+ 'lineno' -- an integer line number, 0 matches all warnings
+ 'append' -- if true, append to the list of filters
+ """
+ if action not in {"error", "ignore", "always", "all", "default", "module", "once"}:
+ raise ValueError(f"invalid action: {action!r}")
+ if not isinstance(lineno, int):
+ raise TypeError("lineno must be an int")
+ if lineno < 0:
+ raise ValueError("lineno must be an int >= 0")
+ _wm._add_filter(action, None, category, None, lineno, append=append)
+
+
+def _filters_mutated():
+ # Even though this function is not part of the public API, it's used by
+ # a fair amount of user code.
+ with _wm._lock:
+ _wm._filters_mutated_lock_held()
+
+
+def _add_filter(*item, append):
+ with _wm._lock:
+ filters = _wm._get_filters()
+ if not append:
+ # Remove possible duplicate filters, so new one will be placed
+ # in correct place. If append=True and duplicate exists, do nothing.
+ try:
+ filters.remove(item)
+ except ValueError:
+ pass
+ filters.insert(0, item)
+ else:
+ if item not in filters:
+ filters.append(item)
+ _wm._filters_mutated_lock_held()
+
+
+def resetwarnings():
+ """Clear the list of warning filters, so that no filters are active."""
+ with _wm._lock:
+ del _wm._get_filters()[:]
+ _wm._filters_mutated_lock_held()
+
+
+class _OptionError(Exception):
+ """Exception used by option processing helpers."""
+ pass
+
+
+# Helper to process -W options passed via sys.warnoptions
+def _processoptions(args):
+ for arg in args:
+ try:
+ _wm._setoption(arg)
+ except _wm._OptionError as msg:
+ print("Invalid -W option ignored:", msg, file=sys.stderr)
+
+
+# Helper for _processoptions()
+def _setoption(arg):
+ parts = arg.split(':')
+ if len(parts) > 5:
+ raise _wm._OptionError("too many fields (max 5): %r" % (arg,))
+ while len(parts) < 5:
+ parts.append('')
+ action, message, category, module, lineno = [s.strip()
+ for s in parts]
+ action = _wm._getaction(action)
+ category = _wm._getcategory(category)
+ if message or module:
+ import re
+ if message:
+ message = re.escape(message)
+ if module:
+ module = re.escape(module) + r'\z'
+ if lineno:
+ try:
+ lineno = int(lineno)
+ if lineno < 0:
+ raise ValueError
+ except (ValueError, OverflowError):
+ raise _wm._OptionError("invalid lineno %r" % (lineno,)) from None
+ else:
+ lineno = 0
+ _wm.filterwarnings(action, message, category, module, lineno)
+
+
+# Helper for _setoption()
+def _getaction(action):
+ if not action:
+ return "default"
+ for a in ('default', 'always', 'all', 'ignore', 'module', 'once', 'error'):
+ if a.startswith(action):
+ return a
+ raise _wm._OptionError("invalid action: %r" % (action,))
+
+
+# Helper for _setoption()
+def _getcategory(category):
+ if not category:
+ return Warning
+ if '.' not in category:
+ import builtins as m
+ klass = category
+ else:
+ module, _, klass = category.rpartition('.')
+ try:
+ m = __import__(module, None, None, [klass])
+ except ImportError:
+ raise _wm._OptionError("invalid module name: %r" % (module,)) from None
+ try:
+ cat = getattr(m, klass)
+ except AttributeError:
+ raise _wm._OptionError("unknown warning category: %r" % (category,)) from None
+ if not issubclass(cat, Warning):
+ raise _wm._OptionError("invalid warning category: %r" % (category,))
+ return cat
+
+
+def _is_internal_filename(filename):
+ return 'importlib' in filename and '_bootstrap' in filename
+
+
+def _is_filename_to_skip(filename, skip_file_prefixes):
+ return any(filename.startswith(prefix) for prefix in skip_file_prefixes)
+
+
+def _is_internal_frame(frame):
+ """Signal whether the frame is an internal CPython implementation detail."""
+ return _is_internal_filename(frame.f_code.co_filename)
+
+
+def _next_external_frame(frame, skip_file_prefixes):
+ """Find the next frame that doesn't involve Python or user internals."""
+ frame = frame.f_back
+ while frame is not None and (
+ _is_internal_filename(filename := frame.f_code.co_filename) or
+ _is_filename_to_skip(filename, skip_file_prefixes)):
+ frame = frame.f_back
+ return frame
+
+
+# Code typically replaced by _warnings
+def warn(message, category=None, stacklevel=1, source=None,
+ *, skip_file_prefixes=()):
+ """Issue a warning, or maybe ignore it or raise an exception."""
+ # Check if message is already a Warning object
+ if isinstance(message, Warning):
+ category = message.__class__
+ # Check category argument
+ if category is None:
+ category = UserWarning
+ if not (isinstance(category, type) and issubclass(category, Warning)):
+ raise TypeError("category must be a Warning subclass, "
+ "not '{:s}'".format(type(category).__name__))
+ if not isinstance(skip_file_prefixes, tuple):
+ # The C version demands a tuple for implementation performance.
+ raise TypeError('skip_file_prefixes must be a tuple of strs.')
+ if skip_file_prefixes:
+ stacklevel = max(2, stacklevel)
+ # Get context information
+ try:
+ if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)):
+ # If frame is too small to care or if the warning originated in
+ # internal code, then do not try to hide any frames.
+ frame = sys._getframe(stacklevel)
+ else:
+ frame = sys._getframe(1)
+ # Look for one frame less since the above line starts us off.
+ for x in range(stacklevel-1):
+ frame = _next_external_frame(frame, skip_file_prefixes)
+ if frame is None:
+ raise ValueError
+ except ValueError:
+ globals = sys.__dict__
+ filename = ""
+ lineno = 0
+ else:
+ globals = frame.f_globals
+ filename = frame.f_code.co_filename
+ lineno = frame.f_lineno
+ if '__name__' in globals:
+ module = globals['__name__']
+ else:
+ module = ""
+ registry = globals.setdefault("__warningregistry__", {})
+ _wm.warn_explicit(
+ message,
+ category,
+ filename,
+ lineno,
+ module,
+ registry,
+ globals,
+ source=source,
+ )
+
+
+def warn_explicit(message, category, filename, lineno,
+ module=None, registry=None, module_globals=None,
+ source=None):
+ lineno = int(lineno)
+ if module is None:
+ module = filename or ""
+ if module[-3:].lower() == ".py":
+ module = module[:-3] # XXX What about leading pathname?
+ if isinstance(message, Warning):
+ text = str(message)
+ category = message.__class__
+ else:
+ text = message
+ message = category(message)
+ key = (text, category, lineno)
+ with _wm._lock:
+ if registry is None:
+ registry = {}
+ if registry.get('version', 0) != _wm._filters_version:
+ registry.clear()
+ registry['version'] = _wm._filters_version
+ # Quick test for common case
+ if registry.get(key):
+ return
+ # Search the filters
+ for item in _wm._get_filters():
+ action, msg, cat, mod, ln = item
+ if ((msg is None or msg.match(text)) and
+ issubclass(category, cat) and
+ (mod is None or mod.match(module)) and
+ (ln == 0 or lineno == ln)):
+ break
+ else:
+ action = _wm.defaultaction
+ # Early exit actions
+ if action == "ignore":
+ return
+
+ if action == "error":
+ raise message
+ # Other actions
+ if action == "once":
+ registry[key] = 1
+ oncekey = (text, category)
+ if _wm.onceregistry.get(oncekey):
+ return
+ _wm.onceregistry[oncekey] = 1
+ elif action in {"always", "all"}:
+ pass
+ elif action == "module":
+ registry[key] = 1
+ altkey = (text, category, 0)
+ if registry.get(altkey):
+ return
+ registry[altkey] = 1
+ elif action == "default":
+ registry[key] = 1
+ else:
+ # Unrecognized actions are errors
+ raise RuntimeError(
+ "Unrecognized action (%r) in warnings.filters:\n %s" %
+ (action, item))
+
+ # Prime the linecache for formatting, in case the
+ # "file" is actually in a zipfile or something.
+ import linecache
+ linecache.getlines(filename, module_globals)
+
+ # Print message and context
+ msg = _wm.WarningMessage(message, category, filename, lineno, source=source)
+ _wm._showwarnmsg(msg)
+
+
+class WarningMessage(object):
+
+ _WARNING_DETAILS = ("message", "category", "filename", "lineno", "file",
+ "line", "source")
+
+ def __init__(self, message, category, filename, lineno, file=None,
+ line=None, source=None):
+ self.message = message
+ self.category = category
+ self.filename = filename
+ self.lineno = lineno
+ self.file = file
+ self.line = line
+ self.source = source
+ self._category_name = category.__name__ if category else None
+
+ def __str__(self):
+ return ("{message : %r, category : %r, filename : %r, lineno : %s, "
+ "line : %r}" % (self.message, self._category_name,
+ self.filename, self.lineno, self.line))
+
+ def __repr__(self):
+ return f'<{type(self).__qualname__} {self}>'
+
+
+class catch_warnings(object):
+
+ """A context manager that copies and restores the warnings filter upon
+ exiting the context.
+
+ The 'record' argument specifies whether warnings should be captured by a
+ custom implementation of warnings.showwarning() and be appended to a list
+ returned by the context manager. Otherwise None is returned by the context
+ manager. The objects appended to the list are arguments whose attributes
+ mirror the arguments to showwarning().
+
+ The 'module' argument is to specify an alternative module to the module
+ named 'warnings' and imported under that name. This argument is only useful
+ when testing the warnings module itself.
+
+ If the 'action' argument is not None, the remaining arguments are passed
+ to warnings.simplefilter() as if it were called immediately on entering the
+ context.
+ """
+
+ def __init__(self, *, record=False, module=None,
+ action=None, category=Warning, lineno=0, append=False):
+ """Specify whether to record warnings and if an alternative module
+ should be used other than sys.modules['warnings'].
+
+ """
+ self._record = record
+ self._module = sys.modules['warnings'] if module is None else module
+ self._entered = False
+ if action is None:
+ self._filter = None
+ else:
+ self._filter = (action, category, lineno, append)
+
+ def __repr__(self):
+ args = []
+ if self._record:
+ args.append("record=True")
+ if self._module is not sys.modules['warnings']:
+ args.append("module=%r" % self._module)
+ name = type(self).__name__
+ return "%s(%s)" % (name, ", ".join(args))
+
+ def __enter__(self):
+ if self._entered:
+ raise RuntimeError("Cannot enter %r twice" % self)
+ self._entered = True
+ with _wm._lock:
+ if _use_context:
+ self._saved_context, context = self._module._new_context()
+ else:
+ context = None
+ self._filters = self._module.filters
+ self._module.filters = self._filters[:]
+ self._showwarnmsg_impl = self._module._showwarnmsg_impl
+ self._showwarning = self._module.showwarning
+ self._module._filters_mutated_lock_held()
+ if self._record:
+ if _use_context:
+ context.log = log = []
+ else:
+ log = []
+ self._module._showwarnmsg_impl = log.append
+ # Reset showwarning() to the default implementation to make sure
+ # that _showwarnmsg() calls _showwarnmsg_impl()
+ self._module.showwarning = self._module._showwarning_orig
+ else:
+ log = None
+ if self._filter is not None:
+ self._module.simplefilter(*self._filter)
+ return log
+
+ def __exit__(self, *exc_info):
+ if not self._entered:
+ raise RuntimeError("Cannot exit %r without entering first" % self)
+ with _wm._lock:
+ if _use_context:
+ self._module._warnings_context.set(self._saved_context)
+ else:
+ self._module.filters = self._filters
+ self._module._showwarnmsg_impl = self._showwarnmsg_impl
+ self._module.showwarning = self._showwarning
+ self._module._filters_mutated_lock_held()
+
+
+class deprecated:
+ """Indicate that a class, function or overload is deprecated.
+
+ When this decorator is applied to an object, the type checker
+ will generate a diagnostic on usage of the deprecated object.
+
+ Usage:
+
+ @deprecated("Use B instead")
+ class A:
+ pass
+
+ @deprecated("Use g instead")
+ def f():
+ pass
+
+ @overload
+ @deprecated("int support is deprecated")
+ def g(x: int) -> int: ...
+ @overload
+ def g(x: str) -> int: ...
+
+ The warning specified by *category* will be emitted at runtime
+ on use of deprecated objects. For functions, that happens on calls;
+ for classes, on instantiation and on creation of subclasses.
+ If the *category* is ``None``, no warning is emitted at runtime.
+ The *stacklevel* determines where the
+ warning is emitted. If it is ``1`` (the default), the warning
+ is emitted at the direct caller of the deprecated object; if it
+ is higher, it is emitted further up the stack.
+ Static type checker behavior is not affected by the *category*
+ and *stacklevel* arguments.
+
+ The deprecation message passed to the decorator is saved in the
+ ``__deprecated__`` attribute on the decorated object.
+ If applied to an overload, the decorator
+ must be after the ``@overload`` decorator for the attribute to
+ exist on the overload as returned by ``get_overloads()``.
+
+ See PEP 702 for details.
+
+ """
+ def __init__(
+ self,
+ message: str,
+ /,
+ *,
+ category: type[Warning] | None = DeprecationWarning,
+ stacklevel: int = 1,
+ ) -> None:
+ if not isinstance(message, str):
+ raise TypeError(
+ f"Expected an object of type str for 'message', not {type(message).__name__!r}"
+ )
+ self.message = message
+ self.category = category
+ self.stacklevel = stacklevel
+
+ def __call__(self, arg, /):
+ # Make sure the inner functions created below don't
+ # retain a reference to self.
+ msg = self.message
+ category = self.category
+ stacklevel = self.stacklevel
+ if category is None:
+ arg.__deprecated__ = msg
+ return arg
+ elif isinstance(arg, type):
+ import functools
+ from types import MethodType
+
+ original_new = arg.__new__
+
+ @functools.wraps(original_new)
+ def __new__(cls, /, *args, **kwargs):
+ if cls is arg:
+ _wm.warn(msg, category=category, stacklevel=stacklevel + 1)
+ if original_new is not object.__new__:
+ return original_new(cls, *args, **kwargs)
+ # Mirrors a similar check in object.__new__.
+ elif cls.__init__ is object.__init__ and (args or kwargs):
+ raise TypeError(f"{cls.__name__}() takes no arguments")
+ else:
+ return original_new(cls)
+
+ arg.__new__ = staticmethod(__new__)
+
+ if "__init_subclass__" in arg.__dict__:
+ # __init_subclass__ is directly present on the decorated class.
+ # Synthesize a wrapper that calls this method directly.
+ original_init_subclass = arg.__init_subclass__
+ # We need slightly different behavior if __init_subclass__
+ # is a bound method (likely if it was implemented in Python).
+ # Otherwise, it likely means it's a builtin such as
+ # object's implementation of __init_subclass__.
+ if isinstance(original_init_subclass, MethodType):
+ original_init_subclass = original_init_subclass.__func__
+
+ @functools.wraps(original_init_subclass)
+ def __init_subclass__(*args, **kwargs):
+ _wm.warn(msg, category=category, stacklevel=stacklevel + 1)
+ return original_init_subclass(*args, **kwargs)
+ else:
+ def __init_subclass__(cls, *args, **kwargs):
+ _wm.warn(msg, category=category, stacklevel=stacklevel + 1)
+ return super(arg, cls).__init_subclass__(*args, **kwargs)
+
+ arg.__init_subclass__ = classmethod(__init_subclass__)
+
+ arg.__deprecated__ = __new__.__deprecated__ = msg
+ __init_subclass__.__deprecated__ = msg
+ return arg
+ elif callable(arg):
+ import functools
+ import inspect
+
+ @functools.wraps(arg)
+ def wrapper(*args, **kwargs):
+ _wm.warn(msg, category=category, stacklevel=stacklevel + 1)
+ return arg(*args, **kwargs)
+
+ if inspect.iscoroutinefunction(arg):
+ wrapper = inspect.markcoroutinefunction(wrapper)
+
+ arg.__deprecated__ = wrapper.__deprecated__ = msg
+ return wrapper
+ else:
+ raise TypeError(
+ "@deprecated decorator with non-None category must be applied to "
+ f"a class or callable, not {arg!r}"
+ )
+
+
+_DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}"
+
+
+def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info):
+ """Warn that *name* is deprecated or should be removed.
+
+ RuntimeError is raised if *remove* specifies a major/minor tuple older than
+ the current Python version or the same version but past the alpha.
+
+ The *message* argument is formatted with *name* and *remove* as a Python
+ version tuple (e.g. (3, 11)).
+
+ """
+ remove_formatted = f"{remove[0]}.{remove[1]}"
+ if (_version[:2] > remove) or (_version[:2] == remove and _version[3] != "alpha"):
+ msg = f"{name!r} was slated for removal after Python {remove_formatted} alpha"
+ raise RuntimeError(msg)
+ else:
+ msg = message.format(name=name, remove=remove_formatted)
+ _wm.warn(msg, DeprecationWarning, stacklevel=3)
+
+
+# Private utility function called by _PyErr_WarnUnawaitedCoroutine
+def _warn_unawaited_coroutine(coro):
+ msg_lines = [
+ f"coroutine '{coro.__qualname__}' was never awaited\n"
+ ]
+ if coro.cr_origin is not None:
+ import linecache, traceback
+ def extract():
+ for filename, lineno, funcname in reversed(coro.cr_origin):
+ line = linecache.getline(filename, lineno)
+ yield (filename, lineno, funcname, line)
+ msg_lines.append("Coroutine created at (most recent call last)\n")
+ msg_lines += traceback.format_list(list(extract()))
+ msg = "".join(msg_lines).rstrip("\n")
+ # Passing source= here means that if the user happens to have tracemalloc
+ # enabled and tracking where the coroutine was created, the warning will
+ # contain that traceback. This does mean that if they have *both*
+ # coroutine origin tracking *and* tracemalloc enabled, they'll get two
+ # partially-redundant tracebacks. If we wanted to be clever we could
+ # probably detect this case and avoid it, but for now we don't bother.
+ _wm.warn(
+ msg, category=RuntimeWarning, stacklevel=2, source=coro
+ )
+
+
+def _setup_defaults():
+ # Several warning categories are ignored by default in regular builds
+ if hasattr(sys, 'gettotalrefcount'):
+ return
+ _wm.filterwarnings("default", category=DeprecationWarning, module="__main__", append=1)
+ _wm.simplefilter("ignore", category=DeprecationWarning, append=1)
+ _wm.simplefilter("ignore", category=PendingDeprecationWarning, append=1)
+ _wm.simplefilter("ignore", category=ImportWarning, append=1)
+ _wm.simplefilter("ignore", category=ResourceWarning, append=1)
diff --git a/PythonLib/full/_pydatetime.py b/PythonLib/full/_pydatetime.py
index ad6292e1e..70251dbb6 100644
--- a/PythonLib/full/_pydatetime.py
+++ b/PythonLib/full/_pydatetime.py
@@ -1,12 +1,10 @@
-"""Concrete date/time and related types.
-
-See http://www.iana.org/time-zones/repository/tz-link.html for
-time zone and DST data sources.
-"""
+"""Pure Python implementation of the datetime module."""
__all__ = ("date", "datetime", "time", "timedelta", "timezone", "tzinfo",
"MINYEAR", "MAXYEAR", "UTC")
+__name__ = "datetime"
+
import time as _time
import math as _math
@@ -18,10 +16,10 @@ def _cmp(x, y):
def _get_class_module(self):
module_name = self.__class__.__module__
- if module_name == '_pydatetime':
- return 'datetime'
+ if module_name == 'datetime':
+ return 'datetime.'
else:
- return module_name
+ return ''
MINYEAR = 1
MAXYEAR = 9999
@@ -64,14 +62,14 @@ def _days_in_month(year, month):
def _days_before_month(year, month):
"year, month -> number of days in year preceding first day of month."
- assert 1 <= month <= 12, 'month must be in 1..12'
+ assert 1 <= month <= 12, f"month must be in 1..12, not {month}"
return _DAYS_BEFORE_MONTH[month] + (month > 2 and _is_leap(year))
def _ymd2ord(year, month, day):
"year, month, day -> ordinal, considering 01-Jan-0001 as day 1."
- assert 1 <= month <= 12, 'month must be in 1..12'
+ assert 1 <= month <= 12, f"month must be in 1..12, not {month}"
dim = _days_in_month(year, month)
- assert 1 <= day <= dim, ('day must be in 1..%d' % dim)
+ assert 1 <= day <= dim, f"day must be in 1..{dim}, not {day}"
return (_days_before_year(year) +
_days_before_month(year, month) +
day)
@@ -204,6 +202,17 @@ def _format_offset(off, sep=':'):
s += '.%06d' % ss.microseconds
return s
+_normalize_century = None
+def _need_normalize_century():
+ global _normalize_century
+ if _normalize_century is None:
+ try:
+ _normalize_century = (
+ _time.strftime("%Y", (99, 1, 1, 0, 0, 0, 0, 1, 0)) != "0099")
+ except ValueError:
+ _normalize_century = True
+ return _normalize_century
+
# Correctly substitute for %z and %Z escapes in strftime formats.
def _wrap_strftime(object, format, timetuple):
# Don't call utcoffset() or tzname() unless actually needed.
@@ -261,6 +270,20 @@ def _wrap_strftime(object, format, timetuple):
# strftime is going to have at this: escape %
Zreplace = s.replace('%', '%%')
newformat.append(Zreplace)
+ # Note that datetime(1000, 1, 1).strftime('%G') == '1000' so
+ # year 1000 for %G can go on the fast path.
+ elif ((ch in 'YG' or ch in 'FC') and
+ object.year < 1000 and _need_normalize_century()):
+ if ch == 'G':
+ year = int(_time.strftime("%G", timetuple))
+ else:
+ year = object.year
+ if ch == 'C':
+ push('{:02}'.format(year // 100))
+ else:
+ push('{:04}'.format(year))
+ if ch == 'F':
+ push('-{:02}-{:02}'.format(*timetuple[1:3]))
else:
push('%')
push(ch)
@@ -399,9 +422,11 @@ def _parse_hh_mm_ss_ff(tstr):
if pos < len_str:
if tstr[pos] not in '.,':
- raise ValueError("Invalid microsecond component")
+ raise ValueError("Invalid microsecond separator")
else:
pos += 1
+ if not all(map(_is_ascii_digit, tstr[pos:])):
+ raise ValueError("Non-digit values in fraction")
len_remainder = len_str - pos
@@ -413,9 +438,6 @@ def _parse_hh_mm_ss_ff(tstr):
time_comps[3] = int(tstr[pos:(pos+to_parse)])
if to_parse < 6:
time_comps[3] *= _FRACTION_CORRECTION[to_parse-1]
- if (len_remainder > to_parse
- and not all(map(_is_ascii_digit, tstr[(pos+to_parse):]))):
- raise ValueError("Non-digit values in unparsed fraction")
return time_comps
@@ -431,6 +453,17 @@ def _parse_isoformat_time(tstr):
time_comps = _parse_hh_mm_ss_ff(timestr)
+ hour, minute, second, microsecond = time_comps
+ became_next_day = False
+ error_from_components = False
+ if (hour == 24):
+ if all(time_comp == 0 for time_comp in time_comps[1:]):
+ hour = 0
+ time_comps[0] = hour
+ became_next_day = True
+ else:
+ error_from_components = True
+
tzi = None
if tz_pos == len_str and tstr[-1] == 'Z':
tzi = timezone.utc
@@ -446,7 +479,7 @@ def _parse_isoformat_time(tstr):
# HH:MM:SS len: 8
# HH:MM:SS.f+ len: 10+
- if len(tzstr) in (0, 1, 3):
+ if len(tzstr) in (0, 1, 3) or tstr[tz_pos-1] == 'Z':
raise ValueError("Malformed time zone string")
tz_comps = _parse_hh_mm_ss_ff(tzstr)
@@ -463,13 +496,13 @@ def _parse_isoformat_time(tstr):
time_comps.append(tzi)
- return time_comps
+ return time_comps, became_next_day, error_from_components
# tuple[int, int, int] -> tuple[int, int, int] version of date.fromisocalendar
def _isoweek_to_gregorian(year, week, day):
# Year is bounded this way because 9999-12-31 is (9999, 52, 5)
if not MINYEAR <= year <= MAXYEAR:
- raise ValueError(f"Year is out of range: {year}")
+ raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}")
if not 0 < week < 53:
out_of_range = True
@@ -502,7 +535,7 @@ def _isoweek_to_gregorian(year, week, day):
def _check_tzname(name):
if name is not None and not isinstance(name, str):
raise TypeError("tzinfo.tzname() must return None or string, "
- "not '%s'" % type(name))
+ f"not {type(name).__name__!r}")
# name is the offset-producing method, "utcoffset" or "dst".
# offset is what it returned.
@@ -515,24 +548,24 @@ def _check_utc_offset(name, offset):
if offset is None:
return
if not isinstance(offset, timedelta):
- raise TypeError("tzinfo.%s() must return None "
- "or timedelta, not '%s'" % (name, type(offset)))
+ raise TypeError(f"tzinfo.{name}() must return None "
+ f"or timedelta, not {type(offset).__name__!r}")
if not -timedelta(1) < offset < timedelta(1):
- raise ValueError("%s()=%s, must be strictly between "
- "-timedelta(hours=24) and timedelta(hours=24)" %
- (name, offset))
+ raise ValueError("offset must be a timedelta "
+ "strictly between -timedelta(hours=24) and "
+ f"timedelta(hours=24), not {offset!r}")
def _check_date_fields(year, month, day):
year = _index(year)
month = _index(month)
day = _index(day)
if not MINYEAR <= year <= MAXYEAR:
- raise ValueError('year must be in %d..%d' % (MINYEAR, MAXYEAR), year)
+ raise ValueError(f"year must be in {MINYEAR}..{MAXYEAR}, not {year}")
if not 1 <= month <= 12:
- raise ValueError('month must be in 1..12', month)
+ raise ValueError(f"month must be in 1..12, not {month}")
dim = _days_in_month(year, month)
if not 1 <= day <= dim:
- raise ValueError('day must be in 1..%d' % dim, day)
+ raise ValueError(f"day {day} must be in range 1..{dim} for month {month} in year {year}")
return year, month, day
def _check_time_fields(hour, minute, second, microsecond, fold):
@@ -541,24 +574,23 @@ def _check_time_fields(hour, minute, second, microsecond, fold):
second = _index(second)
microsecond = _index(microsecond)
if not 0 <= hour <= 23:
- raise ValueError('hour must be in 0..23', hour)
+ raise ValueError(f"hour must be in 0..23, not {hour}")
if not 0 <= minute <= 59:
- raise ValueError('minute must be in 0..59', minute)
+ raise ValueError(f"minute must be in 0..59, not {minute}")
if not 0 <= second <= 59:
- raise ValueError('second must be in 0..59', second)
+ raise ValueError(f"second must be in 0..59, not {second}")
if not 0 <= microsecond <= 999999:
- raise ValueError('microsecond must be in 0..999999', microsecond)
+ raise ValueError(f"microsecond must be in 0..999999, not {microsecond}")
if fold not in (0, 1):
- raise ValueError('fold must be either 0 or 1', fold)
+ raise ValueError(f"fold must be either 0 or 1, not {fold}")
return hour, minute, second, microsecond, fold
def _check_tzinfo_arg(tz):
if tz is not None and not isinstance(tz, tzinfo):
- raise TypeError("tzinfo argument must be None or of a tzinfo subclass")
-
-def _cmperror(x, y):
- raise TypeError("can't compare '%s' to '%s'" % (
- type(x).__name__, type(y).__name__))
+ raise TypeError(
+ "tzinfo argument must be None or of a tzinfo subclass, "
+ f"not {type(tz).__name__!r}"
+ )
def _divide_and_round(a, b):
"""divide a by b and round result to the nearest integer
@@ -612,7 +644,19 @@ def __new__(cls, days=0, seconds=0, microseconds=0,
# guide the C implementation; it's way more convoluted than speed-
# ignoring auto-overflow-to-long idiomatic Python could be.
- # XXX Check that all inputs are ints or floats.
+ for name, value in (
+ ("days", days),
+ ("seconds", seconds),
+ ("microseconds", microseconds),
+ ("milliseconds", milliseconds),
+ ("minutes", minutes),
+ ("hours", hours),
+ ("weeks", weeks)
+ ):
+ if not isinstance(value, (int, float)):
+ raise TypeError(
+ f"unsupported type for timedelta {name} component: {type(value).__name__}"
+ )
# Final values, all integer.
# s and us fit in 32-bit signed ints; d isn't bounded.
@@ -713,9 +757,9 @@ def __repr__(self):
args.append("microseconds=%d" % self._microseconds)
if not args:
args.append('0')
- return "%s.%s(%s)" % (_get_class_module(self),
- self.__class__.__qualname__,
- ', '.join(args))
+ return "%s%s(%s)" % (_get_class_module(self),
+ self.__class__.__qualname__,
+ ', '.join(args))
def __str__(self):
mm, ss = divmod(self._seconds, 60)
@@ -912,6 +956,7 @@ class date:
fromtimestamp()
today()
fromordinal()
+ strptime()
Operators:
@@ -994,8 +1039,12 @@ def fromordinal(cls, n):
@classmethod
def fromisoformat(cls, date_string):
"""Construct a date from a string in ISO 8601 format."""
+
if not isinstance(date_string, str):
- raise TypeError('fromisoformat: argument must be str')
+ raise TypeError('Argument must be a str')
+
+ if not date_string.isascii():
+ raise ValueError('Argument must be an ASCII str')
if len(date_string) not in (7, 8, 10):
raise ValueError(f'Invalid isoformat string: {date_string!r}')
@@ -1012,6 +1061,12 @@ def fromisocalendar(cls, year, week, day):
This is the inverse of the date.isocalendar() function"""
return cls(*_isoweek_to_gregorian(year, week, day))
+ @classmethod
+ def strptime(cls, date_string, format):
+ """Parse a date string according to the given format (like time.strptime())."""
+ import _strptime
+ return _strptime._strptime_datetime_date(cls, date_string, format)
+
# Conversions to string
def __repr__(self):
@@ -1021,11 +1076,11 @@ def __repr__(self):
>>> repr(d)
'datetime.date(2010, 1, 1)'
"""
- return "%s.%s(%d, %d, %d)" % (_get_class_module(self),
- self.__class__.__qualname__,
- self._year,
- self._month,
- self._day)
+ return "%s%s(%d, %d, %d)" % (_get_class_module(self),
+ self.__class__.__qualname__,
+ self._year,
+ self._month,
+ self._day)
# XXX These shouldn't depend on time.localtime(), because that
# clips the usable dates to [1970 .. 2038). At least ctime() is
# easily done without using strftime() -- that's better too because
@@ -1061,8 +1116,8 @@ def isoformat(self):
This is 'YYYY-MM-DD'.
References:
- - http://www.w3.org/TR/NOTE-datetime
- - http://www.cl.cam.ac.uk/~mgk25/iso-time.html
+ - https://www.w3.org/TR/NOTE-datetime
+ - https://www.cl.cam.ac.uk/~mgk25/iso-time.html
"""
return "%04d-%02d-%02d" % (self._year, self._month, self._day)
@@ -1110,35 +1165,38 @@ def replace(self, year=None, month=None, day=None):
day = self._day
return type(self)(year, month, day)
+ __replace__ = replace
+
# Comparisons of date objects with other.
def __eq__(self, other):
- if isinstance(other, date):
+ if isinstance(other, date) and not isinstance(other, datetime):
return self._cmp(other) == 0
return NotImplemented
def __le__(self, other):
- if isinstance(other, date):
+ if isinstance(other, date) and not isinstance(other, datetime):
return self._cmp(other) <= 0
return NotImplemented
def __lt__(self, other):
- if isinstance(other, date):
+ if isinstance(other, date) and not isinstance(other, datetime):
return self._cmp(other) < 0
return NotImplemented
def __ge__(self, other):
- if isinstance(other, date):
+ if isinstance(other, date) and not isinstance(other, datetime):
return self._cmp(other) >= 0
return NotImplemented
def __gt__(self, other):
- if isinstance(other, date):
+ if isinstance(other, date) and not isinstance(other, datetime):
return self._cmp(other) > 0
return NotImplemented
def _cmp(self, other):
assert isinstance(other, date)
+ assert not isinstance(other, datetime)
y, m, d = self._year, self._month, self._day
y2, m2, d2 = other._year, other._month, other._day
return _cmp((y, m, d), (y2, m2, d2))
@@ -1193,7 +1251,7 @@ def isocalendar(self):
The first week is 1; Monday is 1 ... Sunday is 7.
ISO calendar algorithm taken from
- http://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
+ https://www.phys.uu.nl/~vgent/calendar/isocalendar.htm
(used with permission)
"""
year = self._year
@@ -1329,6 +1387,7 @@ class time:
Constructors:
__new__()
+ strptime()
Operators:
@@ -1387,6 +1446,12 @@ def __new__(cls, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold
self._fold = fold
return self
+ @classmethod
+ def strptime(cls, date_string, format):
+ """string, format -> new time parsed from a string (like time.strptime())."""
+ import _strptime
+ return _strptime._strptime_datetime_time(cls, date_string, format)
+
# Read-only field accessors
@property
def hour(self):
@@ -1515,7 +1580,7 @@ def __repr__(self):
s = ", %d" % self._second
else:
s = ""
- s= "%s.%s(%d, %d%s)" % (_get_class_module(self),
+ s = "%s%s(%d, %d%s)" % (_get_class_module(self),
self.__class__.__qualname__,
self._hour, self._minute, s)
if self._tzinfo is not None:
@@ -1557,7 +1622,7 @@ def fromisoformat(cls, time_string):
time_string = time_string.removeprefix('T')
try:
- return cls(*_parse_isoformat_time(time_string))
+ return cls(*_parse_isoformat_time(time_string)[0])
except Exception:
raise ValueError(f'Invalid isoformat string: {time_string!r}')
@@ -1635,6 +1700,8 @@ def replace(self, hour=None, minute=None, second=None, microsecond=None,
fold = self._fold
return type(self)(hour, minute, second, microsecond, tzinfo, fold=fold)
+ __replace__ = replace
+
# Pickle support.
def _getstate(self, protocol=3):
@@ -1682,7 +1749,7 @@ class datetime(date):
The year, month and day arguments are required. tzinfo may be None, or an
instance of a tzinfo subclass. The remaining arguments may be ints.
"""
- __slots__ = date.__slots__ + time.__slots__
+ __slots__ = time.__slots__
def __new__(cls, year, month=None, day=None, hour=0, minute=0, second=0,
microsecond=0, tzinfo=None, *, fold=0):
@@ -1869,10 +1936,27 @@ def fromisoformat(cls, date_string):
if tstr:
try:
- time_components = _parse_isoformat_time(tstr)
+ time_components, became_next_day, error_from_components = _parse_isoformat_time(tstr)
except ValueError:
raise ValueError(
f'Invalid isoformat string: {date_string!r}') from None
+ else:
+ if error_from_components:
+ raise ValueError("minute, second, and microsecond must be 0 when hour is 24")
+
+ if became_next_day:
+ year, month, day = date_components
+ # Only wrap day/month when it was previously valid
+ if month <= 12 and day <= (days_in_month := _days_in_month(year, month)):
+ # Calculate midnight of the next day
+ day += 1
+ if day > days_in_month:
+ day = 1
+ month += 1
+ if month > 12:
+ month = 1
+ year += 1
+ date_components = [year, month, day]
else:
time_components = [0, 0, 0, 0, None]
@@ -1981,6 +2065,8 @@ def replace(self, year=None, month=None, day=None, hour=None,
return type(self)(year, month, day, hour, minute, second,
microsecond, tzinfo, fold=fold)
+ __replace__ = replace
+
def _local_timezone(self):
if self.tzinfo is None:
ts = self._mktime()
@@ -2042,7 +2128,7 @@ def isoformat(self, sep='T', timespec='auto'):
By default, the fractional part is omitted if self.microsecond == 0.
If self.tzinfo is not None, the UTC offset is also attached, giving
- giving a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'.
+ a full format of 'YYYY-MM-DD HH:MM:SS.mmmmmm+HH:MM'.
Optional argument sep specifies the separator between date and
time, default 'T'.
@@ -2070,9 +2156,9 @@ def __repr__(self):
del L[-1]
if L[-1] == 0:
del L[-1]
- s = "%s.%s(%s)" % (_get_class_module(self),
- self.__class__.__qualname__,
- ", ".join(map(str, L)))
+ s = "%s%s(%s)" % (_get_class_module(self),
+ self.__class__.__qualname__,
+ ", ".join(map(str, L)))
if self._tzinfo is not None:
assert s[-1:] == ")"
s = s[:-1] + ", tzinfo=%r" % self._tzinfo + ")"
@@ -2089,7 +2175,7 @@ def __str__(self):
def strptime(cls, date_string, format):
'string, format -> new datetime parsed from a string (like time.strptime()).'
import _strptime
- return _strptime._strptime_datetime(cls, date_string, format)
+ return _strptime._strptime_datetime_datetime(cls, date_string, format)
def utcoffset(self):
"""Return the timezone offset as timedelta positive east of UTC (negative west of
@@ -2133,42 +2219,32 @@ def dst(self):
def __eq__(self, other):
if isinstance(other, datetime):
return self._cmp(other, allow_mixed=True) == 0
- elif not isinstance(other, date):
- return NotImplemented
else:
- return False
+ return NotImplemented
def __le__(self, other):
if isinstance(other, datetime):
return self._cmp(other) <= 0
- elif not isinstance(other, date):
- return NotImplemented
else:
- _cmperror(self, other)
+ return NotImplemented
def __lt__(self, other):
if isinstance(other, datetime):
return self._cmp(other) < 0
- elif not isinstance(other, date):
- return NotImplemented
else:
- _cmperror(self, other)
+ return NotImplemented
def __ge__(self, other):
if isinstance(other, datetime):
return self._cmp(other) >= 0
- elif not isinstance(other, date):
- return NotImplemented
else:
- _cmperror(self, other)
+ return NotImplemented
def __gt__(self, other):
if isinstance(other, datetime):
return self._cmp(other) > 0
- elif not isinstance(other, date):
- return NotImplemented
else:
- _cmperror(self, other)
+ return NotImplemented
def _cmp(self, other, allow_mixed=False):
assert isinstance(other, datetime)
@@ -2313,7 +2389,6 @@ def __reduce__(self):
def _isoweek1monday(year):
# Helper to calculate the day number of the Monday starting week 1
- # XXX This could be done more efficiently
THURSDAY = 3
firstday = _ymd2ord(year, 1, 1)
firstweekday = (firstday + 6) % 7 # See weekday() above
@@ -2340,9 +2415,12 @@ def __new__(cls, offset, name=_Omitted):
if not cls._minoffset <= offset <= cls._maxoffset:
raise ValueError("offset must be a timedelta "
"strictly between -timedelta(hours=24) and "
- "timedelta(hours=24).")
+ f"timedelta(hours=24), not {offset!r}")
return cls._create(offset, name)
+ def __init_subclass__(cls):
+ raise TypeError("type 'datetime.timezone' is not an acceptable base type")
+
@classmethod
def _create(cls, offset, name=None):
self = tzinfo.__new__(cls)
@@ -2377,12 +2455,12 @@ def __repr__(self):
if self is self.utc:
return 'datetime.timezone.utc'
if self._name is None:
- return "%s.%s(%r)" % (_get_class_module(self),
- self.__class__.__qualname__,
- self._offset)
- return "%s.%s(%r, %r)" % (_get_class_module(self),
- self.__class__.__qualname__,
- self._offset, self._name)
+ return "%s%s(%r)" % (_get_class_module(self),
+ self.__class__.__qualname__,
+ self._offset)
+ return "%s%s(%r, %r)" % (_get_class_module(self),
+ self.__class__.__qualname__,
+ self._offset, self._name)
def __str__(self):
return self.tzname(None)
diff --git a/PythonLib/full/_pydecimal.py b/PythonLib/full/_pydecimal.py
index 75df3db26..97a629fe9 100644
--- a/PythonLib/full/_pydecimal.py
+++ b/PythonLib/full/_pydecimal.py
@@ -38,10 +38,10 @@
'ROUND_FLOOR', 'ROUND_UP', 'ROUND_HALF_DOWN', 'ROUND_05UP',
# Functions for manipulating contexts
- 'setcontext', 'getcontext', 'localcontext',
+ 'setcontext', 'getcontext', 'localcontext', 'IEEEContext',
# Limits for the C version for compatibility
- 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY',
+ 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', 'IEEE_CONTEXT_MAX_BITS',
# C version: compile time choice that enables the thread local context (deprecated, now always true)
'HAVE_THREADS',
@@ -83,10 +83,12 @@
MAX_PREC = 999999999999999999
MAX_EMAX = 999999999999999999
MIN_EMIN = -999999999999999999
+ IEEE_CONTEXT_MAX_BITS = 512
else:
MAX_PREC = 425000000
MAX_EMAX = 425000000
MIN_EMIN = -425000000
+ IEEE_CONTEXT_MAX_BITS = 256
MIN_ETINY = MIN_EMIN - (MAX_PREC-1)
@@ -97,7 +99,7 @@ class DecimalException(ArithmeticError):
Used exceptions derive from this.
If an exception derives from another exception besides this (such as
- Underflow (Inexact, Rounded, Subnormal) that indicates that it is only
+ Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only
called if the others are present. This isn't actually used for
anything, though.
@@ -145,7 +147,7 @@ class InvalidOperation(DecimalException):
x ** (+-)INF
An operand is invalid
- The result of the operation after these is a quiet positive NaN,
+ The result of the operation after this is a quiet positive NaN,
except when the cause is a signaling NaN, in which case the result is
also a quiet NaN, but with the original sign, and an optional
diagnostic information.
@@ -417,6 +419,27 @@ def sin(x):
return ctx_manager
+def IEEEContext(bits, /):
+ """
+ Return a context object initialized to the proper values for one of the
+ IEEE interchange formats. The argument must be a multiple of 32 and less
+ than IEEE_CONTEXT_MAX_BITS.
+ """
+ if bits <= 0 or bits > IEEE_CONTEXT_MAX_BITS or bits % 32:
+ raise ValueError("argument must be a multiple of 32, "
+ f"with a maximum of {IEEE_CONTEXT_MAX_BITS}")
+
+ ctx = Context()
+ ctx.prec = 9 * (bits//32) - 2
+ ctx.Emax = 3 * (1 << (bits//16 + 3))
+ ctx.Emin = 1 - ctx.Emax
+ ctx.rounding = ROUND_HALF_EVEN
+ ctx.clamp = 1
+ ctx.traps = dict.fromkeys(_signals, False)
+
+ return ctx
+
+
##### Decimal class #######################################################
# Do not subclass Decimal from numbers.Real and do not register it as such
@@ -582,6 +605,21 @@ def __new__(cls, value="0", context=None):
raise TypeError("Cannot convert %r to Decimal" % value)
+ @classmethod
+ def from_number(cls, number):
+ """Converts a real number to a decimal number, exactly.
+
+ >>> Decimal.from_number(314) # int
+ Decimal('314')
+ >>> Decimal.from_number(0.1) # float
+ Decimal('0.1000000000000000055511151231257827021181583404541015625')
+ >>> Decimal.from_number(Decimal('3.14')) # another decimal instance
+ Decimal('3.14')
+ """
+ if isinstance(number, (int, Decimal, float)):
+ return cls(number)
+ raise TypeError("Cannot convert %r to Decimal" % number)
+
@classmethod
def from_float(cls, f):
"""Converts a float to a decimal number, exactly.
@@ -2425,12 +2463,12 @@ def __pow__(self, other, modulo=None, context=None):
return ans
- def __rpow__(self, other, context=None):
+ def __rpow__(self, other, modulo=None, context=None):
"""Swaps self/other and returns __pow__."""
other = _convert_other(other)
if other is NotImplemented:
return other
- return other.__pow__(self, context=context)
+ return other.__pow__(self, modulo, context=context)
def normalize(self, context=None):
"""Normalize- strip trailing 0s, change anything equal to 0 to 0e0"""
@@ -3302,7 +3340,10 @@ def _fill_logical(self, context, opa, opb):
return opa, opb
def logical_and(self, other, context=None):
- """Applies an 'and' operation between self and other's digits."""
+ """Applies an 'and' operation between self and other's digits.
+
+ Both self and other must be logical numbers.
+ """
if context is None:
context = getcontext()
@@ -3319,14 +3360,20 @@ def logical_and(self, other, context=None):
return _dec_from_triple(0, result.lstrip('0') or '0', 0)
def logical_invert(self, context=None):
- """Invert all its digits."""
+ """Invert all its digits.
+
+ The self must be logical number.
+ """
if context is None:
context = getcontext()
return self.logical_xor(_dec_from_triple(0,'1'*context.prec,0),
context)
def logical_or(self, other, context=None):
- """Applies an 'or' operation between self and other's digits."""
+ """Applies an 'or' operation between self and other's digits.
+
+ Both self and other must be logical numbers.
+ """
if context is None:
context = getcontext()
@@ -3343,7 +3390,10 @@ def logical_or(self, other, context=None):
return _dec_from_triple(0, result.lstrip('0') or '0', 0)
def logical_xor(self, other, context=None):
- """Applies an 'xor' operation between self and other's digits."""
+ """Applies an 'xor' operation between self and other's digits.
+
+ Both self and other must be logical numbers.
+ """
if context is None:
context = getcontext()
@@ -6058,7 +6108,7 @@ def _convert_for_comparison(self, other, equality_op=False):
(?P\d*) # with (possibly empty) diagnostic info.
)
# \s*
- \Z
+ \z
""", re.VERBOSE | re.IGNORECASE).match
_all_zeros = re.compile('0*$').match
@@ -6082,11 +6132,15 @@ def _convert_for_comparison(self, other, equality_op=False):
(?Pz)?
(?P\#)?
(?P0)?
-(?P(?!0)\d+)?
-(?P,)?
-(?:\.(?P0|(?!0)\d+))?
+(?P\d+)?
+(?P[,_])?
+(?:\.
+ (?=[\d,_]) # lookahead for digit or separator
+ (?P\d+)?
+ (?P[,_])?
+)?
(?P[eEfFgGn%])?
-\Z
+\z
""", re.VERBOSE|re.DOTALL)
del re
@@ -6177,6 +6231,9 @@ def _parse_format_specifier(format_spec, _localeconv=None):
format_dict['grouping'] = [3, 0]
format_dict['decimal_point'] = '.'
+ if format_dict['frac_separators'] is None:
+ format_dict['frac_separators'] = ''
+
return format_dict
def _format_align(sign, body, spec):
@@ -6296,6 +6353,11 @@ def _format_number(is_negative, intpart, fracpart, exp, spec):
sign = _format_sign(is_negative, spec)
+ frac_sep = spec['frac_separators']
+ if fracpart and frac_sep:
+ fracpart = frac_sep.join(fracpart[pos:pos + 3]
+ for pos in range(0, len(fracpart), 3))
+
if fracpart or spec['alt']:
fracpart = spec['decimal_point'] + fracpart
diff --git a/PythonLib/full/_pyio.py b/PythonLib/full/_pyio.py
index 687076fbe..116ce4f37 100644
--- a/PythonLib/full/_pyio.py
+++ b/PythonLib/full/_pyio.py
@@ -16,15 +16,16 @@
_setmode = None
import io
-from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END)
+from io import (__all__, SEEK_SET, SEEK_CUR, SEEK_END, Reader, Writer) # noqa: F401
valid_seek_flags = {0, 1, 2} # Hardwired values
if hasattr(os, 'SEEK_HOLE') :
valid_seek_flags.add(os.SEEK_HOLE)
valid_seek_flags.add(os.SEEK_DATA)
-# open() uses st_blksize whenever we can
-DEFAULT_BUFFER_SIZE = 8 * 1024 # bytes
+# open() uses max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)
+# when the device block size is available.
+DEFAULT_BUFFER_SIZE = 128 * 1024 # bytes
# NOTE: Base classes defined here are registered with the "official" ABCs
# defined in io.py. We don't use real inheritance though, because we don't want
@@ -33,11 +34,8 @@
# Rebind for compatibility
BlockingIOError = BlockingIOError
-# Does io.IOBase finalizer log the exception if the close() method fails?
-# The exception is ignored silently by default in release build.
-_IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
# Does open() check its 'errors' argument?
-_CHECK_ERRORS = _IOBASE_EMITS_UNRAISABLE
+_CHECK_ERRORS = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode)
def text_encoding(encoding, stacklevel=2):
@@ -126,10 +124,10 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
the size of a fixed-size chunk buffer. When no buffering argument is
given, the default buffering policy works as follows:
- * Binary files are buffered in fixed-size chunks; the size of the buffer
- is chosen using a heuristic trying to determine the underlying device's
- "block size" and falling back on `io.DEFAULT_BUFFER_SIZE`.
- On many systems, the buffer will typically be 4096 or 8192 bytes long.
+ * Binary files are buffered in fixed-size chunks; the size of the buffer
+ is max(min(blocksize, 8 MiB), DEFAULT_BUFFER_SIZE)
+ when the device block size is available.
+ On most systems, the buffer will typically be 128 kilobytes long.
* "Interactive" text files (files for which isatty() returns True)
use line buffering. Other text files use the policy described above
@@ -241,18 +239,11 @@ def open(file, mode="r", buffering=-1, encoding=None, errors=None,
result = raw
try:
line_buffering = False
- if buffering == 1 or buffering < 0 and raw.isatty():
+ if buffering == 1 or buffering < 0 and raw._isatty_open_only():
buffering = -1
line_buffering = True
if buffering < 0:
- buffering = DEFAULT_BUFFER_SIZE
- try:
- bs = os.fstat(raw.fileno()).st_blksize
- except (OSError, AttributeError):
- pass
- else:
- if bs > 1:
- buffering = bs
+ buffering = max(min(raw._blksize, 8192 * 1024), DEFAULT_BUFFER_SIZE)
if buffering < 0:
raise ValueError("invalid buffering size")
if buffering == 0:
@@ -416,18 +407,12 @@ def __del__(self):
if closed:
return
- if _IOBASE_EMITS_UNRAISABLE:
- self.close()
- else:
- # The try/except block is in case this is called at program
- # exit time, when it's possible that globals have already been
- # deleted, and then the close() call might fail. Since
- # there's nothing we can do about such failures and they annoy
- # the end users, we suppress the traceback.
- try:
- self.close()
- except:
- pass
+ if dealloc_warn := getattr(self, "_dealloc_warn", None):
+ dealloc_warn(self)
+
+ # If close() fails, the caller logs the exception with
+ # sys.unraisablehook. close() must be called at the end at __del__().
+ self.close()
### Inquiries ###
@@ -632,6 +617,8 @@ def read(self, size=-1):
n = self.readinto(b)
if n is None:
return None
+ if n < 0 or n > len(b):
+ raise ValueError(f"readinto returned {n} outside buffer size {len(b)}")
del b[n:]
return bytes(b)
@@ -663,8 +650,6 @@ def write(self, b):
self._unsupported("write")
io.RawIOBase.register(RawIOBase)
-from _io import FileIO
-RawIOBase.register(FileIO)
class BufferedIOBase(IOBase):
@@ -871,6 +856,10 @@ def __repr__(self):
else:
return "<{}.{} name={!r}>".format(modname, clsname, name)
+ def _dealloc_warn(self, source):
+ if dealloc_warn := getattr(self.raw, "_dealloc_warn", None):
+ dealloc_warn(source)
+
### Lower-level APIs ###
def fileno(self):
@@ -946,22 +935,22 @@ def read1(self, size=-1):
return self.read(size)
def write(self, b):
- if self.closed:
- raise ValueError("write to closed file")
if isinstance(b, str):
raise TypeError("can't write str to binary stream")
with memoryview(b) as view:
+ if self.closed:
+ raise ValueError("write to closed file")
+
n = view.nbytes # Size of any bytes-like object
- if n == 0:
- return 0
- pos = self._pos
- if pos > len(self._buffer):
- # Inserts null bytes between the current end of the file
- # and the new write position.
- padding = b'\x00' * (pos - len(self._buffer))
- self._buffer += padding
- self._buffer[pos:pos + n] = b
- self._pos += n
+ if n == 0:
+ return 0
+
+ pos = self._pos
+ if pos > len(self._buffer):
+ # Pad buffer to pos with null bytes.
+ self._buffer.resize(pos)
+ self._buffer[pos:pos + n] = view
+ self._pos += n
return n
def seek(self, pos, whence=0):
@@ -1475,6 +1464,17 @@ def write(self, b):
return BufferedWriter.write(self, b)
+def _new_buffersize(bytes_read):
+ # Parallels _io/fileio.c new_buffersize
+ if bytes_read > 65536:
+ addend = bytes_read >> 3
+ else:
+ addend = 256 + bytes_read
+ if addend < DEFAULT_BUFFER_SIZE:
+ addend = DEFAULT_BUFFER_SIZE
+ return bytes_read + addend
+
+
class FileIO(RawIOBase):
_fd = -1
_created = False
@@ -1499,6 +1499,7 @@ def __init__(self, file, mode='r', closefd=True, opener=None):
"""
if self._fd >= 0:
# Have to close the existing file first.
+ self._stat_atopen = None
try:
if self._closefd:
os.close(self._fd)
@@ -1508,6 +1509,11 @@ def __init__(self, file, mode='r', closefd=True, opener=None):
if isinstance(file, float):
raise TypeError('integer argument expected, got float')
if isinstance(file, int):
+ if isinstance(file, bool):
+ import warnings
+ warnings.warn("bool is used as a file descriptor",
+ RuntimeWarning, stacklevel=2)
+ file = int(file)
fd = file
if fd < 0:
raise ValueError('negative file descriptor')
@@ -1566,24 +1572,22 @@ def __init__(self, file, mode='r', closefd=True, opener=None):
if not isinstance(fd, int):
raise TypeError('expected integer from opener')
if fd < 0:
- raise OSError('Negative file descriptor')
+ # bpo-27066: Raise a ValueError for bad value.
+ raise ValueError(f'opener returned {fd}')
owned_fd = fd
if not noinherit_flag:
os.set_inheritable(fd, False)
self._closefd = closefd
- fdfstat = os.fstat(fd)
+ self._stat_atopen = os.fstat(fd)
try:
- if stat.S_ISDIR(fdfstat.st_mode):
+ if stat.S_ISDIR(self._stat_atopen.st_mode):
raise IsADirectoryError(errno.EISDIR,
os.strerror(errno.EISDIR), file)
except AttributeError:
# Ignore the AttributeError if stat.S_ISDIR or errno.EISDIR
# don't exist.
pass
- self._blksize = getattr(fdfstat, 'st_blksize', 0)
- if self._blksize <= 1:
- self._blksize = DEFAULT_BUFFER_SIZE
if _setmode:
# don't translate newlines (\r\n <=> \n)
@@ -1600,17 +1604,17 @@ def __init__(self, file, mode='r', closefd=True, opener=None):
if e.errno != errno.ESPIPE:
raise
except:
+ self._stat_atopen = None
if owned_fd is not None:
os.close(owned_fd)
raise
self._fd = fd
- def __del__(self):
+ def _dealloc_warn(self, source):
if self._fd >= 0 and self._closefd and not self.closed:
import warnings
- warnings.warn('unclosed file %r' % (self,), ResourceWarning,
+ warnings.warn(f'unclosed file {source!r}', ResourceWarning,
stacklevel=2, source=self)
- self.close()
def __getstate__(self):
raise TypeError(f"cannot pickle {self.__class__.__name__!r} object")
@@ -1629,6 +1633,17 @@ def __repr__(self):
return ('<%s name=%r mode=%r closefd=%r>' %
(class_name, name, self.mode, self._closefd))
+ @property
+ def _blksize(self):
+ if self._stat_atopen is None:
+ return DEFAULT_BUFFER_SIZE
+
+ blksize = getattr(self._stat_atopen, "st_blksize", 0)
+ # WASI sets blsize to 0
+ if not blksize:
+ return DEFAULT_BUFFER_SIZE
+ return blksize
+
def _checkReadable(self):
if not self._readable:
raise UnsupportedOperation('File not open for reading')
@@ -1640,7 +1655,13 @@ def _checkWritable(self, msg=None):
def read(self, size=None):
"""Read at most size bytes, returned as bytes.
- Only makes one system call, so less data may be returned than requested
+ If size is less than 0, read all bytes in the file making
+ multiple read calls. See ``FileIO.readall``.
+
+ Attempts to make only one system call, retrying only per
+ PEP 475 (EINTR). This means less data may be returned than
+ requested.
+
In non-blocking mode, returns None if no data is available.
Return an empty bytes object at EOF.
"""
@@ -1656,45 +1677,57 @@ def read(self, size=None):
def readall(self):
"""Read all data from the file, returned as bytes.
- In non-blocking mode, returns as much as is immediately available,
- or None if no data is available. Return an empty bytes object at EOF.
+ Reads until either there is an error or read() returns size 0
+ (indicates EOF). If the file is already at EOF, returns an
+ empty bytes object.
+
+ In non-blocking mode, returns as much data as could be read
+ before EAGAIN. If no data is available (EAGAIN is returned
+ before bytes are read) returns None.
"""
self._checkClosed()
self._checkReadable()
- bufsize = DEFAULT_BUFFER_SIZE
- try:
- pos = os.lseek(self._fd, 0, SEEK_CUR)
- end = os.fstat(self._fd).st_size
- if end >= pos:
- bufsize = end - pos + 1
- except OSError:
- pass
+ if self._stat_atopen is None or self._stat_atopen.st_size <= 0:
+ bufsize = DEFAULT_BUFFER_SIZE
+ else:
+ # In order to detect end of file, need a read() of at least 1
+ # byte which returns size 0. Oversize the buffer by 1 byte so the
+ # I/O can be completed with two read() calls (one for all data, one
+ # for EOF) without needing to resize the buffer.
+ bufsize = self._stat_atopen.st_size + 1
- result = bytearray()
- while True:
- if len(result) >= bufsize:
- bufsize = len(result)
- bufsize += max(bufsize, DEFAULT_BUFFER_SIZE)
- n = bufsize - len(result)
- try:
- chunk = os.read(self._fd, n)
- except BlockingIOError:
- if result:
- break
+ if self._stat_atopen.st_size > 65536:
+ try:
+ pos = os.lseek(self._fd, 0, SEEK_CUR)
+ if self._stat_atopen.st_size >= pos:
+ bufsize = self._stat_atopen.st_size - pos + 1
+ except OSError:
+ pass
+
+ result = bytearray(bufsize)
+ bytes_read = 0
+ try:
+ while n := os.readinto(self._fd, memoryview(result)[bytes_read:]):
+ bytes_read += n
+ if bytes_read >= len(result):
+ result.resize(_new_buffersize(bytes_read))
+ except BlockingIOError:
+ if not bytes_read:
return None
- if not chunk: # reached the end of the file
- break
- result += chunk
+ assert len(result) - bytes_read >= 1, \
+ "os.readinto buffer size 0 will result in erroneous EOF / returns 0"
+ result.resize(bytes_read)
return bytes(result)
- def readinto(self, b):
+ def readinto(self, buffer):
"""Same as RawIOBase.readinto()."""
- m = memoryview(b).cast('B')
- data = self.read(len(m))
- n = len(data)
- m[:n] = data
- return n
+ self._checkClosed()
+ self._checkReadable()
+ try:
+ return os.readinto(self._fd, buffer)
+ except BlockingIOError:
+ return None
def write(self, b):
"""Write bytes b to file, return number written.
@@ -1744,6 +1777,7 @@ def truncate(self, size=None):
if size is None:
size = self.tell()
os.ftruncate(self._fd, size)
+ self._stat_atopen = None
return size
def close(self):
@@ -1753,8 +1787,9 @@ def close(self):
called more than once without error.
"""
if not self.closed:
+ self._stat_atopen = None
try:
- if self._closefd:
+ if self._closefd and self._fd >= 0:
os.close(self._fd)
finally:
super().close()
@@ -1791,6 +1826,21 @@ def isatty(self):
self._checkClosed()
return os.isatty(self._fd)
+ def _isatty_open_only(self):
+ """Checks whether the file is a TTY using an open-only optimization.
+
+ TTYs are always character devices. If the interpreter knows a file is
+ not a character device when it would call ``isatty``, can skip that
+ call. Inside ``open()`` there is a fresh stat result that contains that
+ information. Use the stat result to skip a system call. Outside of that
+ context TOCTOU issues (the fd could be arbitrarily modified by
+ surrounding code).
+ """
+ if (self._stat_atopen is not None
+ and not stat.S_ISCHR(self._stat_atopen.st_mode)):
+ return False
+ return os.isatty(self._fd)
+
@property
def closefd(self):
"""True if the file descriptor will be closed by close()."""
@@ -2015,8 +2065,7 @@ def __init__(self, buffer, encoding=None, errors=None, newline=None,
raise ValueError("invalid encoding: %r" % encoding)
if not codecs.lookup(encoding)._is_text_encoding:
- msg = ("%r is not a text encoding; "
- "use codecs.open() to handle arbitrary codecs")
+ msg = "%r is not a text encoding"
raise LookupError(msg % encoding)
if errors is None:
@@ -2524,9 +2573,12 @@ def read(self, size=None):
size = size_index()
decoder = self._decoder or self._get_decoder()
if size < 0:
+ chunk = self.buffer.read()
+ if chunk is None:
+ raise BlockingIOError("Read returned None.")
# Read everything.
result = (self._get_decoded_chars() +
- decoder.decode(self.buffer.read(), final=True))
+ decoder.decode(chunk, final=True))
if self._snapshot is not None:
self._set_decoded_chars('')
self._snapshot = None
@@ -2646,6 +2698,10 @@ def readline(self, size=None):
def newlines(self):
return self._decoder.newlines if self._decoder else None
+ def _dealloc_warn(self, source):
+ if dealloc_warn := getattr(self.buffer, "_dealloc_warn", None):
+ dealloc_warn(source)
+
class StringIO(TextIOWrapper):
"""Text I/O implementation using an in-memory buffer.
diff --git a/PythonLib/full/_pylong.py b/PythonLib/full/_pylong.py
index 30bee6fc9..be1acd17c 100644
--- a/PythonLib/full/_pylong.py
+++ b/PythonLib/full/_pylong.py
@@ -19,6 +19,122 @@
except ImportError:
_decimal = None
+# A number of functions have this form, where `w` is a desired number of
+# digits in base `base`:
+#
+# def inner(...w...):
+# if w <= LIMIT:
+# return something
+# lo = w >> 1
+# hi = w - lo
+# something involving base**lo, inner(...lo...), j, and inner(...hi...)
+# figure out largest w needed
+# result = inner(w)
+#
+# They all had some on-the-fly scheme to cache `base**lo` results for reuse.
+# Power is costly.
+#
+# This routine aims to compute all amd only the needed powers in advance, as
+# efficiently as reasonably possible. This isn't trivial, and all the
+# on-the-fly methods did needless work in many cases. The driving code above
+# changes to:
+#
+# figure out largest w needed
+# mycache = compute_powers(w, base, LIMIT)
+# result = inner(w)
+#
+# and `mycache[lo]` replaces `base**lo` in the inner function.
+#
+# If an algorithm wants the powers of ceiling(w/2) instead of the floor,
+# pass keyword argument `need_hi=True`.
+#
+# While this does give minor speedups (a few percent at best), the
+# primary intent is to simplify the functions using this, by eliminating
+# the need for them to craft their own ad-hoc caching schemes.
+#
+# See code near end of file for a block of code that can be enabled to
+# run millions of tests.
+def compute_powers(w, base, more_than, *, need_hi=False, show=False):
+ seen = set()
+ need = set()
+ ws = {w}
+ while ws:
+ w = ws.pop() # any element is fine to use next
+ if w in seen or w <= more_than:
+ continue
+ seen.add(w)
+ lo = w >> 1
+ hi = w - lo
+ # only _need_ one here; the other may, or may not, be needed
+ which = hi if need_hi else lo
+ need.add(which)
+ ws.add(which)
+ if lo != hi:
+ ws.add(w - which)
+
+ # `need` is the set of exponents needed. To compute them all
+ # efficiently, possibly add other exponents to `extra`. The goal is
+ # to ensure that each exponent can be gotten from a smaller one via
+ # multiplying by the base, squaring it, or squaring and then
+ # multiplying by the base.
+ #
+ # If need_hi is False, this is already the case (w can always be
+ # gotten from w >> 1 via one of the squaring strategies). But we do
+ # the work anyway, just in case ;-)
+ #
+ # Note that speed is irrelevant. These loops are working on little
+ # ints (exponents) and go around O(log w) times. The total cost is
+ # insignificant compared to just one of the bigint multiplies.
+ cands = need.copy()
+ extra = set()
+ while cands:
+ w = max(cands)
+ cands.remove(w)
+ lo = w >> 1
+ if lo > more_than and w-1 not in cands and lo not in cands:
+ extra.add(lo)
+ cands.add(lo)
+ assert need_hi or not extra
+
+ d = {}
+ for n in sorted(need | extra):
+ lo = n >> 1
+ hi = n - lo
+ if n-1 in d:
+ if show:
+ print("* base", end="")
+ result = d[n-1] * base # cheap!
+ elif lo in d:
+ # Multiplying a bigint by itself is about twice as fast
+ # in CPython provided it's the same object.
+ if show:
+ print("square", end="")
+ result = d[lo] * d[lo] # same object
+ if hi != lo:
+ if show:
+ print(" * base", end="")
+ assert 2 * lo + 1 == n
+ result *= base
+ else: # rare
+ if show:
+ print("pow", end='')
+ result = base ** n
+ if show:
+ print(" at", n, "needed" if n in need else "extra")
+ d[n] = result
+
+ assert need <= d.keys()
+ if excess := d.keys() - need:
+ assert need_hi
+ for n in excess:
+ del d[n]
+ return d
+
+_unbounded_dec_context = decimal.getcontext().copy()
+_unbounded_dec_context.prec = decimal.MAX_PREC
+_unbounded_dec_context.Emax = decimal.MAX_EMAX
+_unbounded_dec_context.Emin = decimal.MIN_EMIN
+_unbounded_dec_context.traps[decimal.Inexact] = 1 # sanity check
def int_to_decimal(n):
"""Asymptotically fast conversion of an 'int' to Decimal."""
@@ -33,57 +149,32 @@ def int_to_decimal(n):
# "clever" recursive way. If we want a string representation, we
# apply str to _that_.
- D = decimal.Decimal
- D2 = D(2)
-
- BITLIM = 128
-
- mem = {}
-
- def w2pow(w):
- """Return D(2)**w and store the result. Also possibly save some
- intermediate results. In context, these are likely to be reused
- across various levels of the conversion to Decimal."""
- if (result := mem.get(w)) is None:
- if w <= BITLIM:
- result = D2**w
- elif w - 1 in mem:
- result = (t := mem[w - 1]) + t
- else:
- w2 = w >> 1
- # If w happens to be odd, w-w2 is one larger then w2
- # now. Recurse on the smaller first (w2), so that it's
- # in the cache and the larger (w-w2) can be handled by
- # the cheaper `w-1 in mem` branch instead.
- result = w2pow(w2) * w2pow(w - w2)
- mem[w] = result
- return result
+ from decimal import Decimal as D
+ BITLIM = 200
+ # Don't bother caching the "lo" mask in this; the time to compute it is
+ # tiny compared to the multiply.
def inner(n, w):
if w <= BITLIM:
return D(n)
w2 = w >> 1
hi = n >> w2
- lo = n - (hi << w2)
- return inner(lo, w2) + inner(hi, w - w2) * w2pow(w2)
-
- with decimal.localcontext() as ctx:
- ctx.prec = decimal.MAX_PREC
- ctx.Emax = decimal.MAX_EMAX
- ctx.Emin = decimal.MIN_EMIN
- ctx.traps[decimal.Inexact] = 1
+ lo = n & ((1 << w2) - 1)
+ return inner(lo, w2) + inner(hi, w - w2) * w2pow[w2]
+ with decimal.localcontext(_unbounded_dec_context):
+ nbits = n.bit_length()
+ w2pow = compute_powers(nbits, D(2), BITLIM)
if n < 0:
negate = True
n = -n
else:
negate = False
- result = inner(n, n.bit_length())
+ result = inner(n, nbits)
if negate:
result = -result
return result
-
def int_to_decimal_string(n):
"""Asymptotically fast conversion of an 'int' to a decimal string."""
w = n.bit_length()
@@ -97,14 +188,13 @@ def int_to_decimal_string(n):
# available. This algorithm is asymptotically worse than the algorithm
# using the decimal module, but better than the quadratic time
# implementation in longobject.c.
+
+ DIGLIM = 1000
def inner(n, w):
- if w <= 1000:
+ if w <= DIGLIM:
return str(n)
w2 = w >> 1
- d = pow10_cache.get(w2)
- if d is None:
- d = pow10_cache[w2] = 5**w2 << w2 # 10**i = (5*2)**i = 5**i * 2**i
- hi, lo = divmod(n, d)
+ hi, lo = divmod(n, pow10[w2])
return inner(hi, w - w2) + inner(lo, w2).zfill(w2)
# The estimation of the number of decimal digits.
@@ -115,7 +205,9 @@ def inner(n, w):
# only if the number has way more than 10**15 digits, that exceeds
# the 52-bit physical address limit in both Intel64 and AMD64.
w = int(w * 0.3010299956639812 + 1) # log10(2)
- pow10_cache = {}
+ pow10 = compute_powers(w, 5, DIGLIM)
+ for k, v in pow10.items():
+ pow10[k] = v << k # 5**k << k == 5**k * 2**k == 10**k
if n < 0:
n = -n
sign = '-'
@@ -128,7 +220,6 @@ def inner(n, w):
s = s.lstrip('0')
return sign + s
-
def _str_to_int_inner(s):
"""Asymptotically fast conversion of a 'str' to an 'int'."""
@@ -144,38 +235,157 @@ def _str_to_int_inner(s):
DIGLIM = 2048
- mem = {}
-
- def w5pow(w):
- """Return 5**w and store the result.
- Also possibly save some intermediate results. In context, these
- are likely to be reused across various levels of the conversion
- to 'int'.
- """
- if (result := mem.get(w)) is None:
- if w <= DIGLIM:
- result = 5**w
- elif w - 1 in mem:
- result = mem[w - 1] * 5
- else:
- w2 = w >> 1
- # If w happens to be odd, w-w2 is one larger then w2
- # now. Recurse on the smaller first (w2), so that it's
- # in the cache and the larger (w-w2) can be handled by
- # the cheaper `w-1 in mem` branch instead.
- result = w5pow(w2) * w5pow(w - w2)
- mem[w] = result
- return result
-
def inner(a, b):
if b - a <= DIGLIM:
return int(s[a:b])
mid = (a + b + 1) >> 1
- return inner(mid, b) + ((inner(a, mid) * w5pow(b - mid)) << (b - mid))
+ return (inner(mid, b)
+ + ((inner(a, mid) * w5pow[b - mid])
+ << (b - mid)))
+ w5pow = compute_powers(len(s), 5, DIGLIM)
return inner(0, len(s))
+# Asymptotically faster version, using the C decimal module. See
+# comments at the end of the file. This uses decimal arithmetic to
+# convert from base 10 to base 256. The latter is just a string of
+# bytes, which CPython can convert very efficiently to a Python int.
+
+# log of 10 to base 256 with best-possible 53-bit precision. Obtained
+# via:
+# from mpmath import mp
+# mp.prec = 1000
+# print(float(mp.log(10, 256)).hex())
+_LOG_10_BASE_256 = float.fromhex('0x1.a934f0979a371p-2') # about 0.415
+
+# _spread is for internal testing. It maps a key to the number of times
+# that condition obtained in _dec_str_to_int_inner:
+# key 0 - quotient guess was right
+# key 1 - quotient had to be boosted by 1, one time
+# key 999 - one adjustment wasn't enough, so fell back to divmod
+from collections import defaultdict
+_spread = defaultdict(int)
+del defaultdict
+
+def _dec_str_to_int_inner(s, *, GUARD=8):
+ # Yes, BYTELIM is "large". Large enough that CPython will usually
+ # use the Karatsuba _str_to_int_inner to convert the string. This
+ # allowed reducing the cutoff for calling _this_ function from 3.5M
+ # to 2M digits. We could almost certainly do even better by
+ # fine-tuning this and/or using a larger output base than 256.
+ BYTELIM = 100_000
+ D = decimal.Decimal
+ result = bytearray()
+ # See notes at end of file for discussion of GUARD.
+ assert GUARD > 0 # if 0, `decimal` can blow up - .prec 0 not allowed
+
+ def inner(n, w):
+ #assert n < D256 ** w # required, but too expensive to check
+ if w <= BYTELIM:
+ # XXX Stefan Pochmann discovered that, for 1024-bit ints,
+ # `int(Decimal)` took 2.5x longer than `int(str(Decimal))`.
+ # Worse, `int(Decimal) is still quadratic-time for much
+ # larger ints. So unless/until all that is repaired, the
+ # seemingly redundant `str(Decimal)` is crucial to speed.
+ result.extend(int(str(n)).to_bytes(w)) # big-endian default
+ return
+ w1 = w >> 1
+ w2 = w - w1
+ if 0:
+ # This is maximally clear, but "too slow". `decimal`
+ # division is asymptotically fast, but we have no way to
+ # tell it to reuse the high-precision reciprocal it computes
+ # for pow256[w2], so it has to recompute it over & over &
+ # over again :-(
+ hi, lo = divmod(n, pow256[w2][0])
+ else:
+ p256, recip = pow256[w2]
+ # The integer part will have a number of digits about equal
+ # to the difference between the log10s of `n` and `pow256`
+ # (which, since these are integers, is roughly approximated
+ # by `.adjusted()`). That's the working precision we need,
+ ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD
+ hi = +n * +recip # unary `+` chops back to ctx.prec digits
+ ctx.prec = decimal.MAX_PREC
+ hi = hi.to_integral_value() # lose the fractional digits
+ lo = n - hi * p256
+ # Because we've been uniformly rounding down, `hi` is a
+ # lower bound on the correct quotient.
+ assert lo >= 0
+ # Adjust quotient up if needed. It usually isn't. In random
+ # testing on inputs through 5 billion digit strings, the
+ # test triggered once in about 200 thousand tries.
+ count = 0
+ if lo >= p256:
+ count = 1
+ lo -= p256
+ hi += 1
+ if lo >= p256:
+ # Complete correction via an exact computation. I
+ # believe it's not possible to get here provided
+ # GUARD >= 3. It's tested by reducing GUARD below
+ # that.
+ count = 999
+ hi2, lo = divmod(lo, p256)
+ hi += hi2
+ _spread[count] += 1
+ # The assert should always succeed, but way too slow to keep
+ # enabled.
+ #assert hi, lo == divmod(n, pow256[w2][0])
+ inner(hi, w1)
+ del hi # at top levels, can free a lot of RAM "early"
+ inner(lo, w2)
+
+ # How many base 256 digits are needed?. Mathematically, exactly
+ # floor(log256(int(s))) + 1. There is no cheap way to compute this.
+ # But we can get an upper bound, and that's necessary for our error
+ # analysis to make sense. int(s) < 10**len(s), so the log needed is
+ # < log256(10**len(s)) = len(s) * log256(10). However, using
+ # finite-precision floating point for this, it's possible that the
+ # computed value is a little less than the true value. If the true
+ # value is at - or a little higher than - an integer, we can get an
+ # off-by-1 error too low. So we add 2 instead of 1 if chopping lost
+ # a fraction > 0.9.
+
+ # The "WASI" test platform can complain about `len(s)` if it's too
+ # large to fit in its idea of "an index-sized integer".
+ lenS = s.__len__()
+ log_ub = lenS * _LOG_10_BASE_256
+ log_ub_as_int = int(log_ub)
+ w = log_ub_as_int + 1 + (log_ub - log_ub_as_int > 0.9)
+ # And what if we've plain exhausted the limits of HW floats? We
+ # could compute the log to any desired precision using `decimal`,
+ # but it's not plausible that anyone will pass a string requiring
+ # trillions of bytes (unless they're just trying to "break things").
+ if w.bit_length() >= 46:
+ # "Only" had < 53 - 46 = 7 bits to spare in IEEE-754 double.
+ raise ValueError(f"cannot convert string of len {lenS} to int")
+ with decimal.localcontext(_unbounded_dec_context) as ctx:
+ D256 = D(256)
+ pow256 = compute_powers(w, D256, BYTELIM, need_hi=True)
+ rpow256 = compute_powers(w, 1 / D256, BYTELIM, need_hi=True)
+ # We're going to do inexact, chopped arithmetic, multiplying by
+ # an approximation to the reciprocal of 256**i. We chop to get a
+ # lower bound on the true integer quotient. Our approximation is
+ # a lower bound, the multiplication is chopped too, and
+ # to_integral_value() is also chopped.
+ ctx.traps[decimal.Inexact] = 0
+ ctx.rounding = decimal.ROUND_DOWN
+ for k, v in pow256.items():
+ # No need to save much more precision in the reciprocal than
+ # the power of 256 has, plus some guard digits to absorb
+ # most relevant rounding errors. This is highly significant:
+ # 1/2**i has the same number of significant decimal digits
+ # as 5**i, generally over twice the number in 2**i,
+ ctx.prec = v.adjusted() + GUARD + 1
+ # The unary "+" chops the reciprocal back to that precision.
+ pow256[k] = v, +rpow256[k]
+ del rpow256 # exact reciprocals no longer needed
+ ctx.prec = decimal.MAX_PREC
+ inner(D(s), w)
+ return int.from_bytes(result)
+
def int_from_string(s):
"""Asymptotically fast version of PyLong_FromString(), conversion
of a string of decimal digits into an 'int'."""
@@ -184,8 +394,10 @@ def int_from_string(s):
# and underscores, and stripped leading whitespace. The input can still
# contain underscores and have trailing whitespace.
s = s.rstrip().replace('_', '')
- return _str_to_int_inner(s)
-
+ func = _str_to_int_inner
+ if len(s) >= 2_000_000 and _decimal is not None:
+ func = _dec_str_to_int_inner
+ return func(s)
def str_to_int(s):
"""Asymptotically fast version of decimal string to 'int' conversion."""
@@ -318,7 +530,7 @@ def int_divmod(a, b):
Its time complexity is O(n**1.58), where n = #bits(a) + #bits(b).
"""
if b == 0:
- raise ZeroDivisionError
+ raise ZeroDivisionError('division by zero')
elif b < 0:
q, r = int_divmod(-a, -b)
return q, -r
@@ -327,3 +539,191 @@ def int_divmod(a, b):
return ~q, b + ~r
else:
return _divmod_pos(a, b)
+
+
+# Notes on _dec_str_to_int_inner:
+#
+# Stefan Pochmann worked up a str->int function that used the decimal
+# module to, in effect, convert from base 10 to base 256. This is
+# "unnatural", in that it requires multiplying and dividing by large
+# powers of 2, which `decimal` isn't naturally suited to. But
+# `decimal`'s `*` and `/` are asymptotically superior to CPython's, so
+# at _some_ point it could be expected to win.
+#
+# Alas, the crossover point was too high to be of much real interest. I
+# (Tim) then worked on ways to replace its division with multiplication
+# by a cached reciprocal approximation instead, fixing up errors
+# afterwards. This reduced the crossover point significantly,
+#
+# I revisited the code, and found ways to improve and simplify it. The
+# crossover point is at about 3.4 million digits now.
+#
+# About .adjusted()
+# -----------------
+# Restrict to Decimal values x > 0. We don't use negative numbers in the
+# code, and I don't want to have to keep typing, e.g., "absolute value".
+#
+# For convenience, I'll use `x.a` to mean `x.adjusted()`. x.a doesn't
+# look at the digits of x, but instead returns an integer giving x's
+# order of magnitude. These are equivalent:
+#
+# - x.a is the power-of-10 exponent of x's most significant digit.
+# - x.a = the infinitely precise floor(log10(x))
+# - x can be written in this form, where f is a real with 1 <= f < 10:
+# x = f * 10**x.a
+#
+# Observation; if x is an integer, len(str(x)) = x.a + 1.
+#
+# Lemma 1: (x * y).a = x.a + y.a, or one larger
+#
+# Proof: Write x = f * 10**x.a and y = g * 10**y.a, where f and g are in
+# [1, 10). Then x*y = f*g * 10**(x.a + y.a), where 1 <= f*g < 100. If
+# f*g < 10, (x*y).a is x.a+y.a. Else divide f*g by 10 to bring it back
+# into [1, 10], and add 1 to the exponent to compensate. Then (x*y).a is
+# x.a+y.a+1.
+#
+# Lemma 2: ceiling(log10(x/y)) <= x.a - y.a + 1
+#
+# Proof: Express x and y as in Lemma 1. Then x/y = f/g * 10**(x.a -
+# y.a), where 1/10 < f/g < 10. If 1 <= f/g, (x/y).a is x.a-y.a. Else
+# multiply f/g by 10 to bring it back into [1, 10], and subtract 1 from
+# the exponent to compensate. Then (x/y).a is x.a-y.a-1. So the largest
+# (x/y).a can be is x.a-y.a. Since that's the floor of log10(x/y). the
+# ceiling is at most 1 larger (with equality iff f/g = 1 exactly).
+#
+# GUARD digits
+# ------------
+# We only want the integer part of divisions, so don't need to build
+# the full multiplication tree. But using _just_ the number of
+# digits expected in the integer part ignores too much. What's left
+# out can have a very significant effect on the quotient. So we use
+# GUARD additional digits.
+#
+# The default 8 is more than enough so no more than 1 correction step
+# was ever needed for all inputs tried through 2.5 billion digits. In
+# fact, I believe 3 guard digits are always enough - but the proof is
+# very involved, so better safe than sorry.
+#
+# Short course:
+#
+# If prec is the decimal precision in effect, and we're rounding down,
+# the result of an operation is exactly equal to the infinitely precise
+# result times 1-e for some real e with 0 <= e < 10**(1-prec). In
+#
+# ctx.prec = max(n.adjusted() - p256.adjusted(), 0) + GUARD
+# hi = +n * +recip # unary `+` chops to ctx.prec digits
+#
+# we have 3 visible chopped operations, but there's also a 4th:
+# precomputing a truncated `recip` as part of setup.
+#
+# So the computed product is exactly equal to the true product times
+# (1-e1)*(1-e2)*(1-e3)*(1-e4); since the e's are all very small, an
+# excellent approximation to the second factor is 1-(e1+e2+e3+e4) (the
+# 2nd and higher order terms in the expanded product are too tiny to
+# matter). If they're all as large as possible, that's
+#
+# 1 - 4*10**(1-prec). This, BTW, is all bog-standard FP error analysis.
+#
+# That implies the computed product is within 1 of the true product
+# provided prec >= log10(true_product) + 1.602.
+#
+# Here are telegraphic details, rephrasing the initial condition in
+# equivalent ways, step by step:
+#
+# prod - prod * (1 - 4*10**(1-prec)) <= 1
+# prod - prod + prod * 4*10**(1-prec)) <= 1
+# prod * 4*10**(1-prec)) <= 1
+# 10**(log10(prod)) * 4*10**(1-prec)) <= 1
+# 4*10**(1-prec+log10(prod))) <= 1
+# 10**(1-prec+log10(prod))) <= 1/4
+# 1-prec+log10(prod) <= log10(1/4) = -0.602
+# -prec <= -1.602 - log10(prod)
+# prec >= log10(prod) + 1.602
+#
+# The true product is the same as the true ratio n/p256. By Lemma 2
+# above, n.a - p256.a + 1 is an upper bound on the ceiling of
+# log10(prod). Then 2 is the ceiling of 1.602. so n.a - p256.a + 3 is an
+# upper bound on the right hand side of the inequality. Any prec >= that
+# will work.
+#
+# But since this is just a sketch of a proof ;-), the code uses the
+# empirically tested 8 instead of 3. 5 digits more or less makes no
+# practical difference to speed - these ints are huge. And while
+# increasing GUARD above 3 may not be necessary, every increase cuts the
+# percentage of cases that need a correction at all.
+#
+# On Computing Reciprocals
+# ------------------------
+# In general, the exact reciprocals we compute have over twice as many
+# significant digits as needed. 1/256**i has the same number of
+# significant decimal digits as 5**i. It's a significant waste of RAM
+# to store all those unneeded digits.
+#
+# So we cut exact reciprocals back to the least precision that can
+# be needed so that the error analysis above is valid,
+#
+# [Note: turns out it's very significantly faster to do it this way than
+# to compute 1 / 256**i directly to the desired precision, because the
+# power method doesn't require division. It's also faster than computing
+# (1/256)**i directly to the desired precision - no material division
+# there, but `compute_powers()` is much smarter about _how_ to compute
+# all the powers needed than repeated applications of `**` - that
+# function invokes `**` for at most the few smallest powers needed.]
+#
+# The hard part is that chopping back to a shorter width occurs
+# _outside_ of `inner`. We can't know then what `prec` `inner()` will
+# need. We have to pick, for each value of `w2`, the largest possible
+# value `prec` can become when `inner()` is working on `w2`.
+#
+# This is the `prec` inner() uses:
+# max(n.a - p256.a, 0) + GUARD
+# and what setup uses (renaming its `v` to `p256` - same thing):
+# p256.a + GUARD + 1
+#
+# We need that the second is always at least as large as the first,
+# which is the same as requiring
+#
+# n.a - 2 * p256.a <= 1
+#
+# What's the largest n can be? n < 255**w = 256**(w2 + (w - w2)). The
+# worst case in this context is when w ix even. and then w = 2*w2, so
+# n < 256**(2*w2) = (256**w2)**2 = p256**2. By Lemma 1, then, n.a
+# is at most p256.a + p256.a + 1.
+#
+# So the most n.a - 2 * p256.a can be is
+# p256.a + p256.a + 1 - 2 * p256.a = 1. QED
+#
+# Note: an earlier version of the code split on floor(e/2) instead of on
+# the ceiling. The worst case then is odd `w`, and a more involved proof
+# was needed to show that adding 4 (instead of 1) may be necessary.
+# Basically because, in that case, n may be up to 256 times larger than
+# p256**2. Curiously enough, by splitting on the ceiling instead,
+# nothing in any proof here actually depends on the output base (256).
+
+# Enable for brute-force testing of compute_powers(). This takes about a
+# minute, because it tries millions of cases.
+if 0:
+ def consumer(w, limit, need_hi):
+ seen = set()
+ need = set()
+ def inner(w):
+ if w <= limit:
+ return
+ if w in seen:
+ return
+ seen.add(w)
+ lo = w >> 1
+ hi = w - lo
+ need.add(hi if need_hi else lo)
+ inner(lo)
+ inner(hi)
+ inner(w)
+ exp = compute_powers(w, 1, limit, need_hi=need_hi)
+ assert exp.keys() == need
+
+ from itertools import chain
+ for need_hi in (False, True):
+ for limit in (0, 1, 10, 100, 1_000, 10_000, 100_000):
+ for w in chain(range(1, 100_000),
+ (10**i for i in range(5, 30))):
+ consumer(w, limit, need_hi)
diff --git a/PythonLib/full/_pyrepl/__init__.py b/PythonLib/full/_pyrepl/__init__.py
new file mode 100644
index 000000000..1693cbd0b
--- /dev/null
+++ b/PythonLib/full/_pyrepl/__init__.py
@@ -0,0 +1,19 @@
+# Copyright 2000-2008 Michael Hudson-Doyle
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
diff --git a/PythonLib/full/_pyrepl/__main__.py b/PythonLib/full/_pyrepl/__main__.py
new file mode 100644
index 000000000..9c66812e1
--- /dev/null
+++ b/PythonLib/full/_pyrepl/__main__.py
@@ -0,0 +1,10 @@
+# Important: don't add things to this module, as they will end up in the REPL's
+# default globals. Use _pyrepl.main instead.
+
+# Avoid caching this file by linecache and incorrectly report tracebacks.
+# See https://github.com/python/cpython/issues/129098.
+__spec__ = __loader__ = None
+
+if __name__ == "__main__":
+ from .main import interactive_console as __pyrepl_interactive_console
+ __pyrepl_interactive_console()
diff --git a/PythonLib/full/_pyrepl/_module_completer.py b/PythonLib/full/_pyrepl/_module_completer.py
new file mode 100644
index 000000000..2098d0a54
--- /dev/null
+++ b/PythonLib/full/_pyrepl/_module_completer.py
@@ -0,0 +1,426 @@
+from __future__ import annotations
+
+import importlib
+import os
+import pkgutil
+import sys
+import token
+import tokenize
+from importlib.machinery import FileFinder
+from io import StringIO
+from contextlib import contextmanager
+from dataclasses import dataclass
+from itertools import chain
+from tokenize import TokenInfo
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+ from typing import Any, Iterable, Iterator, Mapping
+
+
+HARDCODED_SUBMODULES = {
+ # Standard library submodules that are not detected by pkgutil.iter_modules
+ # but can be imported, so should be proposed in completion
+ "collections": ["abc"],
+ "os": ["path"],
+ "xml.parsers.expat": ["errors", "model"],
+}
+
+
+def make_default_module_completer() -> ModuleCompleter:
+ # Inside pyrepl, __package__ is set to None by default
+ return ModuleCompleter(namespace={'__package__': None})
+
+
+class ModuleCompleter:
+ """A completer for Python import statements.
+
+ Examples:
+ - import
+ - import foo
+ - import foo.
+ - import foo as bar, baz
+
+ - from
+ - from foo
+ - from foo import
+ - from foo import bar
+ - from foo import (bar as baz, qux
+ """
+
+ def __init__(self, namespace: Mapping[str, Any] | None = None) -> None:
+ self.namespace = namespace or {}
+ self._global_cache: list[pkgutil.ModuleInfo] = []
+ self._curr_sys_path: list[str] = sys.path[:]
+ self._stdlib_path = os.path.dirname(importlib.__path__[0])
+
+ def get_completions(self, line: str) -> list[str] | None:
+ """Return the next possible import completions for 'line'."""
+ result = ImportParser(line).parse()
+ if not result:
+ return None
+ try:
+ return self.complete(*result)
+ except Exception:
+ # Some unexpected error occurred, make it look like
+ # no completions are available
+ return []
+
+ def complete(self, from_name: str | None, name: str | None) -> list[str]:
+ if from_name is None:
+ # import x.y.z
+ assert name is not None
+ path, prefix = self.get_path_and_prefix(name)
+ modules = self.find_modules(path, prefix)
+ return [self.format_completion(path, module) for module in modules]
+
+ if name is None:
+ # from x.y.z
+ path, prefix = self.get_path_and_prefix(from_name)
+ modules = self.find_modules(path, prefix)
+ return [self.format_completion(path, module) for module in modules]
+
+ # from x.y import z
+ return self.find_modules(from_name, name)
+
+ def find_modules(self, path: str, prefix: str) -> list[str]:
+ """Find all modules under 'path' that start with 'prefix'."""
+ modules = self._find_modules(path, prefix)
+ # Filter out invalid module names
+ # (for example those containing dashes that cannot be imported with 'import')
+ return [mod for mod in modules if mod.isidentifier()]
+
+ def _find_modules(self, path: str, prefix: str) -> list[str]:
+ if not path:
+ # Top-level import (e.g. `import foo`` or `from foo`)`
+ builtin_modules = [name for name in sys.builtin_module_names
+ if self.is_suggestion_match(name, prefix)]
+ third_party_modules = [module.name for module in self.global_cache
+ if self.is_suggestion_match(module.name, prefix)]
+ return sorted(builtin_modules + third_party_modules)
+
+ if path.startswith('.'):
+ # Convert relative path to absolute path
+ package = self.namespace.get('__package__', '')
+ path = self.resolve_relative_name(path, package) # type: ignore[assignment]
+ if path is None:
+ return []
+
+ modules: Iterable[pkgutil.ModuleInfo] = self.global_cache
+ imported_module = sys.modules.get(path.split('.')[0])
+ if imported_module:
+ # Filter modules to those who name and specs match the
+ # imported module to avoid invalid suggestions
+ spec = imported_module.__spec__
+ if spec:
+ modules = [mod for mod in modules
+ if mod.name == spec.name
+ and mod.module_finder.find_spec(mod.name, None) == spec]
+ else:
+ modules = []
+
+ is_stdlib_import: bool | None = None
+ for segment in path.split('.'):
+ modules = [mod_info for mod_info in modules
+ if mod_info.ispkg and mod_info.name == segment]
+ if is_stdlib_import is None:
+ # Top-level import decide if we import from stdlib or not
+ is_stdlib_import = all(
+ self._is_stdlib_module(mod_info) for mod_info in modules
+ )
+ modules = self.iter_submodules(modules)
+
+ module_names = [module.name for module in modules]
+ if is_stdlib_import:
+ module_names.extend(HARDCODED_SUBMODULES.get(path, ()))
+ return [module_name for module_name in module_names
+ if self.is_suggestion_match(module_name, prefix)]
+
+ def _is_stdlib_module(self, module_info: pkgutil.ModuleInfo) -> bool:
+ return (isinstance(module_info.module_finder, FileFinder)
+ and module_info.module_finder.path == self._stdlib_path)
+
+ def is_suggestion_match(self, module_name: str, prefix: str) -> bool:
+ if prefix:
+ return module_name.startswith(prefix)
+ # For consistency with attribute completion, which
+ # does not suggest private attributes unless requested.
+ return not module_name.startswith("_")
+
+ def iter_submodules(self, parent_modules: list[pkgutil.ModuleInfo]) -> Iterator[pkgutil.ModuleInfo]:
+ """Iterate over all submodules of the given parent modules."""
+ specs = [info.module_finder.find_spec(info.name, None)
+ for info in parent_modules if info.ispkg]
+ search_locations = set(chain.from_iterable(
+ getattr(spec, 'submodule_search_locations', [])
+ for spec in specs if spec
+ ))
+ return pkgutil.iter_modules(search_locations)
+
+ def get_path_and_prefix(self, dotted_name: str) -> tuple[str, str]:
+ """
+ Split a dotted name into an import path and a
+ final prefix that is to be completed.
+
+ Examples:
+ 'foo.bar' -> 'foo', 'bar'
+ 'foo.' -> 'foo', ''
+ '.foo' -> '.', 'foo'
+ """
+ if '.' not in dotted_name:
+ return '', dotted_name
+ if dotted_name.startswith('.'):
+ stripped = dotted_name.lstrip('.')
+ dots = '.' * (len(dotted_name) - len(stripped))
+ if '.' not in stripped:
+ return dots, stripped
+ path, prefix = stripped.rsplit('.', 1)
+ return dots + path, prefix
+ path, prefix = dotted_name.rsplit('.', 1)
+ return path, prefix
+
+ def format_completion(self, path: str, module: str) -> str:
+ if path == '' or path.endswith('.'):
+ return f'{path}{module}'
+ return f'{path}.{module}'
+
+ def resolve_relative_name(self, name: str, package: str) -> str | None:
+ """Resolve a relative module name to an absolute name.
+
+ Example: resolve_relative_name('.foo', 'bar') -> 'bar.foo'
+ """
+ # taken from importlib._bootstrap
+ level = 0
+ for character in name:
+ if character != '.':
+ break
+ level += 1
+ bits = package.rsplit('.', level - 1)
+ if len(bits) < level:
+ return None
+ base = bits[0]
+ name = name[level:]
+ return f'{base}.{name}' if name else base
+
+ @property
+ def global_cache(self) -> list[pkgutil.ModuleInfo]:
+ """Global module cache"""
+ if not self._global_cache or self._curr_sys_path != sys.path:
+ self._curr_sys_path = sys.path[:]
+ self._global_cache = list(pkgutil.iter_modules())
+ return self._global_cache
+
+
+class ImportParser:
+ """
+ Parses incomplete import statements that are
+ suitable for autocomplete suggestions.
+
+ Examples:
+ - import foo -> Result(from_name=None, name='foo')
+ - import foo. -> Result(from_name=None, name='foo.')
+ - from foo -> Result(from_name='foo', name=None)
+ - from foo import bar -> Result(from_name='foo', name='bar')
+ - from .foo import ( -> Result(from_name='.foo', name='')
+
+ Note that the parser works in reverse order, starting from the
+ last token in the input string. This makes the parser more robust
+ when parsing multiple statements.
+ """
+ _ignored_tokens = {
+ token.INDENT, token.DEDENT, token.COMMENT,
+ token.NL, token.NEWLINE, token.ENDMARKER
+ }
+ _keywords = {'import', 'from', 'as'}
+
+ def __init__(self, code: str) -> None:
+ self.code = code
+ tokens = []
+ try:
+ for t in tokenize.generate_tokens(StringIO(code).readline):
+ if t.type not in self._ignored_tokens:
+ tokens.append(t)
+ except tokenize.TokenError as e:
+ if 'unexpected EOF' not in str(e):
+ # unexpected EOF is fine, since we're parsing an
+ # incomplete statement, but other errors are not
+ # because we may not have all the tokens so it's
+ # safer to bail out
+ tokens = []
+ except SyntaxError:
+ tokens = []
+ self.tokens = TokenQueue(tokens[::-1])
+
+ def parse(self) -> tuple[str | None, str | None] | None:
+ if not (res := self._parse()):
+ return None
+ return res.from_name, res.name
+
+ def _parse(self) -> Result | None:
+ with self.tokens.save_state():
+ return self.parse_from_import()
+ with self.tokens.save_state():
+ return self.parse_import()
+
+ def parse_import(self) -> Result:
+ if self.code.rstrip().endswith('import') and self.code.endswith(' '):
+ return Result(name='')
+ if self.tokens.peek_string(','):
+ name = ''
+ else:
+ if self.code.endswith(' '):
+ raise ParseError('parse_import')
+ name = self.parse_dotted_name()
+ if name.startswith('.'):
+ raise ParseError('parse_import')
+ while self.tokens.peek_string(','):
+ self.tokens.pop()
+ self.parse_dotted_as_name()
+ if self.tokens.peek_string('import'):
+ return Result(name=name)
+ raise ParseError('parse_import')
+
+ def parse_from_import(self) -> Result:
+ stripped = self.code.rstrip()
+ if stripped.endswith('import') and self.code.endswith(' '):
+ return Result(from_name=self.parse_empty_from_import(), name='')
+ if stripped.endswith('from') and self.code.endswith(' '):
+ return Result(from_name='')
+ if self.tokens.peek_string('(') or self.tokens.peek_string(','):
+ return Result(from_name=self.parse_empty_from_import(), name='')
+ if self.code.endswith(' '):
+ raise ParseError('parse_from_import')
+ name = self.parse_dotted_name()
+ if '.' in name:
+ self.tokens.pop_string('from')
+ return Result(from_name=name)
+ if self.tokens.peek_string('from'):
+ return Result(from_name=name)
+ from_name = self.parse_empty_from_import()
+ return Result(from_name=from_name, name=name)
+
+ def parse_empty_from_import(self) -> str:
+ if self.tokens.peek_string(','):
+ self.tokens.pop()
+ self.parse_as_names()
+ if self.tokens.peek_string('('):
+ self.tokens.pop()
+ self.tokens.pop_string('import')
+ return self.parse_from()
+
+ def parse_from(self) -> str:
+ from_name = self.parse_dotted_name()
+ self.tokens.pop_string('from')
+ return from_name
+
+ def parse_dotted_as_name(self) -> str:
+ self.tokens.pop_name()
+ if self.tokens.peek_string('as'):
+ self.tokens.pop()
+ with self.tokens.save_state():
+ return self.parse_dotted_name()
+
+ def parse_dotted_name(self) -> str:
+ name = []
+ if self.tokens.peek_string('.'):
+ name.append('.')
+ self.tokens.pop()
+ if (self.tokens.peek_name()
+ and (tok := self.tokens.peek())
+ and tok.string not in self._keywords):
+ name.append(self.tokens.pop_name())
+ if not name:
+ raise ParseError('parse_dotted_name')
+ while self.tokens.peek_string('.'):
+ name.append('.')
+ self.tokens.pop()
+ if (self.tokens.peek_name()
+ and (tok := self.tokens.peek())
+ and tok.string not in self._keywords):
+ name.append(self.tokens.pop_name())
+ else:
+ break
+
+ while self.tokens.peek_string('.'):
+ name.append('.')
+ self.tokens.pop()
+ return ''.join(name[::-1])
+
+ def parse_as_names(self) -> None:
+ self.parse_as_name()
+ while self.tokens.peek_string(','):
+ self.tokens.pop()
+ self.parse_as_name()
+
+ def parse_as_name(self) -> None:
+ self.tokens.pop_name()
+ if self.tokens.peek_string('as'):
+ self.tokens.pop()
+ self.tokens.pop_name()
+
+
+class ParseError(Exception):
+ pass
+
+
+@dataclass(frozen=True)
+class Result:
+ from_name: str | None = None
+ name: str | None = None
+
+
+class TokenQueue:
+ """Provides helper functions for working with a sequence of tokens."""
+
+ def __init__(self, tokens: list[TokenInfo]) -> None:
+ self.tokens: list[TokenInfo] = tokens
+ self.index: int = 0
+ self.stack: list[int] = []
+
+ @contextmanager
+ def save_state(self) -> Any:
+ try:
+ self.stack.append(self.index)
+ yield
+ except ParseError:
+ self.index = self.stack.pop()
+ else:
+ self.stack.pop()
+
+ def __bool__(self) -> bool:
+ return self.index < len(self.tokens)
+
+ def peek(self) -> TokenInfo | None:
+ if not self:
+ return None
+ return self.tokens[self.index]
+
+ def peek_name(self) -> bool:
+ if not (tok := self.peek()):
+ return False
+ return tok.type == token.NAME
+
+ def pop_name(self) -> str:
+ tok = self.pop()
+ if tok.type != token.NAME:
+ raise ParseError('pop_name')
+ return tok.string
+
+ def peek_string(self, string: str) -> bool:
+ if not (tok := self.peek()):
+ return False
+ return tok.string == string
+
+ def pop_string(self, string: str) -> str:
+ tok = self.pop()
+ if tok.string != string:
+ raise ParseError('pop_string')
+ return tok.string
+
+ def pop(self) -> TokenInfo:
+ if not self:
+ raise ParseError('pop')
+ tok = self.tokens[self.index]
+ self.index += 1
+ return tok
diff --git a/PythonLib/full/_pyrepl/_threading_handler.py b/PythonLib/full/_pyrepl/_threading_handler.py
new file mode 100644
index 000000000..82f5e8650
--- /dev/null
+++ b/PythonLib/full/_pyrepl/_threading_handler.py
@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+import traceback
+
+
+TYPE_CHECKING = False
+if TYPE_CHECKING:
+ from threading import Thread
+ from types import TracebackType
+ from typing import Protocol
+
+ class ExceptHookArgs(Protocol):
+ @property
+ def exc_type(self) -> type[BaseException]: ...
+ @property
+ def exc_value(self) -> BaseException | None: ...
+ @property
+ def exc_traceback(self) -> TracebackType | None: ...
+ @property
+ def thread(self) -> Thread | None: ...
+
+ class ShowExceptions(Protocol):
+ def __call__(self) -> int: ...
+ def add(self, s: str) -> None: ...
+
+ from .reader import Reader
+
+
+def install_threading_hook(reader: Reader) -> None:
+ import threading
+
+ @dataclass
+ class ExceptHookHandler:
+ lock: threading.Lock = field(default_factory=threading.Lock)
+ messages: list[str] = field(default_factory=list)
+
+ def show(self) -> int:
+ count = 0
+ with self.lock:
+ if not self.messages:
+ return 0
+ reader.restore()
+ for tb in self.messages:
+ count += 1
+ if tb:
+ print(tb)
+ self.messages.clear()
+ reader.scheduled_commands.append("ctrl-c")
+ reader.prepare()
+ return count
+
+ def add(self, s: str) -> None:
+ with self.lock:
+ self.messages.append(s)
+
+ def exception(self, args: ExceptHookArgs) -> None:
+ lines = traceback.format_exception(
+ args.exc_type,
+ args.exc_value,
+ args.exc_traceback,
+ colorize=reader.can_colorize,
+ ) # type: ignore[call-overload]
+ pre = f"\nException in {args.thread.name}:\n" if args.thread else "\n"
+ tb = pre + "".join(lines)
+ self.add(tb)
+
+ def __call__(self) -> int:
+ return self.show()
+
+
+ handler = ExceptHookHandler()
+ reader.threading_hook = handler
+ threading.excepthook = handler.exception
diff --git a/PythonLib/full/_pyrepl/base_eventqueue.py b/PythonLib/full/_pyrepl/base_eventqueue.py
new file mode 100644
index 000000000..0589a0f43
--- /dev/null
+++ b/PythonLib/full/_pyrepl/base_eventqueue.py
@@ -0,0 +1,110 @@
+# Copyright 2000-2008 Michael Hudson-Doyle
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+OS-independent base for an event and VT sequence scanner
+
+See unix_eventqueue and windows_eventqueue for subclasses.
+"""
+
+from collections import deque
+
+from . import keymap
+from .console import Event
+from .trace import trace
+
+class BaseEventQueue:
+ def __init__(self, encoding: str, keymap_dict: dict[bytes, str]) -> None:
+ self.compiled_keymap = keymap.compile_keymap(keymap_dict)
+ self.keymap = self.compiled_keymap
+ trace("keymap {k!r}", k=self.keymap)
+ self.encoding = encoding
+ self.events: deque[Event] = deque()
+ self.buf = bytearray()
+
+ def get(self) -> Event | None:
+ """
+ Retrieves the next event from the queue.
+ """
+ if self.events:
+ return self.events.popleft()
+ else:
+ return None
+
+ def empty(self) -> bool:
+ """
+ Checks if the queue is empty.
+ """
+ return not self.events
+
+ def flush_buf(self) -> bytearray:
+ """
+ Flushes the buffer and returns its contents.
+ """
+ old = self.buf
+ self.buf = bytearray()
+ return old
+
+ def insert(self, event: Event) -> None:
+ """
+ Inserts an event into the queue.
+ """
+ trace('added event {event}', event=event)
+ self.events.append(event)
+
+ def push(self, char: int | bytes) -> None:
+ """
+ Processes a character by updating the buffer and handling special key mappings.
+ """
+ assert isinstance(char, (int, bytes))
+ ord_char = char if isinstance(char, int) else ord(char)
+ char = ord_char.to_bytes()
+ self.buf.append(ord_char)
+
+ if char in self.keymap:
+ if self.keymap is self.compiled_keymap:
+ # sanity check, buffer is empty when a special key comes
+ assert len(self.buf) == 1
+ k = self.keymap[char]
+ trace('found map {k!r}', k=k)
+ if isinstance(k, dict):
+ self.keymap = k
+ else:
+ self.insert(Event('key', k, bytes(self.flush_buf())))
+ self.keymap = self.compiled_keymap
+
+ elif self.buf and self.buf[0] == 27: # escape
+ # escape sequence not recognized by our keymap: propagate it
+ # outside so that i can be recognized as an M-... key (see also
+ # the docstring in keymap.py
+ trace('unrecognized escape sequence, propagating...')
+ self.keymap = self.compiled_keymap
+ self.insert(Event('key', '\033', b'\033'))
+ for _c in self.flush_buf()[1:]:
+ self.push(_c)
+
+ else:
+ try:
+ decoded = bytes(self.buf).decode(self.encoding)
+ except UnicodeError:
+ return
+ else:
+ self.insert(Event('key', decoded, bytes(self.flush_buf())))
+ self.keymap = self.compiled_keymap
diff --git a/PythonLib/full/_pyrepl/commands.py b/PythonLib/full/_pyrepl/commands.py
new file mode 100644
index 000000000..10127e588
--- /dev/null
+++ b/PythonLib/full/_pyrepl/commands.py
@@ -0,0 +1,505 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Antonio Cuni
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+import os
+import time
+
+# Categories of actions:
+# killing
+# yanking
+# motion
+# editing
+# history
+# finishing
+# [completion]
+
+from .trace import trace
+
+# types
+if False:
+ from .historical_reader import HistoricalReader
+
+
+class Command:
+ finish: bool = False
+ kills_digit_arg: bool = True
+
+ def __init__(
+ self, reader: HistoricalReader, event_name: str, event: list[str]
+ ) -> None:
+ # Reader should really be "any reader" but there's too much usage of
+ # HistoricalReader methods and fields in the code below for us to
+ # refactor at the moment.
+
+ self.reader = reader
+ self.event = event
+ self.event_name = event_name
+
+ def do(self) -> None:
+ pass
+
+
+class KillCommand(Command):
+ def kill_range(self, start: int, end: int) -> None:
+ if start == end:
+ return
+ r = self.reader
+ b = r.buffer
+ text = b[start:end]
+ del b[start:end]
+ if is_kill(r.last_command):
+ if start < r.pos:
+ r.kill_ring[-1] = text + r.kill_ring[-1]
+ else:
+ r.kill_ring[-1] = r.kill_ring[-1] + text
+ else:
+ r.kill_ring.append(text)
+ r.pos = start
+ r.dirty = True
+
+
+class YankCommand(Command):
+ pass
+
+
+class MotionCommand(Command):
+ pass
+
+
+class EditCommand(Command):
+ pass
+
+
+class FinishCommand(Command):
+ finish = True
+ pass
+
+
+def is_kill(command: type[Command] | None) -> bool:
+ return command is not None and issubclass(command, KillCommand)
+
+
+def is_yank(command: type[Command] | None) -> bool:
+ return command is not None and issubclass(command, YankCommand)
+
+
+# etc
+
+
+class digit_arg(Command):
+ kills_digit_arg = False
+
+ def do(self) -> None:
+ r = self.reader
+ c = self.event[-1]
+ if c == "-":
+ if r.arg is not None:
+ r.arg = -r.arg
+ else:
+ r.arg = -1
+ else:
+ d = int(c)
+ if r.arg is None:
+ r.arg = d
+ else:
+ if r.arg < 0:
+ r.arg = 10 * r.arg - d
+ else:
+ r.arg = 10 * r.arg + d
+ r.dirty = True
+
+
+class clear_screen(Command):
+ def do(self) -> None:
+ r = self.reader
+ r.console.clear()
+ r.dirty = True
+
+
+class refresh(Command):
+ def do(self) -> None:
+ self.reader.dirty = True
+
+
+class repaint(Command):
+ def do(self) -> None:
+ self.reader.dirty = True
+ self.reader.console.repaint()
+
+
+class kill_line(KillCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ eol = r.eol()
+ for c in b[r.pos : eol]:
+ if not c.isspace():
+ self.kill_range(r.pos, eol)
+ return
+ else:
+ self.kill_range(r.pos, eol + 1)
+
+
+class unix_line_discard(KillCommand):
+ def do(self) -> None:
+ r = self.reader
+ self.kill_range(r.bol(), r.pos)
+
+
+class unix_word_rubout(KillCommand):
+ def do(self) -> None:
+ r = self.reader
+ for i in range(r.get_arg()):
+ self.kill_range(r.bow(), r.pos)
+
+
+class kill_word(KillCommand):
+ def do(self) -> None:
+ r = self.reader
+ for i in range(r.get_arg()):
+ self.kill_range(r.pos, r.eow())
+
+
+class backward_kill_word(KillCommand):
+ def do(self) -> None:
+ r = self.reader
+ for i in range(r.get_arg()):
+ self.kill_range(r.bow(), r.pos)
+
+
+class yank(YankCommand):
+ def do(self) -> None:
+ r = self.reader
+ if not r.kill_ring:
+ r.error("nothing to yank")
+ return
+ r.insert(r.kill_ring[-1])
+
+
+class yank_pop(YankCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ if not r.kill_ring:
+ r.error("nothing to yank")
+ return
+ if not is_yank(r.last_command):
+ r.error("previous command was not a yank")
+ return
+ repl = len(r.kill_ring[-1])
+ r.kill_ring.insert(0, r.kill_ring.pop())
+ t = r.kill_ring[-1]
+ b[r.pos - repl : r.pos] = t
+ r.pos = r.pos - repl + len(t)
+ r.dirty = True
+
+
+class interrupt(FinishCommand):
+ def do(self) -> None:
+ import signal
+
+ self.reader.console.finish()
+ self.reader.finish()
+ os.kill(os.getpid(), signal.SIGINT)
+
+
+class ctrl_c(Command):
+ def do(self) -> None:
+ self.reader.console.finish()
+ self.reader.finish()
+ raise KeyboardInterrupt
+
+
+class suspend(Command):
+ def do(self) -> None:
+ import signal
+
+ r = self.reader
+ p = r.pos
+ r.console.finish()
+ os.kill(os.getpid(), signal.SIGSTOP)
+ ## this should probably be done
+ ## in a handler for SIGCONT?
+ r.console.prepare()
+ r.pos = p
+ # r.posxy = 0, 0 # XXX this is invalid
+ r.dirty = True
+ r.console.screen = []
+
+
+class up(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ for _ in range(r.get_arg()):
+ x, y = r.pos2xy()
+ new_y = y - 1
+
+ if r.bol() == 0:
+ if r.historyi > 0:
+ r.select_item(r.historyi - 1)
+ return
+ r.pos = 0
+ r.error("start of buffer")
+ return
+
+ if (
+ x
+ > (
+ new_x := r.max_column(new_y)
+ ) # we're past the end of the previous line
+ or x == r.max_column(y)
+ and any(
+ not i.isspace() for i in r.buffer[r.bol() :]
+ ) # move between eols
+ ):
+ x = new_x
+
+ r.setpos_from_xy(x, new_y)
+
+
+class down(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ for _ in range(r.get_arg()):
+ x, y = r.pos2xy()
+ new_y = y + 1
+
+ if r.eol() == len(b):
+ if r.historyi < len(r.history):
+ r.select_item(r.historyi + 1)
+ r.pos = r.eol(0)
+ return
+ r.pos = len(b)
+ r.error("end of buffer")
+ return
+
+ if (
+ x
+ > (
+ new_x := r.max_column(new_y)
+ ) # we're past the end of the previous line
+ or x == r.max_column(y)
+ and any(
+ not i.isspace() for i in r.buffer[r.bol() :]
+ ) # move between eols
+ ):
+ x = new_x
+
+ r.setpos_from_xy(x, new_y)
+
+
+class left(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ for _ in range(r.get_arg()):
+ p = r.pos - 1
+ if p >= 0:
+ r.pos = p
+ else:
+ self.reader.error("start of buffer")
+
+
+class right(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ for _ in range(r.get_arg()):
+ p = r.pos + 1
+ if p <= len(b):
+ r.pos = p
+ else:
+ self.reader.error("end of buffer")
+
+
+class beginning_of_line(MotionCommand):
+ def do(self) -> None:
+ self.reader.pos = self.reader.bol()
+
+
+class end_of_line(MotionCommand):
+ def do(self) -> None:
+ self.reader.pos = self.reader.eol()
+
+
+class home(MotionCommand):
+ def do(self) -> None:
+ self.reader.pos = 0
+
+
+class end(MotionCommand):
+ def do(self) -> None:
+ self.reader.pos = len(self.reader.buffer)
+
+
+class forward_word(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ for i in range(r.get_arg()):
+ r.pos = r.eow()
+
+
+class backward_word(MotionCommand):
+ def do(self) -> None:
+ r = self.reader
+ for i in range(r.get_arg()):
+ r.pos = r.bow()
+
+
+class self_insert(EditCommand):
+ def do(self) -> None:
+ r = self.reader
+ text = self.event * r.get_arg()
+ r.insert(text)
+ if r.paste_mode:
+ data = ""
+ ev = r.console.getpending()
+ data += ev.data
+ if data:
+ r.insert(data)
+ r.last_refresh_cache.invalidated = True
+
+
+class insert_nl(EditCommand):
+ def do(self) -> None:
+ r = self.reader
+ r.insert("\n" * r.get_arg())
+
+
+class transpose_characters(EditCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ s = r.pos - 1
+ if s < 0:
+ r.error("cannot transpose at start of buffer")
+ else:
+ if s == len(b):
+ s -= 1
+ t = min(s + r.get_arg(), len(b) - 1)
+ c = b[s]
+ del b[s]
+ b.insert(t, c)
+ r.pos = t
+ r.dirty = True
+
+
+class backspace(EditCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ for i in range(r.get_arg()):
+ if r.pos > 0:
+ r.pos -= 1
+ del b[r.pos]
+ r.dirty = True
+ else:
+ self.reader.error("can't backspace at start")
+
+
+class delete(EditCommand):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ if self.event[-1] == "\004":
+ if b and b[-1].endswith("\n"):
+ self.finish = True
+ elif (
+ r.pos == 0
+ and len(b) == 0 # this is something of a hack
+ ):
+ r.update_screen()
+ r.console.finish()
+ raise EOFError
+
+ for i in range(r.get_arg()):
+ if r.pos != len(b):
+ del b[r.pos]
+ r.dirty = True
+ else:
+ self.reader.error("end of buffer")
+
+
+class accept(FinishCommand):
+ def do(self) -> None:
+ pass
+
+
+class help(Command):
+ def do(self) -> None:
+ import _sitebuiltins
+
+ with self.reader.suspend():
+ self.reader.msg = _sitebuiltins._Helper()() # type: ignore[assignment]
+
+
+class invalid_key(Command):
+ def do(self) -> None:
+ pending = self.reader.console.getpending()
+ s = "".join(self.event) + pending.data
+ self.reader.error("`%r' not bound" % s)
+
+
+class invalid_command(Command):
+ def do(self) -> None:
+ s = self.event_name
+ self.reader.error("command `%s' not known" % s)
+
+
+class show_history(Command):
+ def do(self) -> None:
+ from .pager import get_pager
+ from site import gethistoryfile
+
+ history = os.linesep.join(self.reader.history[:])
+ self.reader.console.restore()
+ pager = get_pager()
+ pager(history, gethistoryfile())
+ self.reader.console.prepare()
+
+ # We need to copy over the state so that it's consistent between
+ # console and reader, and console does not overwrite/append stuff
+ self.reader.console.screen = self.reader.screen.copy()
+ self.reader.console.posxy = self.reader.cxy
+
+
+class paste_mode(Command):
+ def do(self) -> None:
+ self.reader.paste_mode = not self.reader.paste_mode
+ self.reader.dirty = True
+
+
+class perform_bracketed_paste(Command):
+ def do(self) -> None:
+ done = "\x1b[201~"
+ data = ""
+ start = time.time()
+ while done not in data:
+ ev = self.reader.console.getpending()
+ data += ev.data
+ trace(
+ "bracketed pasting of {l} chars done in {s:.2f}s",
+ l=len(data),
+ s=time.time() - start,
+ )
+ self.reader.insert(data.replace(done, ""))
+ self.reader.last_refresh_cache.invalidated = True
diff --git a/PythonLib/full/_pyrepl/completing_reader.py b/PythonLib/full/_pyrepl/completing_reader.py
new file mode 100644
index 000000000..9d2d43be5
--- /dev/null
+++ b/PythonLib/full/_pyrepl/completing_reader.py
@@ -0,0 +1,299 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Antonio Cuni
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+import re
+from . import commands, console, reader
+from .reader import Reader
+
+
+# types
+Command = commands.Command
+if False:
+ from .types import KeySpec, CommandName
+
+
+def prefix(wordlist: list[str], j: int = 0) -> str:
+ d = {}
+ i = j
+ try:
+ while 1:
+ for word in wordlist:
+ d[word[i]] = 1
+ if len(d) > 1:
+ return wordlist[0][j:i]
+ i += 1
+ d = {}
+ except IndexError:
+ return wordlist[0][j:i]
+ return ""
+
+
+STRIPCOLOR_REGEX = re.compile(r"\x1B\[([0-9]{1,3}(;[0-9]{1,2})?)?[m|K]")
+
+def stripcolor(s: str) -> str:
+ return STRIPCOLOR_REGEX.sub('', s)
+
+
+def real_len(s: str) -> int:
+ return len(stripcolor(s))
+
+
+def left_align(s: str, maxlen: int) -> str:
+ stripped = stripcolor(s)
+ if len(stripped) > maxlen:
+ # too bad, we remove the color
+ return stripped[:maxlen]
+ padding = maxlen - len(stripped)
+ return s + ' '*padding
+
+
+def build_menu(
+ cons: console.Console,
+ wordlist: list[str],
+ start: int,
+ use_brackets: bool,
+ sort_in_column: bool,
+) -> tuple[list[str], int]:
+ if use_brackets:
+ item = "[ %s ]"
+ padding = 4
+ else:
+ item = "%s "
+ padding = 2
+ maxlen = min(max(map(real_len, wordlist)), cons.width - padding)
+ cols = int(cons.width / (maxlen + padding))
+ rows = int((len(wordlist) - 1)/cols + 1)
+
+ if sort_in_column:
+ # sort_in_column=False (default) sort_in_column=True
+ # A B C A D G
+ # D E F B E
+ # G C F
+ #
+ # "fill" the table with empty words, so we always have the same amount
+ # of rows for each column
+ missing = cols*rows - len(wordlist)
+ wordlist = wordlist + ['']*missing
+ indexes = [(i % cols) * rows + i // cols for i in range(len(wordlist))]
+ wordlist = [wordlist[i] for i in indexes]
+ menu = []
+ i = start
+ for r in range(rows):
+ row = []
+ for col in range(cols):
+ row.append(item % left_align(wordlist[i], maxlen))
+ i += 1
+ if i >= len(wordlist):
+ break
+ menu.append(''.join(row))
+ if i >= len(wordlist):
+ i = 0
+ break
+ if r + 5 > cons.height:
+ menu.append(" %d more... " % (len(wordlist) - i))
+ break
+ return menu, i
+
+# this gets somewhat user interface-y, and as a result the logic gets
+# very convoluted.
+#
+# To summarise the summary of the summary:- people are a problem.
+# -- The Hitch-Hikers Guide to the Galaxy, Episode 12
+
+#### Desired behaviour of the completions commands.
+# the considerations are:
+# (1) how many completions are possible
+# (2) whether the last command was a completion
+# (3) if we can assume that the completer is going to return the same set of
+# completions: this is controlled by the ``assume_immutable_completions``
+# variable on the reader, which is True by default to match the historical
+# behaviour of pyrepl, but e.g. False in the ReadlineAlikeReader to match
+# more closely readline's semantics (this is needed e.g. by
+# fancycompleter)
+#
+# if there's no possible completion, beep at the user and point this out.
+# this is easy.
+#
+# if there's only one possible completion, stick it in. if the last thing
+# user did was a completion, point out that he isn't getting anywhere, but
+# only if the ``assume_immutable_completions`` is True.
+#
+# now it gets complicated.
+#
+# for the first press of a completion key:
+# if there's a common prefix, stick it in.
+
+# irrespective of whether anything got stuck in, if the word is now
+# complete, show the "complete but not unique" message
+
+# if there's no common prefix and if the word is not now complete,
+# beep.
+
+# common prefix -> yes no
+# word complete \/
+# yes "cbnu" "cbnu"
+# no - beep
+
+# for the second bang on the completion key
+# there will necessarily be no common prefix
+# show a menu of the choices.
+
+# for subsequent bangs, rotate the menu around (if there are sufficient
+# choices).
+
+
+class complete(commands.Command):
+ def do(self) -> None:
+ r: CompletingReader
+ r = self.reader # type: ignore[assignment]
+ last_is_completer = r.last_command_is(self.__class__)
+ immutable_completions = r.assume_immutable_completions
+ completions_unchangable = last_is_completer and immutable_completions
+ stem = r.get_stem()
+ if not completions_unchangable:
+ r.cmpltn_menu_choices = r.get_completions(stem)
+
+ completions = r.cmpltn_menu_choices
+ if not completions:
+ r.error("no matches")
+ elif len(completions) == 1:
+ if completions_unchangable and len(completions[0]) == len(stem):
+ r.msg = "[ sole completion ]"
+ r.dirty = True
+ r.insert(completions[0][len(stem):])
+ else:
+ p = prefix(completions, len(stem))
+ if p:
+ r.insert(p)
+ if last_is_completer:
+ r.cmpltn_menu_visible = True
+ r.cmpltn_message_visible = False
+ r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+ r.console, completions, r.cmpltn_menu_end,
+ r.use_brackets, r.sort_in_column)
+ r.dirty = True
+ elif not r.cmpltn_menu_visible:
+ r.cmpltn_message_visible = True
+ if stem + p in completions:
+ r.msg = "[ complete but not unique ]"
+ r.dirty = True
+ else:
+ r.msg = "[ not unique ]"
+ r.dirty = True
+
+
+class self_insert(commands.self_insert):
+ def do(self) -> None:
+ r: CompletingReader
+ r = self.reader # type: ignore[assignment]
+
+ commands.self_insert.do(self)
+ if r.cmpltn_menu_visible:
+ stem = r.get_stem()
+ if len(stem) < 1:
+ r.cmpltn_reset()
+ else:
+ completions = [w for w in r.cmpltn_menu_choices
+ if w.startswith(stem)]
+ if completions:
+ r.cmpltn_menu, r.cmpltn_menu_end = build_menu(
+ r.console, completions, 0,
+ r.use_brackets, r.sort_in_column)
+ else:
+ r.cmpltn_reset()
+
+
+@dataclass
+class CompletingReader(Reader):
+ """Adds completion support"""
+
+ ### Class variables
+ # see the comment for the complete command
+ assume_immutable_completions = True
+ use_brackets = True # display completions inside []
+ sort_in_column = False
+
+ ### Instance variables
+ cmpltn_menu: list[str] = field(init=False)
+ cmpltn_menu_visible: bool = field(init=False)
+ cmpltn_message_visible: bool = field(init=False)
+ cmpltn_menu_end: int = field(init=False)
+ cmpltn_menu_choices: list[str] = field(init=False)
+
+ def __post_init__(self) -> None:
+ super().__post_init__()
+ self.cmpltn_reset()
+ for c in (complete, self_insert):
+ self.commands[c.__name__] = c
+ self.commands[c.__name__.replace('_', '-')] = c
+
+ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+ return super().collect_keymap() + (
+ (r'\t', 'complete'),)
+
+ def after_command(self, cmd: Command) -> None:
+ super().after_command(cmd)
+ if not isinstance(cmd, (complete, self_insert)):
+ self.cmpltn_reset()
+
+ def calc_screen(self) -> list[str]:
+ screen = super().calc_screen()
+ if self.cmpltn_menu_visible:
+ # We display the completions menu below the current prompt
+ ly = self.lxy[1] + 1
+ screen[ly:ly] = self.cmpltn_menu
+ # If we're not in the middle of multiline edit, don't append to screeninfo
+ # since that screws up the position calculation in pos2xy function.
+ # This is a hack to prevent the cursor jumping
+ # into the completions menu when pressing left or down arrow.
+ if self.pos != len(self.buffer):
+ self.screeninfo[ly:ly] = [(0, [])]*len(self.cmpltn_menu)
+ return screen
+
+ def finish(self) -> None:
+ super().finish()
+ self.cmpltn_reset()
+
+ def cmpltn_reset(self) -> None:
+ self.cmpltn_menu = []
+ self.cmpltn_menu_visible = False
+ self.cmpltn_message_visible = False
+ self.cmpltn_menu_end = 0
+ self.cmpltn_menu_choices = []
+
+ def get_stem(self) -> str:
+ st = self.syntax_table
+ SW = reader.SYNTAX_WORD
+ b = self.buffer
+ p = self.pos - 1
+ while p >= 0 and st.get(b[p], SW) == SW:
+ p -= 1
+ return ''.join(b[p+1:self.pos])
+
+ def get_completions(self, stem: str) -> list[str]:
+ return []
+
+ def get_line(self) -> str:
+ """Return the current line until the cursor position."""
+ return ''.join(self.buffer[:self.pos])
diff --git a/PythonLib/full/_pyrepl/console.py b/PythonLib/full/_pyrepl/console.py
new file mode 100644
index 000000000..8956fb124
--- /dev/null
+++ b/PythonLib/full/_pyrepl/console.py
@@ -0,0 +1,229 @@
+# Copyright 2000-2004 Michael Hudson-Doyle
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import _colorize
+
+from abc import ABC, abstractmethod
+import ast
+import code
+import linecache
+from dataclasses import dataclass, field
+import os.path
+import sys
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+ from typing import IO
+ from typing import Callable
+
+
+@dataclass
+class Event:
+ evt: str
+ data: str
+ raw: bytes = b""
+
+
+@dataclass
+class Console(ABC):
+ posxy: tuple[int, int]
+ screen: list[str] = field(default_factory=list)
+ height: int = 25
+ width: int = 80
+
+ def __init__(
+ self,
+ f_in: IO[bytes] | int = 0,
+ f_out: IO[bytes] | int = 1,
+ term: str = "",
+ encoding: str = "",
+ ):
+ self.encoding = encoding or sys.getdefaultencoding()
+
+ if isinstance(f_in, int):
+ self.input_fd = f_in
+ else:
+ self.input_fd = f_in.fileno()
+
+ if isinstance(f_out, int):
+ self.output_fd = f_out
+ else:
+ self.output_fd = f_out.fileno()
+
+ @abstractmethod
+ def refresh(self, screen: list[str], xy: tuple[int, int]) -> None: ...
+
+ @abstractmethod
+ def prepare(self) -> None: ...
+
+ @abstractmethod
+ def restore(self) -> None: ...
+
+ @abstractmethod
+ def move_cursor(self, x: int, y: int) -> None: ...
+
+ @abstractmethod
+ def set_cursor_vis(self, visible: bool) -> None: ...
+
+ @abstractmethod
+ def getheightwidth(self) -> tuple[int, int]:
+ """Return (height, width) where height and width are the height
+ and width of the terminal window in characters."""
+ ...
+
+ @abstractmethod
+ def get_event(self, block: bool = True) -> Event | None:
+ """Return an Event instance. Returns None if |block| is false
+ and there is no event pending, otherwise waits for the
+ completion of an event."""
+ ...
+
+ @abstractmethod
+ def push_char(self, char: int | bytes) -> None:
+ """
+ Push a character to the console event queue.
+ """
+ ...
+
+ @abstractmethod
+ def beep(self) -> None: ...
+
+ @abstractmethod
+ def clear(self) -> None:
+ """Wipe the screen"""
+ ...
+
+ @abstractmethod
+ def finish(self) -> None:
+ """Move the cursor to the end of the display and otherwise get
+ ready for end. XXX could be merged with restore? Hmm."""
+ ...
+
+ @abstractmethod
+ def flushoutput(self) -> None:
+ """Flush all output to the screen (assuming there's some
+ buffering going on somewhere)."""
+ ...
+
+ @abstractmethod
+ def forgetinput(self) -> None:
+ """Forget all pending, but not yet processed input."""
+ ...
+
+ @abstractmethod
+ def getpending(self) -> Event:
+ """Return the characters that have been typed but not yet
+ processed."""
+ ...
+
+ @abstractmethod
+ def wait(self, timeout: float | None) -> bool:
+ """Wait for an event. The return value is True if an event is
+ available, False if the timeout has been reached. If timeout is
+ None, wait forever. The timeout is in milliseconds."""
+ ...
+
+ @property
+ def input_hook(self) -> Callable[[], int] | None:
+ """Returns the current input hook."""
+ ...
+
+ @abstractmethod
+ def repaint(self) -> None: ...
+
+
+class InteractiveColoredConsole(code.InteractiveConsole):
+ STATEMENT_FAILED = object()
+
+ def __init__(
+ self,
+ locals: dict[str, object] | None = None,
+ filename: str = "",
+ *,
+ local_exit: bool = False,
+ ) -> None:
+ super().__init__(locals=locals, filename=filename, local_exit=local_exit)
+ self.can_colorize = _colorize.can_colorize()
+
+ def showsyntaxerror(self, filename=None, **kwargs):
+ super().showsyntaxerror(filename=filename, **kwargs)
+
+ def _excepthook(self, typ, value, tb):
+ import traceback
+ lines = traceback.format_exception(
+ typ, value, tb,
+ colorize=self.can_colorize,
+ limit=traceback.BUILTIN_EXCEPTION_LIMIT)
+ self.write(''.join(lines))
+
+ def runcode(self, code):
+ try:
+ exec(code, self.locals)
+ except SystemExit:
+ raise
+ except BaseException:
+ self.showtraceback()
+ return self.STATEMENT_FAILED
+ return None
+
+ def runsource(self, source, filename="", symbol="single"):
+ try:
+ tree = self.compile.compiler(
+ source,
+ filename,
+ "exec",
+ ast.PyCF_ONLY_AST,
+ incomplete_input=False,
+ )
+ except (SyntaxError, OverflowError, ValueError):
+ self.showsyntaxerror(filename, source=source)
+ return False
+ if tree.body:
+ *_, last_stmt = tree.body
+ for stmt in tree.body:
+ wrapper = ast.Interactive if stmt is last_stmt else ast.Module
+ the_symbol = symbol if stmt is last_stmt else "exec"
+ item = wrapper([stmt])
+ try:
+ code = self.compile.compiler(item, filename, the_symbol)
+ linecache._register_code(code, source, filename)
+ except SyntaxError as e:
+ if e.args[0] == "'await' outside function":
+ python = os.path.basename(sys.executable)
+ e.add_note(
+ f"Try the asyncio REPL ({python} -m asyncio) to use"
+ f" top-level 'await' and run background asyncio tasks."
+ )
+ self.showsyntaxerror(filename, source=source)
+ return False
+ except (OverflowError, ValueError):
+ self.showsyntaxerror(filename, source=source)
+ return False
+
+ if code is None:
+ return True
+
+ result = self.runcode(code)
+ if result is self.STATEMENT_FAILED:
+ break
+ return False
diff --git a/PythonLib/full/_pyrepl/fancy_termios.py b/PythonLib/full/_pyrepl/fancy_termios.py
new file mode 100644
index 000000000..8d5bd183f
--- /dev/null
+++ b/PythonLib/full/_pyrepl/fancy_termios.py
@@ -0,0 +1,82 @@
+# Copyright 2000-2004 Michael Hudson-Doyle
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import termios
+
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+ from typing import cast
+else:
+ cast = lambda typ, val: val
+
+
+class TermState:
+ def __init__(self, attrs: list[int | list[bytes]]) -> None:
+ self.iflag = cast(int, attrs[0])
+ self.oflag = cast(int, attrs[1])
+ self.cflag = cast(int, attrs[2])
+ self.lflag = cast(int, attrs[3])
+ self.ispeed = cast(int, attrs[4])
+ self.ospeed = cast(int, attrs[5])
+ self.cc = cast(list[bytes], attrs[6])
+
+ def as_list(self) -> list[int | list[bytes]]:
+ return [
+ self.iflag,
+ self.oflag,
+ self.cflag,
+ self.lflag,
+ self.ispeed,
+ self.ospeed,
+ # Always return a copy of the control characters list to ensure
+ # there are not any additional references to self.cc
+ self.cc[:],
+ ]
+
+ def copy(self) -> "TermState":
+ return self.__class__(self.as_list())
+
+
+def tcgetattr(fd: int) -> TermState:
+ return TermState(termios.tcgetattr(fd))
+
+
+def tcsetattr(fd: int, when: int, attrs: TermState) -> None:
+ termios.tcsetattr(fd, when, attrs.as_list())
+
+
+class Term(TermState):
+ TS__init__ = TermState.__init__
+
+ def __init__(self, fd: int = 0) -> None:
+ self.TS__init__(termios.tcgetattr(fd))
+ self.fd = fd
+ self.stack: list[list[int | list[bytes]]] = []
+
+ def save(self) -> None:
+ self.stack.append(self.as_list())
+
+ def set(self, when: int = termios.TCSANOW) -> None:
+ termios.tcsetattr(self.fd, when, self.as_list())
+
+ def restore(self) -> None:
+ self.TS__init__(self.stack.pop())
+ self.set()
diff --git a/PythonLib/full/_pyrepl/historical_reader.py b/PythonLib/full/_pyrepl/historical_reader.py
new file mode 100644
index 000000000..c4b95fa2e
--- /dev/null
+++ b/PythonLib/full/_pyrepl/historical_reader.py
@@ -0,0 +1,419 @@
+# Copyright 2000-2004 Michael Hudson-Doyle
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field
+
+from . import commands, input
+from .reader import Reader
+
+
+if False:
+ from .types import SimpleContextManager, KeySpec, CommandName
+
+
+isearch_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+ [("\\%03o" % c, "isearch-end") for c in range(256) if chr(c) != "\\"]
+ + [(c, "isearch-add-character") for c in map(chr, range(32, 127)) if c != "\\"]
+ + [
+ ("\\%03o" % c, "isearch-add-character")
+ for c in range(256)
+ if chr(c).isalpha() and chr(c) != "\\"
+ ]
+ + [
+ ("\\\\", "self-insert"),
+ (r"\C-r", "isearch-backwards"),
+ (r"\C-s", "isearch-forwards"),
+ (r"\C-c", "isearch-cancel"),
+ (r"\C-g", "isearch-cancel"),
+ (r"\", "isearch-backspace"),
+ ]
+)
+
+ISEARCH_DIRECTION_NONE = ""
+ISEARCH_DIRECTION_BACKWARDS = "r"
+ISEARCH_DIRECTION_FORWARDS = "f"
+
+
+class next_history(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ if r.historyi == len(r.history):
+ r.error("end of history list")
+ return
+ r.select_item(r.historyi + 1)
+
+
+class previous_history(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ if r.historyi == 0:
+ r.error("start of history list")
+ return
+ r.select_item(r.historyi - 1)
+
+
+class history_search_backward(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.search_next(forwards=False)
+
+
+class history_search_forward(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.search_next(forwards=True)
+
+
+class restore_history(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ if r.historyi != len(r.history):
+ if r.get_unicode() != r.history[r.historyi]:
+ r.buffer = list(r.history[r.historyi])
+ r.pos = len(r.buffer)
+ r.dirty = True
+
+
+class first_history(commands.Command):
+ def do(self) -> None:
+ self.reader.select_item(0)
+
+
+class last_history(commands.Command):
+ def do(self) -> None:
+ self.reader.select_item(len(self.reader.history))
+
+
+class operate_and_get_next(commands.FinishCommand):
+ def do(self) -> None:
+ self.reader.next_history = self.reader.historyi + 1
+
+
+class yank_arg(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ if r.last_command is self.__class__:
+ r.yank_arg_i += 1
+ else:
+ r.yank_arg_i = 0
+ if r.historyi < r.yank_arg_i:
+ r.error("beginning of history list")
+ return
+ a = r.get_arg(-1)
+ # XXX how to split?
+ words = r.get_item(r.historyi - r.yank_arg_i - 1).split()
+ if a < -len(words) or a >= len(words):
+ r.error("no such arg")
+ return
+ w = words[a]
+ b = r.buffer
+ if r.yank_arg_i > 0:
+ o = len(r.yank_arg_yanked)
+ else:
+ o = 0
+ b[r.pos - o : r.pos] = list(w)
+ r.yank_arg_yanked = w
+ r.pos += len(w) - o
+ r.dirty = True
+
+
+class forward_history_isearch(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+ r.isearch_start = r.historyi, r.pos
+ r.isearch_term = ""
+ r.dirty = True
+ r.push_input_trans(r.isearch_trans)
+
+
+class reverse_history_isearch(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+ r.dirty = True
+ r.isearch_term = ""
+ r.push_input_trans(r.isearch_trans)
+ r.isearch_start = r.historyi, r.pos
+
+
+class isearch_cancel(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_NONE
+ r.pop_input_trans()
+ r.select_item(r.isearch_start[0])
+ r.pos = r.isearch_start[1]
+ r.dirty = True
+
+
+class isearch_add_character(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ r.isearch_term += self.event[-1]
+ r.dirty = True
+ p = r.pos + len(r.isearch_term) - 1
+ if b[p : p + 1] != [r.isearch_term[-1]]:
+ r.isearch_next()
+
+
+class isearch_backspace(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ if len(r.isearch_term) > 0:
+ r.isearch_term = r.isearch_term[:-1]
+ r.dirty = True
+ else:
+ r.error("nothing to rubout")
+
+
+class isearch_forwards(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_FORWARDS
+ r.isearch_next()
+
+
+class isearch_backwards(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_BACKWARDS
+ r.isearch_next()
+
+
+class isearch_end(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ r.isearch_direction = ISEARCH_DIRECTION_NONE
+ r.console.forgetinput()
+ r.pop_input_trans()
+ r.dirty = True
+
+
+@dataclass
+class HistoricalReader(Reader):
+ """Adds history support (with incremental history searching) to the
+ Reader class.
+ """
+
+ history: list[str] = field(default_factory=list)
+ historyi: int = 0
+ next_history: int | None = None
+ transient_history: dict[int, str] = field(default_factory=dict)
+ isearch_term: str = ""
+ isearch_direction: str = ISEARCH_DIRECTION_NONE
+ isearch_start: tuple[int, int] = field(init=False)
+ isearch_trans: input.KeymapTranslator = field(init=False)
+ yank_arg_i: int = 0
+ yank_arg_yanked: str = ""
+
+ def __post_init__(self) -> None:
+ super().__post_init__()
+ for c in [
+ next_history,
+ previous_history,
+ restore_history,
+ first_history,
+ last_history,
+ yank_arg,
+ forward_history_isearch,
+ reverse_history_isearch,
+ isearch_end,
+ isearch_add_character,
+ isearch_cancel,
+ isearch_add_character,
+ isearch_backspace,
+ isearch_forwards,
+ isearch_backwards,
+ operate_and_get_next,
+ history_search_backward,
+ history_search_forward,
+ ]:
+ self.commands[c.__name__] = c
+ self.commands[c.__name__.replace("_", "-")] = c
+ self.isearch_start = self.historyi, self.pos
+ self.isearch_trans = input.KeymapTranslator(
+ isearch_keymap, invalid_cls=isearch_end, character_cls=isearch_add_character
+ )
+
+ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+ return super().collect_keymap() + (
+ (r"\C-n", "next-history"),
+ (r"\C-p", "previous-history"),
+ (r"\C-o", "operate-and-get-next"),
+ (r"\C-r", "reverse-history-isearch"),
+ (r"\C-s", "forward-history-isearch"),
+ (r"\M-r", "restore-history"),
+ (r"\M-.", "yank-arg"),
+ (r"\", "history-search-forward"),
+ (r"\x1b[6~", "history-search-forward"),
+ (r"\", "history-search-backward"),
+ (r"\x1b[5~", "history-search-backward"),
+ )
+
+ def select_item(self, i: int) -> None:
+ self.transient_history[self.historyi] = self.get_unicode()
+ buf = self.transient_history.get(i)
+ if buf is None:
+ buf = self.history[i].rstrip()
+ self.buffer = list(buf)
+ self.historyi = i
+ self.pos = len(self.buffer)
+ self.dirty = True
+ self.last_refresh_cache.invalidated = True
+
+ def get_item(self, i: int) -> str:
+ if i != len(self.history):
+ return self.transient_history.get(i, self.history[i])
+ else:
+ return self.transient_history.get(i, self.get_unicode())
+
+ @contextmanager
+ def suspend(self) -> SimpleContextManager:
+ with super().suspend(), self.suspend_history():
+ yield
+
+ @contextmanager
+ def suspend_history(self) -> SimpleContextManager:
+ try:
+ old_history = self.history[:]
+ del self.history[:]
+ yield
+ finally:
+ self.history[:] = old_history
+
+ def prepare(self) -> None:
+ super().prepare()
+ try:
+ self.transient_history = {}
+ if self.next_history is not None and self.next_history < len(self.history):
+ self.historyi = self.next_history
+ self.buffer[:] = list(self.history[self.next_history])
+ self.pos = len(self.buffer)
+ self.transient_history[len(self.history)] = ""
+ else:
+ self.historyi = len(self.history)
+ self.next_history = None
+ except:
+ self.restore()
+ raise
+
+ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+ if cursor_on_line and self.isearch_direction != ISEARCH_DIRECTION_NONE:
+ d = "rf"[self.isearch_direction == ISEARCH_DIRECTION_FORWARDS]
+ return "(%s-search `%s') " % (d, self.isearch_term)
+ else:
+ return super().get_prompt(lineno, cursor_on_line)
+
+ def search_next(self, *, forwards: bool) -> None:
+ """Search history for the current line contents up to the cursor.
+
+ Selects the first item found. If nothing is under the cursor, any next
+ item in history is selected.
+ """
+ pos = self.pos
+ s = self.get_unicode()
+ history_index = self.historyi
+
+ # In multiline contexts, we're only interested in the current line.
+ nl_index = s.rfind('\n', 0, pos)
+ prefix = s[nl_index + 1:pos]
+ pos = len(prefix)
+
+ match_prefix = len(prefix)
+ len_item = 0
+ if history_index < len(self.history):
+ len_item = len(self.get_item(history_index))
+ if len_item and pos == len_item:
+ match_prefix = False
+ elif not pos:
+ match_prefix = False
+
+ while 1:
+ if forwards:
+ out_of_bounds = history_index >= len(self.history) - 1
+ else:
+ out_of_bounds = history_index == 0
+ if out_of_bounds:
+ if forwards and not match_prefix:
+ self.pos = 0
+ self.buffer = []
+ self.dirty = True
+ else:
+ self.error("not found")
+ return
+
+ history_index += 1 if forwards else -1
+ s = self.get_item(history_index)
+
+ if not match_prefix:
+ self.select_item(history_index)
+ return
+
+ len_acc = 0
+ for i, line in enumerate(s.splitlines(keepends=True)):
+ if line.startswith(prefix):
+ self.select_item(history_index)
+ self.pos = pos + len_acc
+ return
+ len_acc += len(line)
+
+ def isearch_next(self) -> None:
+ st = self.isearch_term
+ p = self.pos
+ i = self.historyi
+ s = self.get_unicode()
+ forwards = self.isearch_direction == ISEARCH_DIRECTION_FORWARDS
+ while 1:
+ if forwards:
+ p = s.find(st, p + 1)
+ else:
+ p = s.rfind(st, 0, p + len(st) - 1)
+ if p != -1:
+ self.select_item(i)
+ self.pos = p
+ return
+ elif (forwards and i >= len(self.history) - 1) or (not forwards and i == 0):
+ self.error("not found")
+ return
+ else:
+ if forwards:
+ i += 1
+ s = self.get_item(i)
+ p = -1
+ else:
+ i -= 1
+ s = self.get_item(i)
+ p = len(s)
+
+ def finish(self) -> None:
+ super().finish()
+ ret = self.get_unicode()
+ for i, t in self.transient_history.items():
+ if i < len(self.history) and i != self.historyi:
+ self.history[i] = t
+ if ret and should_auto_add_history:
+ self.history.append(ret)
+
+
+should_auto_add_history = True
diff --git a/PythonLib/full/_pyrepl/input.py b/PythonLib/full/_pyrepl/input.py
new file mode 100644
index 000000000..21c24eb5c
--- /dev/null
+++ b/PythonLib/full/_pyrepl/input.py
@@ -0,0 +1,114 @@
+# Copyright 2000-2004 Michael Hudson-Doyle
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+# (naming modules after builtin functions is not such a hot idea...)
+
+# an KeyTrans instance translates Event objects into Command objects
+
+# hmm, at what level do we want [C-i] and [tab] to be equivalent?
+# [meta-a] and [esc a]? obviously, these are going to be equivalent
+# for the UnixConsole, but should they be for PygameConsole?
+
+# it would in any situation seem to be a bad idea to bind, say, [tab]
+# and [C-i] to *different* things... but should binding one bind the
+# other?
+
+# executive, temporary decision: [tab] and [C-i] are distinct, but
+# [meta-key] is identified with [esc key]. We demand that any console
+# class does quite a lot towards emulating a unix terminal.
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+import unicodedata
+from collections import deque
+
+
+# types
+if False:
+ from .types import EventTuple
+
+
+class InputTranslator(ABC):
+ @abstractmethod
+ def push(self, evt: EventTuple) -> None:
+ pass
+
+ @abstractmethod
+ def get(self) -> EventTuple | None:
+ return None
+
+ @abstractmethod
+ def empty(self) -> bool:
+ return True
+
+
+class KeymapTranslator(InputTranslator):
+ def __init__(self, keymap, verbose=False, invalid_cls=None, character_cls=None):
+ self.verbose = verbose
+ from .keymap import compile_keymap, parse_keys
+
+ self.keymap = keymap
+ self.invalid_cls = invalid_cls
+ self.character_cls = character_cls
+ d = {}
+ for keyspec, command in keymap:
+ keyseq = tuple(parse_keys(keyspec))
+ d[keyseq] = command
+ if self.verbose:
+ print(d)
+ self.k = self.ck = compile_keymap(d, ())
+ self.results = deque()
+ self.stack = []
+
+ def push(self, evt):
+ if self.verbose:
+ print("pushed", evt.data, end="")
+ key = evt.data
+ d = self.k.get(key)
+ if isinstance(d, dict):
+ if self.verbose:
+ print("transition")
+ self.stack.append(key)
+ self.k = d
+ else:
+ if d is None:
+ if self.verbose:
+ print("invalid")
+ if self.stack or len(key) > 1 or unicodedata.category(key) == "C":
+ self.results.append((self.invalid_cls, self.stack + [key]))
+ else:
+ # small optimization:
+ self.k[key] = self.character_cls
+ self.results.append((self.character_cls, [key]))
+ else:
+ if self.verbose:
+ print("matched", d)
+ self.results.append((d, self.stack + [key]))
+ self.stack = []
+ self.k = self.ck
+
+ def get(self):
+ if self.results:
+ return self.results.popleft()
+ else:
+ return None
+
+ def empty(self) -> bool:
+ return not self.results
diff --git a/PythonLib/full/_pyrepl/keymap.py b/PythonLib/full/_pyrepl/keymap.py
new file mode 100644
index 000000000..d11df4b51
--- /dev/null
+++ b/PythonLib/full/_pyrepl/keymap.py
@@ -0,0 +1,213 @@
+# Copyright 2000-2008 Michael Hudson-Doyle
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""
+Keymap contains functions for parsing keyspecs and turning keyspecs into
+appropriate sequences.
+
+A keyspec is a string representing a sequence of key presses that can
+be bound to a command. All characters other than the backslash represent
+themselves. In the traditional manner, a backslash introduces an escape
+sequence.
+
+pyrepl uses its own keyspec format that is meant to be a strict superset of
+readline's KEYSEQ format. This means that if a spec is found that readline
+accepts that this doesn't, it should be logged as a bug. Note that this means
+we're using the '\\C-o' style of readline's keyspec, not the 'Control-o' sort.
+
+The extension to readline is that the sequence \\ denotes the
+sequence of characters produced by hitting KEY.
+
+Examples:
+'a' - what you get when you hit the 'a' key
+'\\EOA' - Escape - O - A (up, on my terminal)
+'\\' - the up arrow key
+'\\' - ditto (keynames are case-insensitive)
+'\\C-o', '\\c-o' - control-o
+'\\M-.' - meta-period
+'\\E.' - ditto (that's how meta works for pyrepl)
+'\\', '\\', '\\t', '\\011', '\\x09', '\\X09', '\\C-i', '\\C-I'
+ - all of these are the tab character.
+"""
+
+_escapes = {
+ "\\": "\\",
+ "'": "'",
+ '"': '"',
+ "a": "\a",
+ "b": "\b",
+ "e": "\033",
+ "f": "\f",
+ "n": "\n",
+ "r": "\r",
+ "t": "\t",
+ "v": "\v",
+}
+
+_keynames = {
+ "backspace": "backspace",
+ "delete": "delete",
+ "down": "down",
+ "end": "end",
+ "enter": "\r",
+ "escape": "\033",
+ "f1": "f1",
+ "f2": "f2",
+ "f3": "f3",
+ "f4": "f4",
+ "f5": "f5",
+ "f6": "f6",
+ "f7": "f7",
+ "f8": "f8",
+ "f9": "f9",
+ "f10": "f10",
+ "f11": "f11",
+ "f12": "f12",
+ "f13": "f13",
+ "f14": "f14",
+ "f15": "f15",
+ "f16": "f16",
+ "f17": "f17",
+ "f18": "f18",
+ "f19": "f19",
+ "f20": "f20",
+ "home": "home",
+ "insert": "insert",
+ "left": "left",
+ "page down": "page down",
+ "page up": "page up",
+ "return": "\r",
+ "right": "right",
+ "space": " ",
+ "tab": "\t",
+ "up": "up",
+}
+
+
+class KeySpecError(Exception):
+ pass
+
+
+def parse_keys(keys: str) -> list[str]:
+ """Parse keys in keyspec format to a sequence of keys."""
+ s = 0
+ r: list[str] = []
+ while s < len(keys):
+ k, s = _parse_single_key_sequence(keys, s)
+ r.extend(k)
+ return r
+
+
+def _parse_single_key_sequence(key: str, s: int) -> tuple[list[str], int]:
+ ctrl = 0
+ meta = 0
+ ret = ""
+ while not ret and s < len(key):
+ if key[s] == "\\":
+ c = key[s + 1].lower()
+ if c in _escapes:
+ ret = _escapes[c]
+ s += 2
+ elif c == "c":
+ if key[s + 2] != "-":
+ raise KeySpecError(
+ "\\C must be followed by `-' (char %d of %s)"
+ % (s + 2, repr(key))
+ )
+ if ctrl:
+ raise KeySpecError(
+ "doubled \\C- (char %d of %s)" % (s + 1, repr(key))
+ )
+ ctrl = 1
+ s += 3
+ elif c == "m":
+ if key[s + 2] != "-":
+ raise KeySpecError(
+ "\\M must be followed by `-' (char %d of %s)"
+ % (s + 2, repr(key))
+ )
+ if meta:
+ raise KeySpecError(
+ "doubled \\M- (char %d of %s)" % (s + 1, repr(key))
+ )
+ meta = 1
+ s += 3
+ elif c.isdigit():
+ n = key[s + 1 : s + 4]
+ ret = chr(int(n, 8))
+ s += 4
+ elif c == "x":
+ n = key[s + 2 : s + 4]
+ ret = chr(int(n, 16))
+ s += 4
+ elif c == "<":
+ t = key.find(">", s)
+ if t == -1:
+ raise KeySpecError(
+ "unterminated \\< starting at char %d of %s"
+ % (s + 1, repr(key))
+ )
+ ret = key[s + 2 : t].lower()
+ if ret not in _keynames:
+ raise KeySpecError(
+ "unrecognised keyname `%s' at char %d of %s"
+ % (ret, s + 2, repr(key))
+ )
+ ret = _keynames[ret]
+ s = t + 1
+ else:
+ raise KeySpecError(
+ "unknown backslash escape %s at char %d of %s"
+ % (repr(c), s + 2, repr(key))
+ )
+ else:
+ ret = key[s]
+ s += 1
+ if ctrl:
+ if len(ret) == 1:
+ ret = chr(ord(ret) & 0x1F) # curses.ascii.ctrl()
+ elif ret in {"left", "right"}:
+ ret = f"ctrl {ret}"
+ else:
+ raise KeySpecError("\\C- followed by invalid key")
+
+ result = [ret], s
+ if meta:
+ result[0].insert(0, "\033")
+ return result
+
+
+def compile_keymap(keymap, empty=b""):
+ r = {}
+ for key, value in keymap.items():
+ if isinstance(key, bytes):
+ first = key[:1]
+ else:
+ first = key[0]
+ r.setdefault(first, {})[key[1:]] = value
+ for key, value in r.items():
+ if empty in value:
+ if len(value) != 1:
+ raise KeySpecError("key definitions for %s clash" % (value.values(),))
+ else:
+ r[key] = value[empty]
+ else:
+ r[key] = compile_keymap(value, empty)
+ return r
diff --git a/PythonLib/full/_pyrepl/main.py b/PythonLib/full/_pyrepl/main.py
new file mode 100644
index 000000000..447eb1e55
--- /dev/null
+++ b/PythonLib/full/_pyrepl/main.py
@@ -0,0 +1,58 @@
+import errno
+import os
+import sys
+import types
+
+
+CAN_USE_PYREPL: bool
+FAIL_REASON: str
+try:
+ if sys.platform == "win32" and sys.getwindowsversion().build < 10586:
+ raise RuntimeError("Windows 10 TH2 or later required")
+ if not os.isatty(sys.stdin.fileno()):
+ raise OSError(errno.ENOTTY, "tty required", "stdin")
+ from .simple_interact import check
+ if err := check():
+ raise RuntimeError(err)
+except Exception as e:
+ CAN_USE_PYREPL = False
+ FAIL_REASON = f"warning: can't use pyrepl: {e}"
+else:
+ CAN_USE_PYREPL = True
+ FAIL_REASON = ""
+
+
+def interactive_console(mainmodule=None, quiet=False, pythonstartup=False):
+ if not CAN_USE_PYREPL:
+ if not os.getenv('PYTHON_BASIC_REPL') and FAIL_REASON:
+ from .trace import trace
+ trace(FAIL_REASON)
+ print(FAIL_REASON, file=sys.stderr)
+ return sys._baserepl()
+
+ if not mainmodule:
+ mainmodule = types.ModuleType("__main__")
+
+ namespace = mainmodule.__dict__
+
+ # sys._baserepl() above does this internally, we do it here
+ startup_path = os.getenv("PYTHONSTARTUP")
+ if pythonstartup and startup_path:
+ sys.audit("cpython.run_startup", startup_path)
+
+ import tokenize
+ with tokenize.open(startup_path) as f:
+ startup_code = compile(f.read(), startup_path, "exec")
+ exec(startup_code, namespace)
+
+ # set sys.{ps1,ps2} just before invoking the interactive interpreter. This
+ # mimics what CPython does in pythonrun.c
+ if not hasattr(sys, "ps1"):
+ sys.ps1 = ">>> "
+ if not hasattr(sys, "ps2"):
+ sys.ps2 = "... "
+
+ from .console import InteractiveColoredConsole
+ from .simple_interact import run_multiline_interactive_console
+ console = InteractiveColoredConsole(namespace, filename="")
+ run_multiline_interactive_console(console)
diff --git a/PythonLib/full/_pyrepl/mypy.ini b/PythonLib/full/_pyrepl/mypy.ini
new file mode 100644
index 000000000..9375a55b5
--- /dev/null
+++ b/PythonLib/full/_pyrepl/mypy.ini
@@ -0,0 +1,25 @@
+# Config file for running mypy on _pyrepl.
+# Run mypy by invoking `mypy --config-file Lib/_pyrepl/mypy.ini`
+# on the command-line from the repo root
+
+[mypy]
+files = Lib/_pyrepl
+mypy_path = $MYPY_CONFIG_FILE_DIR/../../Misc/mypy
+explicit_package_bases = True
+python_version = 3.13
+platform = linux
+pretty = True
+
+# Enable most stricter settings
+enable_error_code = ignore-without-code,redundant-expr
+strict = True
+
+# Various stricter settings that we can't yet enable
+# Try to enable these in the following order:
+disallow_untyped_calls = False
+disallow_untyped_defs = False
+check_untyped_defs = False
+
+# Various internal modules that typeshed deliberately doesn't have stubs for:
+[mypy-_abc.*,_opcode.*,_overlapped.*,_testcapi.*,_testinternalcapi.*,test.*]
+ignore_missing_imports = True
diff --git a/PythonLib/full/_pyrepl/pager.py b/PythonLib/full/_pyrepl/pager.py
new file mode 100644
index 000000000..1fddc63e3
--- /dev/null
+++ b/PythonLib/full/_pyrepl/pager.py
@@ -0,0 +1,175 @@
+from __future__ import annotations
+
+import io
+import os
+import re
+import sys
+
+
+# types
+if False:
+ from typing import Protocol
+ class Pager(Protocol):
+ def __call__(self, text: str, title: str = "") -> None:
+ ...
+
+
+def get_pager() -> Pager:
+ """Decide what method to use for paging through text."""
+ if not hasattr(sys.stdin, "isatty"):
+ return plain_pager
+ if not hasattr(sys.stdout, "isatty"):
+ return plain_pager
+ if not sys.stdin.isatty() or not sys.stdout.isatty():
+ return plain_pager
+ if sys.platform == "emscripten":
+ return plain_pager
+ use_pager = os.environ.get('MANPAGER') or os.environ.get('PAGER')
+ if use_pager:
+ if sys.platform == 'win32': # pipes completely broken in Windows
+ return lambda text, title='': tempfile_pager(plain(text), use_pager)
+ elif os.environ.get('TERM') in ('dumb', 'emacs'):
+ return lambda text, title='': pipe_pager(plain(text), use_pager, title)
+ else:
+ return lambda text, title='': pipe_pager(text, use_pager, title)
+ if os.environ.get('TERM') in ('dumb', 'emacs'):
+ return plain_pager
+ if sys.platform == 'win32':
+ return lambda text, title='': tempfile_pager(plain(text), 'more <')
+ if hasattr(os, 'system') and os.system('(pager) 2>/dev/null') == 0:
+ return lambda text, title='': pipe_pager(text, 'pager', title)
+ if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0:
+ return lambda text, title='': pipe_pager(text, 'less', title)
+
+ import tempfile
+ (fd, filename) = tempfile.mkstemp()
+ os.close(fd)
+ try:
+ if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0:
+ return lambda text, title='': pipe_pager(text, 'more', title)
+ else:
+ return tty_pager
+ finally:
+ os.unlink(filename)
+
+
+def escape_stdout(text: str) -> str:
+ # Escape non-encodable characters to avoid encoding errors later
+ encoding = getattr(sys.stdout, 'encoding', None) or 'utf-8'
+ return text.encode(encoding, 'backslashreplace').decode(encoding)
+
+
+def escape_less(s: str) -> str:
+ return re.sub(r'([?:.%\\])', r'\\\1', s)
+
+
+def plain(text: str) -> str:
+ """Remove boldface formatting from text."""
+ return re.sub('.\b', '', text)
+
+
+def tty_pager(text: str, title: str = '') -> None:
+ """Page through text on a text terminal."""
+ lines = plain(escape_stdout(text)).split('\n')
+ has_tty = False
+ try:
+ import tty
+ import termios
+ fd = sys.stdin.fileno()
+ old = termios.tcgetattr(fd)
+ tty.setcbreak(fd)
+ has_tty = True
+
+ def getchar() -> str:
+ return sys.stdin.read(1)
+
+ except (ImportError, AttributeError, io.UnsupportedOperation):
+ def getchar() -> str:
+ return sys.stdin.readline()[:-1][:1]
+
+ try:
+ try:
+ h = int(os.environ.get('LINES', 0))
+ except ValueError:
+ h = 0
+ if h <= 1:
+ h = 25
+ r = inc = h - 1
+ sys.stdout.write('\n'.join(lines[:inc]) + '\n')
+ while lines[r:]:
+ sys.stdout.write('-- more --')
+ sys.stdout.flush()
+ c = getchar()
+
+ if c in ('q', 'Q'):
+ sys.stdout.write('\r \r')
+ break
+ elif c in ('\r', '\n'):
+ sys.stdout.write('\r \r' + lines[r] + '\n')
+ r = r + 1
+ continue
+ if c in ('b', 'B', '\x1b'):
+ r = r - inc - inc
+ if r < 0: r = 0
+ sys.stdout.write('\n' + '\n'.join(lines[r:r+inc]) + '\n')
+ r = r + inc
+
+ finally:
+ if has_tty:
+ termios.tcsetattr(fd, termios.TCSAFLUSH, old)
+
+
+def plain_pager(text: str, title: str = '') -> None:
+ """Simply print unformatted text. This is the ultimate fallback."""
+ sys.stdout.write(plain(escape_stdout(text)))
+
+
+def pipe_pager(text: str, cmd: str, title: str = '') -> None:
+ """Page through text by feeding it to another program."""
+ import subprocess
+ env = os.environ.copy()
+ if title:
+ title += ' '
+ esc_title = escape_less(title)
+ prompt_string = (
+ f' {esc_title}' +
+ '?ltline %lt?L/%L.'
+ ':byte %bB?s/%s.'
+ '.'
+ '?e (END):?pB %pB\\%..'
+ ' (press h for help or q to quit)')
+ env['LESS'] = '-RmPm{0}$PM{0}$'.format(prompt_string)
+ proc = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE,
+ errors='backslashreplace', env=env)
+ assert proc.stdin is not None
+ try:
+ with proc.stdin as pipe:
+ try:
+ pipe.write(text)
+ except KeyboardInterrupt:
+ # We've hereby abandoned whatever text hasn't been written,
+ # but the pager is still in control of the terminal.
+ pass
+ except OSError:
+ pass # Ignore broken pipes caused by quitting the pager program.
+ while True:
+ try:
+ proc.wait()
+ break
+ except KeyboardInterrupt:
+ # Ignore ctl-c like the pager itself does. Otherwise the pager is
+ # left running and the terminal is in raw mode and unusable.
+ pass
+
+
+def tempfile_pager(text: str, cmd: str, title: str = '') -> None:
+ """Page through text by invoking a program on a temporary file."""
+ import tempfile
+ with tempfile.TemporaryDirectory() as tempdir:
+ filename = os.path.join(tempdir, 'pydoc.out')
+ with open(filename, 'w', errors='backslashreplace',
+ encoding=os.device_encoding(0) if
+ sys.platform == 'win32' else None
+ ) as file:
+ file.write(text)
+ os.system(cmd + ' "' + filename + '"')
diff --git a/PythonLib/full/_pyrepl/reader.py b/PythonLib/full/_pyrepl/reader.py
new file mode 100644
index 000000000..f0116e742
--- /dev/null
+++ b/PythonLib/full/_pyrepl/reader.py
@@ -0,0 +1,773 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Antonio Cuni
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import sys
+import _colorize
+
+from contextlib import contextmanager
+from dataclasses import dataclass, field, fields
+
+from . import commands, console, input
+from .utils import wlen, unbracket, disp_str, gen_colors, THEME
+from .trace import trace
+
+
+# types
+Command = commands.Command
+from .types import Callback, SimpleContextManager, KeySpec, CommandName
+
+
+# syntax classes
+SYNTAX_WHITESPACE, SYNTAX_WORD, SYNTAX_SYMBOL = range(3)
+
+
+def make_default_syntax_table() -> dict[str, int]:
+ # XXX perhaps should use some unicodedata here?
+ st: dict[str, int] = {}
+ for c in map(chr, range(256)):
+ st[c] = SYNTAX_SYMBOL
+ for c in [a for a in map(chr, range(256)) if a.isalnum()]:
+ st[c] = SYNTAX_WORD
+ st["\n"] = st[" "] = SYNTAX_WHITESPACE
+ return st
+
+
+def make_default_commands() -> dict[CommandName, type[Command]]:
+ result: dict[CommandName, type[Command]] = {}
+ for v in vars(commands).values():
+ if isinstance(v, type) and issubclass(v, Command) and v.__name__[0].islower():
+ result[v.__name__] = v
+ result[v.__name__.replace("_", "-")] = v
+ return result
+
+
+default_keymap: tuple[tuple[KeySpec, CommandName], ...] = tuple(
+ [
+ (r"\C-a", "beginning-of-line"),
+ (r"\C-b", "left"),
+ (r"\C-c", "interrupt"),
+ (r"\C-d", "delete"),
+ (r"\C-e", "end-of-line"),
+ (r"\C-f", "right"),
+ (r"\C-g", "cancel"),
+ (r"\C-h", "backspace"),
+ (r"\C-j", "accept"),
+ (r"\", "accept"),
+ (r"\C-k", "kill-line"),
+ (r"\C-l", "clear-screen"),
+ (r"\C-m", "accept"),
+ (r"\C-t", "transpose-characters"),
+ (r"\C-u", "unix-line-discard"),
+ (r"\C-w", "unix-word-rubout"),
+ (r"\C-x\C-u", "upcase-region"),
+ (r"\C-y", "yank"),
+ *(() if sys.platform == "win32" else ((r"\C-z", "suspend"), )),
+ (r"\M-b", "backward-word"),
+ (r"\M-c", "capitalize-word"),
+ (r"\M-d", "kill-word"),
+ (r"\M-f", "forward-word"),
+ (r"\M-l", "downcase-word"),
+ (r"\M-t", "transpose-words"),
+ (r"\M-u", "upcase-word"),
+ (r"\M-y", "yank-pop"),
+ (r"\M--", "digit-arg"),
+ (r"\M-0", "digit-arg"),
+ (r"\M-1", "digit-arg"),
+ (r"\M-2", "digit-arg"),
+ (r"\M-3", "digit-arg"),
+ (r"\M-4", "digit-arg"),
+ (r"\M-5", "digit-arg"),
+ (r"\M-6", "digit-arg"),
+ (r"\M-7", "digit-arg"),
+ (r"\M-8", "digit-arg"),
+ (r"\M-9", "digit-arg"),
+ (r"\M-\n", "accept"),
+ ("\\\\", "self-insert"),
+ (r"\x1b[200~", "perform-bracketed-paste"),
+ (r"\x03", "ctrl-c"),
+ ]
+ + [(c, "self-insert") for c in map(chr, range(32, 127)) if c != "\\"]
+ + [(c, "self-insert") for c in map(chr, range(128, 256)) if c.isalpha()]
+ + [
+ (r"\", "up"),
+ (r"\", "down"),
+ (r"\", "left"),
+ (r"\C-\", "backward-word"),
+ (r"\", "right"),
+ (r"\C-\", "forward-word"),
+ (r"\", "delete"),
+ (r"\x1b[3~", "delete"),
+ (r"\", "backspace"),
+ (r"\M-\", "backward-kill-word"),
+ (r"\", "end-of-line"), # was 'end'
+ (r"\", "beginning-of-line"), # was 'home'
+ (r"\", "help"),
+ (r"\", "show-history"),
+ (r"\", "paste-mode"),
+ (r"\EOF", "end"), # the entries in the terminfo database for xterms
+ (r"\EOH", "home"), # seem to be wrong. this is a less than ideal
+ # workaround
+ ]
+)
+
+
+@dataclass(slots=True)
+class Reader:
+ """The Reader class implements the bare bones of a command reader,
+ handling such details as editing and cursor motion. What it does
+ not support are such things as completion or history support -
+ these are implemented elsewhere.
+
+ Instance variables of note include:
+
+ * buffer:
+ A per-character list containing all the characters that have been
+ entered. Does not include color information.
+ * console:
+ Hopefully encapsulates the OS dependent stuff.
+ * pos:
+ A 0-based index into 'buffer' for where the insertion point
+ is.
+ * screeninfo:
+ A list of screen position tuples. Each list element is a tuple
+ representing information on visible line length for a given line.
+ Allows for efficient skipping of color escape sequences.
+ * cxy, lxy:
+ the position of the insertion point in screen ...
+ * syntax_table:
+ Dictionary mapping characters to 'syntax class'; read the
+ emacs docs to see what this means :-)
+ * commands:
+ Dictionary mapping command names to command classes.
+ * arg:
+ The emacs-style prefix argument. It will be None if no such
+ argument has been provided.
+ * dirty:
+ True if we need to refresh the display.
+ * kill_ring:
+ The emacs-style kill-ring; manipulated with yank & yank-pop
+ * ps1, ps2, ps3, ps4:
+ prompts. ps1 is the prompt for a one-line input; for a
+ multiline input it looks like:
+ ps2> first line of input goes here
+ ps3> second and further
+ ps3> lines get ps3
+ ...
+ ps4> and the last one gets ps4
+ As with the usual top-level, you can set these to instances if
+ you like; str() will be called on them (once) at the beginning
+ of each command. Don't put really long or newline containing
+ strings here, please!
+ This is just the default policy; you can change it freely by
+ overriding get_prompt() (and indeed some standard subclasses
+ do).
+ * finished:
+ handle1 will set this to a true value if a command signals
+ that we're done.
+ """
+
+ console: console.Console
+
+ ## state
+ buffer: list[str] = field(default_factory=list)
+ pos: int = 0
+ ps1: str = "->> "
+ ps2: str = "/>> "
+ ps3: str = "|.. "
+ ps4: str = R"\__ "
+ kill_ring: list[list[str]] = field(default_factory=list)
+ msg: str = ""
+ arg: int | None = None
+ dirty: bool = False
+ finished: bool = False
+ paste_mode: bool = False
+ commands: dict[str, type[Command]] = field(default_factory=make_default_commands)
+ last_command: type[Command] | None = None
+ syntax_table: dict[str, int] = field(default_factory=make_default_syntax_table)
+ keymap: tuple[tuple[str, str], ...] = ()
+ input_trans: input.KeymapTranslator = field(init=False)
+ input_trans_stack: list[input.KeymapTranslator] = field(default_factory=list)
+ screen: list[str] = field(default_factory=list)
+ screeninfo: list[tuple[int, list[int]]] = field(init=False)
+ cxy: tuple[int, int] = field(init=False)
+ lxy: tuple[int, int] = field(init=False)
+ scheduled_commands: list[str] = field(default_factory=list)
+ can_colorize: bool = False
+ threading_hook: Callback | None = None
+
+ ## cached metadata to speed up screen refreshes
+ @dataclass
+ class RefreshCache:
+ screen: list[str] = field(default_factory=list)
+ screeninfo: list[tuple[int, list[int]]] = field(init=False)
+ line_end_offsets: list[int] = field(default_factory=list)
+ pos: int = field(init=False)
+ cxy: tuple[int, int] = field(init=False)
+ dimensions: tuple[int, int] = field(init=False)
+ invalidated: bool = False
+
+ def update_cache(self,
+ reader: Reader,
+ screen: list[str],
+ screeninfo: list[tuple[int, list[int]]],
+ ) -> None:
+ self.screen = screen.copy()
+ self.screeninfo = screeninfo.copy()
+ self.pos = reader.pos
+ self.cxy = reader.cxy
+ self.dimensions = reader.console.width, reader.console.height
+ self.invalidated = False
+
+ def valid(self, reader: Reader) -> bool:
+ if self.invalidated:
+ return False
+ dimensions = reader.console.width, reader.console.height
+ dimensions_changed = dimensions != self.dimensions
+ return not dimensions_changed
+
+ def get_cached_location(self, reader: Reader) -> tuple[int, int]:
+ if self.invalidated:
+ raise ValueError("Cache is invalidated")
+ offset = 0
+ earliest_common_pos = min(reader.pos, self.pos)
+ num_common_lines = len(self.line_end_offsets)
+ while num_common_lines > 0:
+ offset = self.line_end_offsets[num_common_lines - 1]
+ if earliest_common_pos > offset:
+ break
+ num_common_lines -= 1
+ else:
+ offset = 0
+ return offset, num_common_lines
+
+ last_refresh_cache: RefreshCache = field(default_factory=RefreshCache)
+
+ def __post_init__(self) -> None:
+ # Enable the use of `insert` without a `prepare` call - necessary to
+ # facilitate the tab completion hack implemented for
+ # .
+ self.keymap = self.collect_keymap()
+ self.input_trans = input.KeymapTranslator(
+ self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+ )
+ self.screeninfo = [(0, [])]
+ self.cxy = self.pos2xy()
+ self.lxy = (self.pos, 0)
+ self.can_colorize = _colorize.can_colorize()
+
+ self.last_refresh_cache.screeninfo = self.screeninfo
+ self.last_refresh_cache.pos = self.pos
+ self.last_refresh_cache.cxy = self.cxy
+ self.last_refresh_cache.dimensions = (0, 0)
+
+ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+ return default_keymap
+
+ def calc_screen(self) -> list[str]:
+ """Translate changes in self.buffer into changes in self.console.screen."""
+ # Since the last call to calc_screen:
+ # screen and screeninfo may differ due to a completion menu being shown
+ # pos and cxy may differ due to edits, cursor movements, or completion menus
+
+ # Lines that are above both the old and new cursor position can't have changed,
+ # unless the terminal has been resized (which might cause reflowing) or we've
+ # entered or left paste mode (which changes prompts, causing reflowing).
+ num_common_lines = 0
+ offset = 0
+ if self.last_refresh_cache.valid(self):
+ offset, num_common_lines = self.last_refresh_cache.get_cached_location(self)
+
+ screen = self.last_refresh_cache.screen
+ del screen[num_common_lines:]
+
+ screeninfo = self.last_refresh_cache.screeninfo
+ del screeninfo[num_common_lines:]
+
+ last_refresh_line_end_offsets = self.last_refresh_cache.line_end_offsets
+ del last_refresh_line_end_offsets[num_common_lines:]
+
+ pos = self.pos
+ pos -= offset
+
+ prompt_from_cache = (offset and self.buffer[offset - 1] != "\n")
+
+ if self.can_colorize:
+ colors = list(gen_colors(self.get_unicode()))
+ else:
+ colors = None
+ trace("colors = {colors}", colors=colors)
+ lines = "".join(self.buffer[offset:]).split("\n")
+ cursor_found = False
+ lines_beyond_cursor = 0
+ for ln, line in enumerate(lines, num_common_lines):
+ line_len = len(line)
+ if 0 <= pos <= line_len:
+ self.lxy = pos, ln
+ cursor_found = True
+ elif cursor_found:
+ lines_beyond_cursor += 1
+ if lines_beyond_cursor > self.console.height:
+ # No need to keep formatting lines.
+ # The console can't show them.
+ break
+ if prompt_from_cache:
+ # Only the first line's prompt can come from the cache
+ prompt_from_cache = False
+ prompt = ""
+ else:
+ prompt = self.get_prompt(ln, line_len >= pos >= 0)
+ while "\n" in prompt:
+ pre_prompt, _, prompt = prompt.partition("\n")
+ last_refresh_line_end_offsets.append(offset)
+ screen.append(pre_prompt)
+ screeninfo.append((0, []))
+ pos -= line_len + 1
+ prompt, prompt_len = self.process_prompt(prompt)
+ chars, char_widths = disp_str(line, colors, offset)
+ wrapcount = (sum(char_widths) + prompt_len) // self.console.width
+ if wrapcount == 0 or not char_widths:
+ offset += line_len + 1 # Takes all of the line plus the newline
+ last_refresh_line_end_offsets.append(offset)
+ screen.append(prompt + "".join(chars))
+ screeninfo.append((prompt_len, char_widths))
+ else:
+ pre = prompt
+ prelen = prompt_len
+ for wrap in range(wrapcount + 1):
+ index_to_wrap_before = 0
+ column = 0
+ for char_width in char_widths:
+ if column + char_width + prelen >= self.console.width:
+ break
+ index_to_wrap_before += 1
+ column += char_width
+ if len(chars) > index_to_wrap_before:
+ offset += index_to_wrap_before
+ post = "\\"
+ after = [1]
+ else:
+ offset += index_to_wrap_before + 1 # Takes the newline
+ post = ""
+ after = []
+ last_refresh_line_end_offsets.append(offset)
+ render = pre + "".join(chars[:index_to_wrap_before]) + post
+ render_widths = char_widths[:index_to_wrap_before] + after
+ screen.append(render)
+ screeninfo.append((prelen, render_widths))
+ chars = chars[index_to_wrap_before:]
+ char_widths = char_widths[index_to_wrap_before:]
+ pre = ""
+ prelen = 0
+ self.screeninfo = screeninfo
+ self.cxy = self.pos2xy()
+ if self.msg:
+ for mline in self.msg.split("\n"):
+ screen.append(mline)
+ screeninfo.append((0, []))
+
+ self.last_refresh_cache.update_cache(self, screen, screeninfo)
+ return screen
+
+ @staticmethod
+ def process_prompt(prompt: str) -> tuple[str, int]:
+ r"""Return a tuple with the prompt string and its visible length.
+
+ The prompt string has the zero-width brackets recognized by shells
+ (\x01 and \x02) removed. The length ignores anything between those
+ brackets as well as any ANSI escape sequences.
+ """
+ out_prompt = unbracket(prompt, including_content=False)
+ visible_prompt = unbracket(prompt, including_content=True)
+ return out_prompt, wlen(visible_prompt)
+
+ def bow(self, p: int | None = None) -> int:
+ """Return the 0-based index of the word break preceding p most
+ immediately.
+
+ p defaults to self.pos; word boundaries are determined using
+ self.syntax_table."""
+ if p is None:
+ p = self.pos
+ st = self.syntax_table
+ b = self.buffer
+ p -= 1
+ while p >= 0 and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+ p -= 1
+ while p >= 0 and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+ p -= 1
+ return p + 1
+
+ def eow(self, p: int | None = None) -> int:
+ """Return the 0-based index of the word break following p most
+ immediately.
+
+ p defaults to self.pos; word boundaries are determined using
+ self.syntax_table."""
+ if p is None:
+ p = self.pos
+ st = self.syntax_table
+ b = self.buffer
+ while p < len(b) and st.get(b[p], SYNTAX_WORD) != SYNTAX_WORD:
+ p += 1
+ while p < len(b) and st.get(b[p], SYNTAX_WORD) == SYNTAX_WORD:
+ p += 1
+ return p
+
+ def bol(self, p: int | None = None) -> int:
+ """Return the 0-based index of the line break preceding p most
+ immediately.
+
+ p defaults to self.pos."""
+ if p is None:
+ p = self.pos
+ b = self.buffer
+ p -= 1
+ while p >= 0 and b[p] != "\n":
+ p -= 1
+ return p + 1
+
+ def eol(self, p: int | None = None) -> int:
+ """Return the 0-based index of the line break following p most
+ immediately.
+
+ p defaults to self.pos."""
+ if p is None:
+ p = self.pos
+ b = self.buffer
+ while p < len(b) and b[p] != "\n":
+ p += 1
+ return p
+
+ def max_column(self, y: int) -> int:
+ """Return the last x-offset for line y"""
+ return self.screeninfo[y][0] + sum(self.screeninfo[y][1])
+
+ def max_row(self) -> int:
+ return len(self.screeninfo) - 1
+
+ def get_arg(self, default: int = 1) -> int:
+ """Return any prefix argument that the user has supplied,
+ returning 'default' if there is None. Defaults to 1.
+ """
+ if self.arg is None:
+ return default
+ return self.arg
+
+ def get_prompt(self, lineno: int, cursor_on_line: bool) -> str:
+ """Return what should be in the left-hand margin for line
+ 'lineno'."""
+ if self.arg is not None and cursor_on_line:
+ prompt = f"(arg: {self.arg}) "
+ elif self.paste_mode:
+ prompt = "(paste) "
+ elif "\n" in self.buffer:
+ if lineno == 0:
+ prompt = self.ps2
+ elif self.ps4 and lineno == self.buffer.count("\n"):
+ prompt = self.ps4
+ else:
+ prompt = self.ps3
+ else:
+ prompt = self.ps1
+
+ if self.can_colorize:
+ t = THEME()
+ prompt = f"{t.prompt}{prompt}{t.reset}"
+ return prompt
+
+ def push_input_trans(self, itrans: input.KeymapTranslator) -> None:
+ self.input_trans_stack.append(self.input_trans)
+ self.input_trans = itrans
+
+ def pop_input_trans(self) -> None:
+ self.input_trans = self.input_trans_stack.pop()
+
+ def setpos_from_xy(self, x: int, y: int) -> None:
+ """Set pos according to coordinates x, y"""
+ pos = 0
+ i = 0
+ while i < y:
+ prompt_len, char_widths = self.screeninfo[i]
+ offset = len(char_widths)
+ in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width
+ if in_wrapped_line:
+ pos += offset - 1 # -1 cause backslash is not in buffer
+ else:
+ pos += offset + 1 # +1 cause newline is in buffer
+ i += 1
+
+ j = 0
+ cur_x = self.screeninfo[i][0]
+ while cur_x < x:
+ if self.screeninfo[i][1][j] == 0:
+ j += 1 # prevent potential future infinite loop
+ continue
+ cur_x += self.screeninfo[i][1][j]
+ j += 1
+ pos += 1
+
+ self.pos = pos
+
+ def pos2xy(self) -> tuple[int, int]:
+ """Return the x, y coordinates of position 'pos'."""
+
+ prompt_len, y = 0, 0
+ char_widths: list[int] = []
+ pos = self.pos
+ assert 0 <= pos <= len(self.buffer)
+
+ # optimize for the common case: typing at the end of the buffer
+ if pos == len(self.buffer) and len(self.screeninfo) > 0:
+ y = len(self.screeninfo) - 1
+ prompt_len, char_widths = self.screeninfo[y]
+ return prompt_len + sum(char_widths), y
+
+ for prompt_len, char_widths in self.screeninfo:
+ offset = len(char_widths)
+ in_wrapped_line = prompt_len + sum(char_widths) >= self.console.width
+ if in_wrapped_line:
+ offset -= 1 # need to remove line-wrapping backslash
+
+ if offset >= pos:
+ break
+
+ if not in_wrapped_line:
+ offset += 1 # there's a newline in buffer
+
+ pos -= offset
+ y += 1
+ return prompt_len + sum(char_widths[:pos]), y
+
+ def insert(self, text: str | list[str]) -> None:
+ """Insert 'text' at the insertion point."""
+ self.buffer[self.pos : self.pos] = list(text)
+ self.pos += len(text)
+ self.dirty = True
+
+ def update_cursor(self) -> None:
+ """Move the cursor to reflect changes in self.pos"""
+ self.cxy = self.pos2xy()
+ trace("update_cursor({pos}) = {cxy}", pos=self.pos, cxy=self.cxy)
+ self.console.move_cursor(*self.cxy)
+
+ def after_command(self, cmd: Command) -> None:
+ """This function is called to allow post command cleanup."""
+ if getattr(cmd, "kills_digit_arg", True):
+ if self.arg is not None:
+ self.dirty = True
+ self.arg = None
+
+ def prepare(self) -> None:
+ """Get ready to run. Call restore when finished. You must not
+ write to the console in between the calls to prepare and
+ restore."""
+ try:
+ self.console.prepare()
+ self.arg = None
+ self.finished = False
+ del self.buffer[:]
+ self.pos = 0
+ self.dirty = True
+ self.last_command = None
+ self.calc_screen()
+ except BaseException:
+ self.restore()
+ raise
+
+ while self.scheduled_commands:
+ cmd = self.scheduled_commands.pop()
+ self.do_cmd((cmd, []))
+
+ def last_command_is(self, cls: type) -> bool:
+ if not self.last_command:
+ return False
+ return issubclass(cls, self.last_command)
+
+ def restore(self) -> None:
+ """Clean up after a run."""
+ self.console.restore()
+
+ @contextmanager
+ def suspend(self) -> SimpleContextManager:
+ """A context manager to delegate to another reader."""
+ prev_state = {f.name: getattr(self, f.name) for f in fields(self)}
+ try:
+ self.restore()
+ yield
+ finally:
+ for arg in ("msg", "ps1", "ps2", "ps3", "ps4", "paste_mode"):
+ setattr(self, arg, prev_state[arg])
+ self.prepare()
+
+ @contextmanager
+ def suspend_colorization(self) -> SimpleContextManager:
+ try:
+ old_can_colorize = self.can_colorize
+ self.can_colorize = False
+ yield
+ finally:
+ self.can_colorize = old_can_colorize
+
+
+ def finish(self) -> None:
+ """Called when a command signals that we're finished."""
+ pass
+
+ def error(self, msg: str = "none") -> None:
+ self.msg = "! " + msg + " "
+ self.dirty = True
+ self.console.beep()
+
+ def update_screen(self) -> None:
+ if self.dirty:
+ self.refresh()
+
+ def refresh(self) -> None:
+ """Recalculate and refresh the screen."""
+ self.console.height, self.console.width = self.console.getheightwidth()
+ # this call sets up self.cxy, so call it first.
+ self.screen = self.calc_screen()
+ self.console.refresh(self.screen, self.cxy)
+ self.dirty = False
+
+ def do_cmd(self, cmd: tuple[str, list[str]]) -> None:
+ """`cmd` is a tuple of "event_name" and "event", which in the current
+ implementation is always just the "buffer" which happens to be a list
+ of single-character strings."""
+
+ trace("received command {cmd}", cmd=cmd)
+ if isinstance(cmd[0], str):
+ command_type = self.commands.get(cmd[0], commands.invalid_command)
+ elif isinstance(cmd[0], type):
+ command_type = cmd[0]
+ else:
+ return # nothing to do
+
+ command = command_type(self, *cmd) # type: ignore[arg-type]
+ command.do()
+
+ self.after_command(command)
+
+ if self.dirty:
+ self.refresh()
+ else:
+ self.update_cursor()
+
+ if not isinstance(cmd, commands.digit_arg):
+ self.last_command = command_type
+
+ self.finished = bool(command.finish)
+ if self.finished:
+ self.console.finish()
+ self.finish()
+
+ def run_hooks(self) -> None:
+ threading_hook = self.threading_hook
+ if threading_hook is None and 'threading' in sys.modules:
+ from ._threading_handler import install_threading_hook
+ install_threading_hook(self)
+ if threading_hook is not None:
+ try:
+ threading_hook()
+ except Exception:
+ pass
+
+ input_hook = self.console.input_hook
+ if input_hook:
+ try:
+ input_hook()
+ except Exception:
+ pass
+
+ def handle1(self, block: bool = True) -> bool:
+ """Handle a single event. Wait as long as it takes if block
+ is true (the default), otherwise return False if no event is
+ pending."""
+
+ if self.msg:
+ self.msg = ""
+ self.dirty = True
+
+ while True:
+ # We use the same timeout as in readline.c: 100ms
+ self.run_hooks()
+ self.console.wait(100)
+ event = self.console.get_event(block=False)
+ if not event:
+ if block:
+ continue
+ return False
+
+ translate = True
+
+ if event.evt == "key":
+ self.input_trans.push(event)
+ elif event.evt == "scroll":
+ self.refresh()
+ elif event.evt == "resize":
+ self.refresh()
+ else:
+ translate = False
+
+ if translate:
+ cmd = self.input_trans.get()
+ else:
+ cmd = [event.evt, event.data]
+
+ if cmd is None:
+ if block:
+ continue
+ return False
+
+ self.do_cmd(cmd)
+ return True
+
+ def push_char(self, char: int | bytes) -> None:
+ self.console.push_char(char)
+ self.handle1(block=False)
+
+ def readline(self, startup_hook: Callback | None = None) -> str:
+ """Read a line. The implementation of this method also shows
+ how to drive Reader if you want more control over the event
+ loop."""
+ self.prepare()
+ try:
+ if startup_hook is not None:
+ startup_hook()
+ self.refresh()
+ while not self.finished:
+ self.handle1()
+ return self.get_unicode()
+
+ finally:
+ self.restore()
+
+ def bind(self, spec: KeySpec, command: CommandName) -> None:
+ self.keymap = self.keymap + ((spec, command),)
+ self.input_trans = input.KeymapTranslator(
+ self.keymap, invalid_cls="invalid-key", character_cls="self-insert"
+ )
+
+ def get_unicode(self) -> str:
+ """Return the current buffer as a unicode string."""
+ return "".join(self.buffer)
diff --git a/PythonLib/full/_pyrepl/readline.py b/PythonLib/full/_pyrepl/readline.py
new file mode 100644
index 000000000..23b8fa6b9
--- /dev/null
+++ b/PythonLib/full/_pyrepl/readline.py
@@ -0,0 +1,620 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Alex Gaynor
+# Antonio Cuni
+# Armin Rigo
+# Holger Krekel
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""A compatibility wrapper reimplementing the 'readline' standard module
+on top of pyrepl. Not all functionalities are supported. Contains
+extensions for multiline input.
+"""
+
+from __future__ import annotations
+
+import warnings
+from dataclasses import dataclass, field
+
+import os
+from site import gethistoryfile
+import sys
+from rlcompleter import Completer as RLCompleter
+
+from . import commands, historical_reader
+from .completing_reader import CompletingReader
+from .console import Console as ConsoleType
+from ._module_completer import ModuleCompleter, make_default_module_completer
+
+Console: type[ConsoleType]
+_error: tuple[type[Exception], ...] | type[Exception]
+
+if os.name == "nt":
+ from .windows_console import WindowsConsole as Console, _error
+else:
+ from .unix_console import UnixConsole as Console, _error
+
+ENCODING = sys.getdefaultencoding() or "latin1"
+
+
+# types
+Command = commands.Command
+from collections.abc import Callable, Collection
+from .types import Callback, Completer, KeySpec, CommandName
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+ from typing import Any, Mapping
+
+
+MoreLinesCallable = Callable[[str], bool]
+
+
+__all__ = [
+ "add_history",
+ "clear_history",
+ "get_begidx",
+ "get_completer",
+ "get_completer_delims",
+ "get_current_history_length",
+ "get_endidx",
+ "get_history_item",
+ "get_history_length",
+ "get_line_buffer",
+ "insert_text",
+ "parse_and_bind",
+ "read_history_file",
+ # "read_init_file",
+ # "redisplay",
+ "remove_history_item",
+ "replace_history_item",
+ "set_auto_history",
+ "set_completer",
+ "set_completer_delims",
+ "set_history_length",
+ # "set_pre_input_hook",
+ "set_startup_hook",
+ "write_history_file",
+ "append_history_file",
+ # ---- multiline extensions ----
+ "multiline_input",
+]
+
+# ____________________________________________________________
+
+@dataclass
+class ReadlineConfig:
+ readline_completer: Completer | None = None
+ completer_delims: frozenset[str] = frozenset(" \t\n`~!@#$%^&*()-=+[{]}\\|;:'\",<>/?")
+ module_completer: ModuleCompleter = field(default_factory=make_default_module_completer)
+
+@dataclass(kw_only=True)
+class ReadlineAlikeReader(historical_reader.HistoricalReader, CompletingReader):
+ # Class fields
+ assume_immutable_completions = False
+ use_brackets = False
+ sort_in_column = True
+
+ # Instance fields
+ config: ReadlineConfig
+ more_lines: MoreLinesCallable | None = None
+ last_used_indentation: str | None = None
+
+ def __post_init__(self) -> None:
+ super().__post_init__()
+ self.commands["maybe_accept"] = maybe_accept
+ self.commands["maybe-accept"] = maybe_accept
+ self.commands["backspace_dedent"] = backspace_dedent
+ self.commands["backspace-dedent"] = backspace_dedent
+
+ def error(self, msg: str = "none") -> None:
+ pass # don't show error messages by default
+
+ def get_stem(self) -> str:
+ b = self.buffer
+ p = self.pos - 1
+ completer_delims = self.config.completer_delims
+ while p >= 0 and b[p] not in completer_delims:
+ p -= 1
+ return "".join(b[p + 1 : self.pos])
+
+ def get_completions(self, stem: str) -> list[str]:
+ module_completions = self.get_module_completions()
+ if module_completions is not None:
+ return module_completions
+ if len(stem) == 0 and self.more_lines is not None:
+ b = self.buffer
+ p = self.pos
+ while p > 0 and b[p - 1] != "\n":
+ p -= 1
+ num_spaces = 4 - ((self.pos - p) % 4)
+ return [" " * num_spaces]
+ result = []
+ function = self.config.readline_completer
+ if function is not None:
+ try:
+ stem = str(stem) # rlcompleter.py seems to not like unicode
+ except UnicodeEncodeError:
+ pass # but feed unicode anyway if we have no choice
+ state = 0
+ while True:
+ try:
+ next = function(stem, state)
+ except Exception:
+ break
+ if not isinstance(next, str):
+ break
+ result.append(next)
+ state += 1
+ # emulate the behavior of the standard readline that sorts
+ # the completions before displaying them.
+ result.sort()
+ return result
+
+ def get_module_completions(self) -> list[str] | None:
+ line = self.get_line()
+ return self.config.module_completer.get_completions(line)
+
+ def get_trimmed_history(self, maxlength: int) -> list[str]:
+ if maxlength >= 0:
+ cut = len(self.history) - maxlength
+ if cut < 0:
+ cut = 0
+ else:
+ cut = 0
+ return self.history[cut:]
+
+ def update_last_used_indentation(self) -> None:
+ indentation = _get_first_indentation(self.buffer)
+ if indentation is not None:
+ self.last_used_indentation = indentation
+
+ # --- simplified support for reading multiline Python statements ---
+
+ def collect_keymap(self) -> tuple[tuple[KeySpec, CommandName], ...]:
+ return super().collect_keymap() + (
+ (r"\n", "maybe-accept"),
+ (r"\", "backspace-dedent"),
+ )
+
+ def after_command(self, cmd: Command) -> None:
+ super().after_command(cmd)
+ if self.more_lines is None:
+ # Force single-line input if we are in raw_input() mode.
+ # Although there is no direct way to add a \n in this mode,
+ # multiline buffers can still show up using various
+ # commands, e.g. navigating the history.
+ try:
+ index = self.buffer.index("\n")
+ except ValueError:
+ pass
+ else:
+ self.buffer = self.buffer[:index]
+ if self.pos > len(self.buffer):
+ self.pos = len(self.buffer)
+
+
+def set_auto_history(_should_auto_add_history: bool) -> None:
+ """Enable or disable automatic history"""
+ historical_reader.should_auto_add_history = bool(_should_auto_add_history)
+
+
+def _get_this_line_indent(buffer: list[str], pos: int) -> int:
+ indent = 0
+ while pos > 0 and buffer[pos - 1] in " \t":
+ indent += 1
+ pos -= 1
+ if pos > 0 and buffer[pos - 1] == "\n":
+ return indent
+ return 0
+
+
+def _get_previous_line_indent(buffer: list[str], pos: int) -> tuple[int, int | None]:
+ prevlinestart = pos
+ while prevlinestart > 0 and buffer[prevlinestart - 1] != "\n":
+ prevlinestart -= 1
+ prevlinetext = prevlinestart
+ while prevlinetext < pos and buffer[prevlinetext] in " \t":
+ prevlinetext += 1
+ if prevlinetext == pos:
+ indent = None
+ else:
+ indent = prevlinetext - prevlinestart
+ return prevlinestart, indent
+
+
+def _get_first_indentation(buffer: list[str]) -> str | None:
+ indented_line_start = None
+ for i in range(len(buffer)):
+ if (i < len(buffer) - 1
+ and buffer[i] == "\n"
+ and buffer[i + 1] in " \t"
+ ):
+ indented_line_start = i + 1
+ elif indented_line_start is not None and buffer[i] not in " \t\n":
+ return ''.join(buffer[indented_line_start : i])
+ return None
+
+
+def _should_auto_indent(buffer: list[str], pos: int) -> bool:
+ # check if last character before "pos" is a colon, ignoring
+ # whitespaces and comments.
+ last_char = None
+ while pos > 0:
+ pos -= 1
+ if last_char is None:
+ if buffer[pos] not in " \t\n#": # ignore whitespaces and comments
+ last_char = buffer[pos]
+ else:
+ # even if we found a non-whitespace character before
+ # original pos, we keep going back until newline is reached
+ # to make sure we ignore comments
+ if buffer[pos] == "\n":
+ break
+ if buffer[pos] == "#":
+ last_char = None
+ return last_char == ":"
+
+
+class maybe_accept(commands.Command):
+ def do(self) -> None:
+ r: ReadlineAlikeReader
+ r = self.reader # type: ignore[assignment]
+ r.dirty = True # this is needed to hide the completion menu, if visible
+
+ # if there are already several lines and the cursor
+ # is not on the last one, always insert a new \n.
+ text = r.get_unicode()
+
+ if "\n" in r.buffer[r.pos :] or (
+ r.more_lines is not None and r.more_lines(text)
+ ):
+ def _newline_before_pos():
+ before_idx = r.pos - 1
+ while before_idx > 0 and text[before_idx].isspace():
+ before_idx -= 1
+ return text[before_idx : r.pos].count("\n") > 0
+
+ # if there's already a new line before the cursor then
+ # even if the cursor is followed by whitespace, we assume
+ # the user is trying to terminate the block
+ if _newline_before_pos() and text[r.pos:].isspace():
+ self.finish = True
+ return
+
+ # auto-indent the next line like the previous line
+ prevlinestart, indent = _get_previous_line_indent(r.buffer, r.pos)
+ r.insert("\n")
+ if not self.reader.paste_mode:
+ if indent:
+ for i in range(prevlinestart, prevlinestart + indent):
+ r.insert(r.buffer[i])
+ r.update_last_used_indentation()
+ if _should_auto_indent(r.buffer, r.pos):
+ if r.last_used_indentation is not None:
+ indentation = r.last_used_indentation
+ else:
+ # default
+ indentation = " " * 4
+ r.insert(indentation)
+ elif not self.reader.paste_mode:
+ self.finish = True
+ else:
+ r.insert("\n")
+
+
+class backspace_dedent(commands.Command):
+ def do(self) -> None:
+ r = self.reader
+ b = r.buffer
+ if r.pos > 0:
+ repeat = 1
+ if b[r.pos - 1] != "\n":
+ indent = _get_this_line_indent(b, r.pos)
+ if indent > 0:
+ ls = r.pos - indent
+ while ls > 0:
+ ls, pi = _get_previous_line_indent(b, ls - 1)
+ if pi is not None and pi < indent:
+ repeat = indent - pi
+ break
+ r.pos -= repeat
+ del b[r.pos : r.pos + repeat]
+ r.dirty = True
+ else:
+ self.reader.error("can't backspace at start")
+
+
+# ____________________________________________________________
+
+
+@dataclass(slots=True)
+class _ReadlineWrapper:
+ f_in: int = -1
+ f_out: int = -1
+ reader: ReadlineAlikeReader | None = field(default=None, repr=False)
+ saved_history_length: int = -1
+ startup_hook: Callback | None = None
+ config: ReadlineConfig = field(default_factory=ReadlineConfig, repr=False)
+
+ def __post_init__(self) -> None:
+ if self.f_in == -1:
+ self.f_in = os.dup(0)
+ if self.f_out == -1:
+ self.f_out = os.dup(1)
+
+ def get_reader(self) -> ReadlineAlikeReader:
+ if self.reader is None:
+ console = Console(self.f_in, self.f_out, encoding=ENCODING)
+ self.reader = ReadlineAlikeReader(console=console, config=self.config)
+ return self.reader
+
+ def input(self, prompt: object = "") -> str:
+ try:
+ reader = self.get_reader()
+ except _error:
+ assert raw_input is not None
+ return raw_input(prompt)
+ prompt_str = str(prompt)
+ reader.ps1 = prompt_str
+ sys.audit("builtins.input", prompt_str)
+ result = reader.readline(startup_hook=self.startup_hook)
+ sys.audit("builtins.input/result", result)
+ return result
+
+ def multiline_input(self, more_lines: MoreLinesCallable, ps1: str, ps2: str) -> str:
+ """Read an input on possibly multiple lines, asking for more
+ lines as long as 'more_lines(unicodetext)' returns an object whose
+ boolean value is true.
+ """
+ reader = self.get_reader()
+ saved = reader.more_lines
+ try:
+ reader.more_lines = more_lines
+ reader.ps1 = ps1
+ reader.ps2 = ps1
+ reader.ps3 = ps2
+ reader.ps4 = ""
+ with warnings.catch_warnings(action="ignore"):
+ return reader.readline()
+ finally:
+ reader.more_lines = saved
+ reader.paste_mode = False
+
+ def parse_and_bind(self, string: str) -> None:
+ pass # XXX we don't support parsing GNU-readline-style init files
+
+ def set_completer(self, function: Completer | None = None) -> None:
+ self.config.readline_completer = function
+
+ def get_completer(self) -> Completer | None:
+ return self.config.readline_completer
+
+ def set_completer_delims(self, delimiters: Collection[str]) -> None:
+ self.config.completer_delims = frozenset(delimiters)
+
+ def get_completer_delims(self) -> str:
+ return "".join(sorted(self.config.completer_delims))
+
+ def _histline(self, line: str) -> str:
+ line = line.rstrip("\n")
+ return line
+
+ def get_history_length(self) -> int:
+ return self.saved_history_length
+
+ def set_history_length(self, length: int) -> None:
+ self.saved_history_length = length
+
+ def get_current_history_length(self) -> int:
+ return len(self.get_reader().history)
+
+ def read_history_file(self, filename: str = gethistoryfile()) -> None:
+ # multiline extension (really a hack) for the end of lines that
+ # are actually continuations inside a single multiline_input()
+ # history item: we use \r\n instead of just \n. If the history
+ # file is passed to GNU readline, the extra \r are just ignored.
+ history = self.get_reader().history
+
+ with open(os.path.expanduser(filename), 'rb') as f:
+ is_editline = f.readline().startswith(b"_HiStOrY_V2_")
+ if is_editline:
+ encoding = "unicode-escape"
+ else:
+ f.seek(0)
+ encoding = "utf-8"
+
+ lines = [line.decode(encoding, errors='replace') for line in f.read().split(b'\n')]
+ buffer = []
+ for line in lines:
+ if line.endswith("\r"):
+ buffer.append(line+'\n')
+ else:
+ line = self._histline(line)
+ if buffer:
+ line = self._histline("".join(buffer).replace("\r", "") + line)
+ del buffer[:]
+ if line:
+ history.append(line)
+ self.set_history_length(self.get_current_history_length())
+
+ def write_history_file(self, filename: str = gethistoryfile()) -> None:
+ maxlength = self.saved_history_length
+ history = self.get_reader().get_trimmed_history(maxlength)
+ f = open(os.path.expanduser(filename), "w",
+ encoding="utf-8", newline="\n")
+ with f:
+ for entry in history:
+ entry = entry.replace("\n", "\r\n") # multiline history support
+ f.write(entry + "\n")
+
+ def append_history_file(self, filename: str = gethistoryfile()) -> None:
+ reader = self.get_reader()
+ saved_length = self.get_history_length()
+ length = self.get_current_history_length() - saved_length
+ history = reader.get_trimmed_history(length)
+ f = open(os.path.expanduser(filename), "a",
+ encoding="utf-8", newline="\n")
+ with f:
+ for entry in history:
+ entry = entry.replace("\n", "\r\n") # multiline history support
+ f.write(entry + "\n")
+ self.set_history_length(saved_length + length)
+
+ def clear_history(self) -> None:
+ del self.get_reader().history[:]
+
+ def get_history_item(self, index: int) -> str | None:
+ history = self.get_reader().history
+ if 1 <= index <= len(history):
+ return history[index - 1]
+ else:
+ return None # like readline.c
+
+ def remove_history_item(self, index: int) -> None:
+ history = self.get_reader().history
+ if 0 <= index < len(history):
+ del history[index]
+ else:
+ raise ValueError("No history item at position %d" % index)
+ # like readline.c
+
+ def replace_history_item(self, index: int, line: str) -> None:
+ history = self.get_reader().history
+ if 0 <= index < len(history):
+ history[index] = self._histline(line)
+ else:
+ raise ValueError("No history item at position %d" % index)
+ # like readline.c
+
+ def add_history(self, line: str) -> None:
+ self.get_reader().history.append(self._histline(line))
+
+ def set_startup_hook(self, function: Callback | None = None) -> None:
+ self.startup_hook = function
+
+ def get_line_buffer(self) -> str:
+ return self.get_reader().get_unicode()
+
+ def _get_idxs(self) -> tuple[int, int]:
+ start = cursor = self.get_reader().pos
+ buf = self.get_line_buffer()
+ for i in range(cursor - 1, -1, -1):
+ if buf[i] in self.get_completer_delims():
+ break
+ start = i
+ return start, cursor
+
+ def get_begidx(self) -> int:
+ return self._get_idxs()[0]
+
+ def get_endidx(self) -> int:
+ return self._get_idxs()[1]
+
+ def insert_text(self, text: str) -> None:
+ self.get_reader().insert(text)
+
+
+_wrapper = _ReadlineWrapper()
+
+# ____________________________________________________________
+# Public API
+
+parse_and_bind = _wrapper.parse_and_bind
+set_completer = _wrapper.set_completer
+get_completer = _wrapper.get_completer
+set_completer_delims = _wrapper.set_completer_delims
+get_completer_delims = _wrapper.get_completer_delims
+get_history_length = _wrapper.get_history_length
+set_history_length = _wrapper.set_history_length
+get_current_history_length = _wrapper.get_current_history_length
+read_history_file = _wrapper.read_history_file
+write_history_file = _wrapper.write_history_file
+append_history_file = _wrapper.append_history_file
+clear_history = _wrapper.clear_history
+get_history_item = _wrapper.get_history_item
+remove_history_item = _wrapper.remove_history_item
+replace_history_item = _wrapper.replace_history_item
+add_history = _wrapper.add_history
+set_startup_hook = _wrapper.set_startup_hook
+get_line_buffer = _wrapper.get_line_buffer
+get_begidx = _wrapper.get_begidx
+get_endidx = _wrapper.get_endidx
+insert_text = _wrapper.insert_text
+
+# Extension
+multiline_input = _wrapper.multiline_input
+
+# Internal hook
+_get_reader = _wrapper.get_reader
+
+# ____________________________________________________________
+# Stubs
+
+
+def _make_stub(_name: str, _ret: object) -> None:
+ def stub(*args: object, **kwds: object) -> None:
+ import warnings
+
+ warnings.warn("readline.%s() not implemented" % _name, stacklevel=2)
+
+ stub.__name__ = _name
+ globals()[_name] = stub
+
+
+for _name, _ret in [
+ ("read_init_file", None),
+ ("redisplay", None),
+ ("set_pre_input_hook", None),
+]:
+ assert _name not in globals(), _name
+ _make_stub(_name, _ret)
+
+# ____________________________________________________________
+
+
+def _setup(namespace: Mapping[str, Any]) -> None:
+ global raw_input
+ if raw_input is not None:
+ return # don't run _setup twice
+
+ try:
+ f_in = sys.stdin.fileno()
+ f_out = sys.stdout.fileno()
+ except (AttributeError, ValueError):
+ return
+ if not os.isatty(f_in) or not os.isatty(f_out):
+ return
+
+ _wrapper.f_in = f_in
+ _wrapper.f_out = f_out
+
+ # set up namespace in rlcompleter, which requires it to be a bona fide dict
+ if not isinstance(namespace, dict):
+ namespace = dict(namespace)
+ _wrapper.config.module_completer = ModuleCompleter(namespace)
+ _wrapper.config.readline_completer = RLCompleter(namespace).complete
+
+ # this is not really what readline.c does. Better than nothing I guess
+ import builtins
+ raw_input = builtins.input
+ builtins.input = _wrapper.input
+
+
+raw_input: Callable[[object], str] | None = None
diff --git a/PythonLib/full/_pyrepl/simple_interact.py b/PythonLib/full/_pyrepl/simple_interact.py
new file mode 100644
index 000000000..6508f0233
--- /dev/null
+++ b/PythonLib/full/_pyrepl/simple_interact.py
@@ -0,0 +1,181 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+"""This is an alternative to python_reader which tries to emulate
+the CPython prompt as closely as possible, with the exception of
+allowing multiline input and multiline history entries.
+"""
+
+from __future__ import annotations
+
+import _sitebuiltins
+import functools
+import os
+import sys
+import code
+import warnings
+import errno
+
+from .readline import _get_reader, multiline_input, append_history_file
+
+
+_error: tuple[type[Exception], ...] | type[Exception]
+try:
+ from .unix_console import _error
+except ModuleNotFoundError:
+ from .windows_console import _error
+
+def check() -> str:
+ """Returns the error message if there is a problem initializing the state."""
+ try:
+ _get_reader()
+ except _error as e:
+ if term := os.environ.get("TERM", ""):
+ term = f"; TERM={term}"
+ return str(str(e) or repr(e) or "unknown error") + term
+ return ""
+
+
+def _strip_final_indent(text: str) -> str:
+ # kill spaces and tabs at the end, but only if they follow '\n'.
+ # meant to remove the auto-indentation only (although it would of
+ # course also remove explicitly-added indentation).
+ short = text.rstrip(" \t")
+ n = len(short)
+ if n > 0 and text[n - 1] == "\n":
+ return short
+ return text
+
+
+def _clear_screen():
+ reader = _get_reader()
+ reader.scheduled_commands.append("clear_screen")
+
+
+REPL_COMMANDS = {
+ "exit": _sitebuiltins.Quitter('exit', ''),
+ "quit": _sitebuiltins.Quitter('quit' ,''),
+ "copyright": _sitebuiltins._Printer('copyright', sys.copyright),
+ "help": _sitebuiltins._Helper(),
+ "clear": _clear_screen,
+ "\x1a": _sitebuiltins.Quitter('\x1a', ''),
+}
+
+
+def _more_lines(console: code.InteractiveConsole, unicodetext: str) -> bool:
+ # ooh, look at the hack:
+ src = _strip_final_indent(unicodetext)
+ try:
+ code = console.compile(src, "", "single")
+ except (OverflowError, SyntaxError, ValueError):
+ lines = src.splitlines(keepends=True)
+ if len(lines) == 1:
+ return False
+
+ last_line = lines[-1]
+ was_indented = last_line.startswith((" ", "\t"))
+ not_empty = last_line.strip() != ""
+ incomplete = not last_line.endswith("\n")
+ return (was_indented or not_empty) and incomplete
+ else:
+ return code is None
+
+
+def run_multiline_interactive_console(
+ console: code.InteractiveConsole,
+ *,
+ future_flags: int = 0,
+) -> None:
+ from .readline import _setup
+ _setup(console.locals)
+ if future_flags:
+ console.compile.compiler.flags |= future_flags
+
+ more_lines = functools.partial(_more_lines, console)
+ input_n = 0
+
+ _is_x_showrefcount_set = sys._xoptions.get("showrefcount")
+ _is_pydebug_build = hasattr(sys, "gettotalrefcount")
+ show_ref_count = _is_x_showrefcount_set and _is_pydebug_build
+
+ def maybe_run_command(statement: str) -> bool:
+ statement = statement.strip()
+ if statement in console.locals or statement not in REPL_COMMANDS:
+ return False
+
+ reader = _get_reader()
+ reader.history.pop() # skip internal commands in history
+ command = REPL_COMMANDS[statement]
+ if callable(command):
+ # Make sure that history does not change because of commands
+ with reader.suspend_history(), reader.suspend_colorization():
+ command()
+ return True
+ return False
+
+ while True:
+ try:
+ try:
+ sys.stdout.flush()
+ except Exception:
+ pass
+
+ ps1 = getattr(sys, "ps1", ">>> ")
+ ps2 = getattr(sys, "ps2", "... ")
+ try:
+ statement = multiline_input(more_lines, ps1, ps2)
+ except EOFError:
+ break
+
+ if maybe_run_command(statement):
+ continue
+
+ input_name = f""
+ more = console.push(_strip_final_indent(statement), filename=input_name, _symbol="single") # type: ignore[call-arg]
+ assert not more
+ try:
+ append_history_file()
+ except (FileNotFoundError, PermissionError, OSError) as e:
+ warnings.warn(f"failed to open the history file for writing: {e}")
+
+ input_n += 1
+ except KeyboardInterrupt:
+ r = _get_reader()
+ r.cmpltn_reset()
+ if r.input_trans is r.isearch_trans:
+ r.do_cmd(("isearch-end", [""]))
+ r.pos = len(r.get_unicode())
+ r.dirty = True
+ r.refresh()
+ console.write("\nKeyboardInterrupt\n")
+ console.resetbuffer()
+ except MemoryError:
+ console.write("\nMemoryError\n")
+ console.resetbuffer()
+ except SystemExit:
+ raise
+ except:
+ console.showtraceback()
+ console.resetbuffer()
+ if show_ref_count:
+ console.write(
+ f"[{sys.gettotalrefcount()} refs,"
+ f" {sys.getallocatedblocks()} blocks]\n"
+ )
diff --git a/PythonLib/full/_pyrepl/terminfo.py b/PythonLib/full/_pyrepl/terminfo.py
new file mode 100644
index 000000000..d02ef69cc
--- /dev/null
+++ b/PythonLib/full/_pyrepl/terminfo.py
@@ -0,0 +1,488 @@
+"""Pure Python curses-like terminal capability queries."""
+
+from dataclasses import dataclass, field
+import errno
+import os
+from pathlib import Path
+import re
+import struct
+
+
+# Terminfo constants
+MAGIC16 = 0o432 # Magic number for 16-bit terminfo format
+MAGIC32 = 0o1036 # Magic number for 32-bit terminfo format
+
+# Special values for absent/cancelled capabilities
+ABSENT_BOOLEAN = -1
+ABSENT_NUMERIC = -1
+CANCELLED_NUMERIC = -2
+ABSENT_STRING = None
+CANCELLED_STRING = None
+
+
+# Standard string capability names from ncurses Caps file
+# This matches the order used by ncurses when compiling terminfo
+# fmt: off
+_STRING_NAMES: tuple[str, ...] = (
+ "cbt", "bel", "cr", "csr", "tbc", "clear", "el", "ed", "hpa", "cmdch",
+ "cup", "cud1", "home", "civis", "cub1", "mrcup", "cnorm", "cuf1", "ll",
+ "cuu1", "cvvis", "dch1", "dl1", "dsl", "hd", "smacs", "blink", "bold",
+ "smcup", "smdc", "dim", "smir", "invis", "prot", "rev", "smso", "smul",
+ "ech", "rmacs", "sgr0", "rmcup", "rmdc", "rmir", "rmso", "rmul", "flash",
+ "ff", "fsl", "is1", "is2", "is3", "if", "ich1", "il1", "ip", "kbs", "ktbc",
+ "kclr", "kctab", "kdch1", "kdl1", "kcud1", "krmir", "kel", "ked", "kf0",
+ "kf1", "kf10", "kf2", "kf3", "kf4", "kf5", "kf6", "kf7", "kf8", "kf9",
+ "khome", "kich1", "kil1", "kcub1", "kll", "knp", "kpp", "kcuf1", "kind",
+ "kri", "khts", "kcuu1", "rmkx", "smkx", "lf0", "lf1", "lf10", "lf2", "lf3",
+ "lf4", "lf5", "lf6", "lf7", "lf8", "lf9", "rmm", "smm", "nel", "pad", "dch",
+ "dl", "cud", "ich", "indn", "il", "cub", "cuf", "rin", "cuu", "pfkey",
+ "pfloc", "pfx", "mc0", "mc4", "mc5", "rep", "rs1", "rs2", "rs3", "rf", "rc",
+ "vpa", "sc", "ind", "ri", "sgr", "hts", "wind", "ht", "tsl", "uc", "hu",
+ "iprog", "ka1", "ka3", "kb2", "kc1", "kc3", "mc5p", "rmp", "acsc", "pln",
+ "kcbt", "smxon", "rmxon", "smam", "rmam", "xonc", "xoffc", "enacs", "smln",
+ "rmln", "kbeg", "kcan", "kclo", "kcmd", "kcpy", "kcrt", "kend", "kent",
+ "kext", "kfnd", "khlp", "kmrk", "kmsg", "kmov", "knxt", "kopn", "kopt",
+ "kprv", "kprt", "krdo", "kref", "krfr", "krpl", "krst", "kres", "ksav",
+ "kspd", "kund", "kBEG", "kCAN", "kCMD", "kCPY", "kCRT", "kDC", "kDL",
+ "kslt", "kEND", "kEOL", "kEXT", "kFND", "kHLP", "kHOM", "kIC", "kLFT",
+ "kMSG", "kMOV", "kNXT", "kOPT", "kPRV", "kPRT", "kRDO", "kRPL", "kRIT",
+ "kRES", "kSAV", "kSPD", "kUND", "rfi", "kf11", "kf12", "kf13", "kf14",
+ "kf15", "kf16", "kf17", "kf18", "kf19", "kf20", "kf21", "kf22", "kf23",
+ "kf24", "kf25", "kf26", "kf27", "kf28", "kf29", "kf30", "kf31", "kf32",
+ "kf33", "kf34", "kf35", "kf36", "kf37", "kf38", "kf39", "kf40", "kf41",
+ "kf42", "kf43", "kf44", "kf45", "kf46", "kf47", "kf48", "kf49", "kf50",
+ "kf51", "kf52", "kf53", "kf54", "kf55", "kf56", "kf57", "kf58", "kf59",
+ "kf60", "kf61", "kf62", "kf63", "el1", "mgc", "smgl", "smgr", "fln", "sclk",
+ "dclk", "rmclk", "cwin", "wingo", "hup","dial", "qdial", "tone", "pulse",
+ "hook", "pause", "wait", "u0", "u1", "u2", "u3", "u4", "u5", "u6", "u7",
+ "u8", "u9", "op", "oc", "initc", "initp", "scp", "setf", "setb", "cpi",
+ "lpi", "chr", "cvr", "defc", "swidm", "sdrfq", "sitm", "slm", "smicm",
+ "snlq", "snrmq", "sshm", "ssubm", "ssupm", "sum", "rwidm", "ritm", "rlm",
+ "rmicm", "rshm", "rsubm", "rsupm", "rum", "mhpa", "mcud1", "mcub1", "mcuf1",
+ "mvpa", "mcuu1", "porder", "mcud", "mcub", "mcuf", "mcuu", "scs", "smgb",
+ "smgbp", "smglp", "smgrp", "smgt", "smgtp", "sbim", "scsd", "rbim", "rcsd",
+ "subcs", "supcs", "docr", "zerom", "csnm", "kmous", "minfo", "reqmp",
+ "getm", "setaf", "setab", "pfxl", "devt", "csin", "s0ds", "s1ds", "s2ds",
+ "s3ds", "smglr", "smgtb", "birep", "binel", "bicr", "colornm", "defbi",
+ "endbi", "setcolor", "slines", "dispc", "smpch", "rmpch", "smsc", "rmsc",
+ "pctrm", "scesc", "scesa", "ehhlm", "elhlm", "elohlm", "erhlm", "ethlm",
+ "evhlm", "sgr1", "slength", "OTi2", "OTrs", "OTnl", "OTbc", "OTko", "OTma",
+ "OTG2", "OTG3", "OTG1", "OTG4", "OTGR", "OTGL", "OTGU", "OTGD", "OTGH",
+ "OTGV", "OTGC","meml", "memu", "box1"
+)
+# fmt: on
+
+
+def _get_terminfo_dirs() -> list[Path]:
+ """Get list of directories to search for terminfo files.
+
+ Based on ncurses behavior in:
+ - ncurses/tinfo/db_iterator.c:_nc_next_db()
+ - ncurses/tinfo/read_entry.c:_nc_read_entry()
+ """
+ dirs = []
+
+ terminfo = os.environ.get("TERMINFO")
+ if terminfo:
+ dirs.append(terminfo)
+
+ try:
+ home = Path.home()
+ dirs.append(str(home / ".terminfo"))
+ except RuntimeError:
+ pass
+
+ # Check TERMINFO_DIRS
+ terminfo_dirs = os.environ.get("TERMINFO_DIRS", "")
+ if terminfo_dirs:
+ for d in terminfo_dirs.split(":"):
+ if d:
+ dirs.append(d)
+
+ dirs.extend(
+ [
+ "/etc/terminfo",
+ "/lib/terminfo",
+ "/usr/lib/terminfo",
+ "/usr/share/terminfo",
+ "/usr/share/lib/terminfo",
+ "/usr/share/misc/terminfo",
+ "/usr/local/lib/terminfo",
+ "/usr/local/share/terminfo",
+ ]
+ )
+
+ return [Path(d) for d in dirs if Path(d).is_dir()]
+
+
+def _validate_terminal_name_or_raise(terminal_name: str) -> None:
+ if not isinstance(terminal_name, str):
+ raise TypeError("`terminal_name` must be a string")
+
+ if not terminal_name:
+ raise ValueError("`terminal_name` cannot be empty")
+
+ if "\x00" in terminal_name:
+ raise ValueError("NUL character found in `terminal_name`")
+
+ t = Path(terminal_name)
+ if len(t.parts) > 1:
+ raise ValueError("`terminal_name` cannot contain path separators")
+
+
+def _read_terminfo_file(terminal_name: str) -> bytes:
+ """Find and read terminfo file for given terminal name.
+
+ Terminfo files are stored in directories using the first character
+ of the terminal name as a subdirectory.
+ """
+ _validate_terminal_name_or_raise(terminal_name)
+ first_char = terminal_name[0].lower()
+ filename = terminal_name
+
+ for directory in _get_terminfo_dirs():
+ path = directory / first_char / filename
+ if path.is_file():
+ return path.read_bytes()
+
+ # Try with hex encoding of first char (for special chars)
+ hex_dir = "%02x" % ord(first_char)
+ path = directory / hex_dir / filename
+ if path.is_file():
+ return path.read_bytes()
+
+ raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), filename)
+
+
+# Hard-coded terminal capabilities for common terminals
+# This is a minimal subset needed by PyREPL
+_TERMINAL_CAPABILITIES = {
+ # ANSI/xterm-compatible terminals
+ "ansi": {
+ # Bell
+ "bel": b"\x07",
+ # Cursor movement
+ "cub": b"\x1b[%p1%dD", # Move cursor left N columns
+ "cud": b"\x1b[%p1%dB", # Move cursor down N rows
+ "cuf": b"\x1b[%p1%dC", # Move cursor right N columns
+ "cuu": b"\x1b[%p1%dA", # Move cursor up N rows
+ "cub1": b"\x08", # Move cursor left 1 column
+ "cud1": b"\n", # Move cursor down 1 row
+ "cuf1": b"\x1b[C", # Move cursor right 1 column
+ "cuu1": b"\x1b[A", # Move cursor up 1 row
+ "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column
+ "hpa": b"\x1b[%i%p1%dG", # Move cursor to column
+ # Clear operations
+ "clear": b"\x1b[H\x1b[2J", # Clear screen and home cursor
+ "el": b"\x1b[K", # Clear to end of line
+ # Insert/delete
+ "dch": b"\x1b[%p1%dP", # Delete N characters
+ "dch1": b"\x1b[P", # Delete 1 character
+ "ich": b"\x1b[%p1%d@", # Insert N characters
+ "ich1": b"", # Insert 1 character
+ # Cursor visibility
+ "civis": b"\x1b[?25l", # Make cursor invisible
+ "cnorm": b"\x1b[?12l\x1b[?25h", # Make cursor normal (visible)
+ # Scrolling
+ "ind": b"\n", # Scroll up one line
+ "ri": b"\x1bM", # Scroll down one line
+ # Keypad mode
+ "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode
+ "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode
+ # Padding (not used in modern terminals)
+ "pad": b"",
+ # Function keys and special keys
+ "kdch1": b"\x1b[3~", # Delete key
+ "kcud1": b"\x1bOB", # Down arrow
+ "kend": b"\x1bOF", # End key
+ "kent": b"\x1bOM", # Enter key
+ "khome": b"\x1bOH", # Home key
+ "kich1": b"\x1b[2~", # Insert key
+ "kcub1": b"\x1bOD", # Left arrow
+ "knp": b"\x1b[6~", # Page down
+ "kpp": b"\x1b[5~", # Page up
+ "kcuf1": b"\x1bOC", # Right arrow
+ "kcuu1": b"\x1bOA", # Up arrow
+ # Function keys F1-F20
+ "kf1": b"\x1bOP",
+ "kf2": b"\x1bOQ",
+ "kf3": b"\x1bOR",
+ "kf4": b"\x1bOS",
+ "kf5": b"\x1b[15~",
+ "kf6": b"\x1b[17~",
+ "kf7": b"\x1b[18~",
+ "kf8": b"\x1b[19~",
+ "kf9": b"\x1b[20~",
+ "kf10": b"\x1b[21~",
+ "kf11": b"\x1b[23~",
+ "kf12": b"\x1b[24~",
+ "kf13": b"\x1b[1;2P",
+ "kf14": b"\x1b[1;2Q",
+ "kf15": b"\x1b[1;2R",
+ "kf16": b"\x1b[1;2S",
+ "kf17": b"\x1b[15;2~",
+ "kf18": b"\x1b[17;2~",
+ "kf19": b"\x1b[18;2~",
+ "kf20": b"\x1b[19;2~",
+ },
+ # Dumb terminal - minimal capabilities
+ "dumb": {
+ "bel": b"\x07", # Bell
+ "cud1": b"\n", # Move down 1 row (newline)
+ "ind": b"\n", # Scroll up one line (newline)
+ },
+ # Linux console
+ "linux": {
+ # Bell
+ "bel": b"\x07",
+ # Cursor movement
+ "cub": b"\x1b[%p1%dD", # Move cursor left N columns
+ "cud": b"\x1b[%p1%dB", # Move cursor down N rows
+ "cuf": b"\x1b[%p1%dC", # Move cursor right N columns
+ "cuu": b"\x1b[%p1%dA", # Move cursor up N rows
+ "cub1": b"\x08", # Move cursor left 1 column (backspace)
+ "cud1": b"\n", # Move cursor down 1 row (newline)
+ "cuf1": b"\x1b[C", # Move cursor right 1 column
+ "cuu1": b"\x1b[A", # Move cursor up 1 row
+ "cup": b"\x1b[%i%p1%d;%p2%dH", # Move cursor to row, column
+ "hpa": b"\x1b[%i%p1%dG", # Move cursor to column
+ # Clear operations
+ "clear": b"\x1b[H\x1b[J", # Clear screen and home cursor (different from ansi!)
+ "el": b"\x1b[K", # Clear to end of line
+ # Insert/delete
+ "dch": b"\x1b[%p1%dP", # Delete N characters
+ "dch1": b"\x1b[P", # Delete 1 character
+ "ich": b"\x1b[%p1%d@", # Insert N characters
+ "ich1": b"\x1b[@", # Insert 1 character
+ # Cursor visibility
+ "civis": b"\x1b[?25l\x1b[?1c", # Make cursor invisible
+ "cnorm": b"\x1b[?25h\x1b[?0c", # Make cursor normal
+ # Scrolling
+ "ind": b"\n", # Scroll up one line
+ "ri": b"\x1bM", # Scroll down one line
+ # Keypad mode
+ "smkx": b"\x1b[?1h\x1b=", # Enable keypad mode
+ "rmkx": b"\x1b[?1l\x1b>", # Disable keypad mode
+ # Function keys and special keys
+ "kdch1": b"\x1b[3~", # Delete key
+ "kcud1": b"\x1b[B", # Down arrow
+ "kend": b"\x1b[4~", # End key (different from ansi!)
+ "khome": b"\x1b[1~", # Home key (different from ansi!)
+ "kich1": b"\x1b[2~", # Insert key
+ "kcub1": b"\x1b[D", # Left arrow
+ "knp": b"\x1b[6~", # Page down
+ "kpp": b"\x1b[5~", # Page up
+ "kcuf1": b"\x1b[C", # Right arrow
+ "kcuu1": b"\x1b[A", # Up arrow
+ # Function keys
+ "kf1": b"\x1b[[A",
+ "kf2": b"\x1b[[B",
+ "kf3": b"\x1b[[C",
+ "kf4": b"\x1b[[D",
+ "kf5": b"\x1b[[E",
+ "kf6": b"\x1b[17~",
+ "kf7": b"\x1b[18~",
+ "kf8": b"\x1b[19~",
+ "kf9": b"\x1b[20~",
+ "kf10": b"\x1b[21~",
+ "kf11": b"\x1b[23~",
+ "kf12": b"\x1b[24~",
+ "kf13": b"\x1b[25~",
+ "kf14": b"\x1b[26~",
+ "kf15": b"\x1b[28~",
+ "kf16": b"\x1b[29~",
+ "kf17": b"\x1b[31~",
+ "kf18": b"\x1b[32~",
+ "kf19": b"\x1b[33~",
+ "kf20": b"\x1b[34~",
+ },
+}
+
+# Map common TERM values to capability sets
+_TERM_ALIASES = {
+ "xterm": "ansi",
+ "xterm-color": "ansi",
+ "xterm-256color": "ansi",
+ "screen": "ansi",
+ "screen-256color": "ansi",
+ "tmux": "ansi",
+ "tmux-256color": "ansi",
+ "vt100": "ansi",
+ "vt220": "ansi",
+ "rxvt": "ansi",
+ "rxvt-unicode": "ansi",
+ "rxvt-unicode-256color": "ansi",
+ "unknown": "dumb",
+}
+
+
+@dataclass
+class TermInfo:
+ terminal_name: str | bytes | None
+ fallback: bool = True
+
+ _capabilities: dict[str, bytes] = field(default_factory=dict)
+
+ def __post_init__(self) -> None:
+ """Initialize terminal capabilities for the given terminal type.
+
+ Based on ncurses implementation in:
+ - ncurses/tinfo/lib_setup.c:setupterm() and _nc_setupterm()
+ - ncurses/tinfo/lib_setup.c:TINFO_SETUP_TERM()
+
+ This version first attempts to read terminfo database files like ncurses,
+ then, if `fallback` is True, falls back to hardcoded capabilities for
+ common terminal types.
+ """
+ # If termstr is None or empty, try to get from environment
+ if not self.terminal_name:
+ self.terminal_name = os.environ.get("TERM") or "ANSI"
+
+ if isinstance(self.terminal_name, bytes):
+ self.terminal_name = self.terminal_name.decode("ascii")
+
+ try:
+ self._parse_terminfo_file(self.terminal_name)
+ except (OSError, ValueError):
+ if not self.fallback:
+ raise
+
+ term_type = _TERM_ALIASES.get(
+ self.terminal_name, self.terminal_name
+ )
+ if term_type not in _TERMINAL_CAPABILITIES:
+ term_type = "dumb"
+ self._capabilities = _TERMINAL_CAPABILITIES[term_type].copy()
+
+ def _parse_terminfo_file(self, terminal_name: str) -> None:
+ """Parse a terminfo file.
+
+ Populate the _capabilities dict for easy retrieval
+
+ Based on ncurses implementation in:
+ - ncurses/tinfo/read_entry.c:_nc_read_termtype()
+ - ncurses/tinfo/read_entry.c:_nc_read_file_entry()
+ - ncurses/tinfo/lib_ti.c:tigetstr()
+ """
+ data = _read_terminfo_file(terminal_name)
+ too_short = f"TermInfo file for {terminal_name!r} too short"
+ offset = 12
+ if len(data) < offset:
+ raise ValueError(too_short)
+
+ magic, name_size, bool_count, num_count, str_count, str_size = (
+ struct.unpack(" len(data):
+ raise ValueError(too_short)
+
+ # Read string offsets
+ end_offset = offset + 2 * str_count
+ if offset > len(data):
+ raise ValueError(too_short)
+ string_offset_data = data[offset:end_offset]
+ string_offsets = [
+ off for [off] in struct.iter_unpack(" len(data):
+ raise ValueError(too_short)
+ string_table = data[offset : offset + str_size]
+
+ # Extract strings from string table
+ capabilities = {}
+ for cap, off in zip(_STRING_NAMES, string_offsets):
+ if off < 0:
+ # CANCELLED_STRING; we do not store those
+ continue
+ elif off < len(string_table):
+ # Find null terminator
+ end = string_table.find(0, off)
+ if end >= 0:
+ capabilities[cap] = string_table[off:end]
+ # in other cases this is ABSENT_STRING; we don't store those.
+
+ # Note: we don't support extended capabilities since PyREPL doesn't
+ # need them.
+
+ self._capabilities = capabilities
+
+ def get(self, cap: str) -> bytes | None:
+ """Get terminal capability string by name.
+ """
+ if not isinstance(cap, str):
+ raise TypeError(f"`cap` must be a string, not {type(cap)}")
+
+ return self._capabilities.get(cap)
+
+
+def tparm(cap_bytes: bytes, *params: int) -> bytes:
+ """Parameterize a terminal capability string.
+
+ Based on ncurses implementation in:
+ - ncurses/tinfo/lib_tparm.c:tparm()
+ - ncurses/tinfo/lib_tparm.c:tparam_internal()
+
+ The ncurses version implements a full stack-based interpreter for
+ terminfo parameter strings. This pure Python version implements only
+ the subset of parameter substitution operations needed by PyREPL:
+ - %i (increment parameters for 1-based indexing)
+ - %p[1-9]%d (parameter substitution)
+ - %p[1-9]%{n}%+%d (parameter plus constant)
+ """
+ if not isinstance(cap_bytes, bytes):
+ raise TypeError(f"`cap` must be bytes, not {type(cap_bytes)}")
+
+ result = cap_bytes
+
+ # %i - increment parameters (1-based instead of 0-based)
+ increment = b"%i" in result
+ if increment:
+ result = result.replace(b"%i", b"")
+
+ # Replace %p1%d, %p2%d, etc. with actual parameter values
+ for i in range(len(params)):
+ pattern = b"%%p%d%%d" % (i + 1)
+ if pattern in result:
+ value = params[i]
+ if increment:
+ value += 1
+ result = result.replace(pattern, str(value).encode("ascii"))
+
+ # Handle %p1%{1}%+%d (parameter plus constant)
+ # Used in some cursor positioning sequences
+ pattern_re = re.compile(rb"%p(\d)%\{(\d+)\}%\+%d")
+ matches = list(pattern_re.finditer(result))
+ for match in reversed(matches): # reversed to maintain positions
+ param_idx = int(match.group(1))
+ constant = int(match.group(2))
+ value = params[param_idx] + constant
+ result = (
+ result[: match.start()]
+ + str(value).encode("ascii")
+ + result[match.end() :]
+ )
+
+ return result
diff --git a/PythonLib/full/_pyrepl/trace.py b/PythonLib/full/_pyrepl/trace.py
new file mode 100644
index 000000000..943ee12f9
--- /dev/null
+++ b/PythonLib/full/_pyrepl/trace.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import os
+import sys
+
+# types
+if False:
+ from typing import IO
+
+
+trace_file: IO[str] | None = None
+if trace_filename := os.environ.get("PYREPL_TRACE"):
+ trace_file = open(trace_filename, "a")
+
+
+
+if sys.platform == "emscripten":
+ from posix import _emscripten_log
+
+ def trace(line: str, *k: object, **kw: object) -> None:
+ if "PYREPL_TRACE" not in os.environ:
+ return
+ if k or kw:
+ line = line.format(*k, **kw)
+ _emscripten_log(line)
+
+else:
+ def trace(line: str, *k: object, **kw: object) -> None:
+ if trace_file is None:
+ return
+ if k or kw:
+ line = line.format(*k, **kw)
+ trace_file.write(line + "\n")
+ trace_file.flush()
diff --git a/PythonLib/full/_pyrepl/types.py b/PythonLib/full/_pyrepl/types.py
new file mode 100644
index 000000000..c5b7ebc1a
--- /dev/null
+++ b/PythonLib/full/_pyrepl/types.py
@@ -0,0 +1,10 @@
+from collections.abc import Callable, Iterator
+
+type Callback = Callable[[], object]
+type SimpleContextManager = Iterator[None]
+type KeySpec = str # like r"\C-c"
+type CommandName = str # like "interrupt"
+type EventTuple = tuple[CommandName, str]
+type Completer = Callable[[str, int], str | None]
+type CharBuffer = list[str]
+type CharWidths = list[int]
diff --git a/PythonLib/full/_pyrepl/unix_console.py b/PythonLib/full/_pyrepl/unix_console.py
new file mode 100644
index 000000000..639d16db3
--- /dev/null
+++ b/PythonLib/full/_pyrepl/unix_console.py
@@ -0,0 +1,842 @@
+# Copyright 2000-2010 Michael Hudson-Doyle
+# Antonio Cuni
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import errno
+import os
+import re
+import select
+import signal
+import struct
+import termios
+import time
+import types
+import platform
+from fcntl import ioctl
+
+from . import terminfo
+from .console import Console, Event
+from .fancy_termios import tcgetattr, tcsetattr, TermState
+from .trace import trace
+from .unix_eventqueue import EventQueue
+from .utils import wlen
+
+# declare posix optional to allow None assignment on other platforms
+posix: types.ModuleType | None
+try:
+ import posix
+except ImportError:
+ posix = None
+
+TYPE_CHECKING = False
+
+# types
+if TYPE_CHECKING:
+ from typing import AbstractSet, IO, Literal, overload, cast
+else:
+ overload = lambda func: None
+ cast = lambda typ, val: val
+
+
+class InvalidTerminal(RuntimeError):
+ def __init__(self, message: str) -> None:
+ super().__init__(errno.EIO, message)
+
+
+_error = (termios.error, InvalidTerminal)
+_error_codes_to_ignore = frozenset([errno.EIO, errno.ENXIO, errno.EPERM])
+
+SIGWINCH_EVENT = "repaint"
+
+FIONREAD = getattr(termios, "FIONREAD", None)
+TIOCGWINSZ = getattr(termios, "TIOCGWINSZ", None)
+
+# ------------ start of baudrate definitions ------------
+
+# Add (possibly) missing baudrates (check termios man page) to termios
+
+
+def add_baudrate_if_supported(dictionary: dict[int, int], rate: int) -> None:
+ baudrate_name = "B%d" % rate
+ if hasattr(termios, baudrate_name):
+ dictionary[getattr(termios, baudrate_name)] = rate
+
+
+# Check the termios man page (Line speed) to know where these
+# values come from.
+potential_baudrates = [
+ 0,
+ 110,
+ 115200,
+ 1200,
+ 134,
+ 150,
+ 1800,
+ 19200,
+ 200,
+ 230400,
+ 2400,
+ 300,
+ 38400,
+ 460800,
+ 4800,
+ 50,
+ 57600,
+ 600,
+ 75,
+ 9600,
+]
+
+ratedict: dict[int, int] = {}
+for rate in potential_baudrates:
+ add_baudrate_if_supported(ratedict, rate)
+
+# Clean up variables to avoid unintended usage
+del rate, add_baudrate_if_supported
+
+# ------------ end of baudrate definitions ------------
+
+delayprog = re.compile(b"\\$<([0-9]+)((?:/|\\*){0,2})>")
+
+try:
+ poll: type[select.poll] = select.poll
+except AttributeError:
+ # this is exactly the minimum necessary to support what we
+ # do with poll objects
+ class MinimalPoll:
+ def __init__(self):
+ pass
+
+ def register(self, fd, flag):
+ self.fd = fd
+
+ # note: The 'timeout' argument is received as *milliseconds*
+ def poll(self, timeout: float | None = None) -> list[int]:
+ if timeout is None:
+ r, w, e = select.select([self.fd], [], [])
+ else:
+ r, w, e = select.select([self.fd], [], [], timeout / 1000)
+ return r
+
+ poll = MinimalPoll # type: ignore[assignment]
+
+
+class UnixConsole(Console):
+ def __init__(
+ self,
+ f_in: IO[bytes] | int = 0,
+ f_out: IO[bytes] | int = 1,
+ term: str = "",
+ encoding: str = "",
+ ):
+ """
+ Initialize the UnixConsole.
+
+ Parameters:
+ - f_in (int or file-like object): Input file descriptor or object.
+ - f_out (int or file-like object): Output file descriptor or object.
+ - term (str): Terminal name.
+ - encoding (str): Encoding to use for I/O operations.
+ """
+ super().__init__(f_in, f_out, term, encoding)
+
+ self.pollob = poll()
+ self.pollob.register(self.input_fd, select.POLLIN)
+ self.terminfo = terminfo.TermInfo(term or None)
+ self.term = term
+ self.is_apple_terminal = (
+ platform.system() == "Darwin"
+ and os.getenv("TERM_PROGRAM") == "Apple_Terminal"
+ )
+
+ try:
+ self.__input_fd_set(tcgetattr(self.input_fd), ignore=frozenset())
+ except _error as e:
+ raise RuntimeError(f"termios failure ({e.args[1]})")
+
+ @overload
+ def _my_getstr(
+ cap: str, optional: Literal[False] = False
+ ) -> bytes: ...
+
+ @overload
+ def _my_getstr(cap: str, optional: bool) -> bytes | None: ...
+
+ def _my_getstr(cap: str, optional: bool = False) -> bytes | None:
+ r = self.terminfo.get(cap)
+ if not optional and r is None:
+ raise InvalidTerminal(
+ f"terminal doesn't have the required {cap} capability"
+ )
+ return r
+
+ self._bel = _my_getstr("bel")
+ self._civis = _my_getstr("civis", optional=True)
+ self._clear = _my_getstr("clear")
+ self._cnorm = _my_getstr("cnorm", optional=True)
+ self._cub = _my_getstr("cub", optional=True)
+ self._cub1 = _my_getstr("cub1", optional=True)
+ self._cud = _my_getstr("cud", optional=True)
+ self._cud1 = _my_getstr("cud1", optional=True)
+ self._cuf = _my_getstr("cuf", optional=True)
+ self._cuf1 = _my_getstr("cuf1", optional=True)
+ self._cup = _my_getstr("cup")
+ self._cuu = _my_getstr("cuu", optional=True)
+ self._cuu1 = _my_getstr("cuu1", optional=True)
+ self._dch1 = _my_getstr("dch1", optional=True)
+ self._dch = _my_getstr("dch", optional=True)
+ self._el = _my_getstr("el")
+ self._hpa = _my_getstr("hpa", optional=True)
+ self._ich = _my_getstr("ich", optional=True)
+ self._ich1 = _my_getstr("ich1", optional=True)
+ self._ind = _my_getstr("ind", optional=True)
+ self._pad = _my_getstr("pad", optional=True)
+ self._ri = _my_getstr("ri", optional=True)
+ self._rmkx = _my_getstr("rmkx", optional=True)
+ self._smkx = _my_getstr("smkx", optional=True)
+
+ self.__setup_movement()
+
+ self.event_queue = EventQueue(
+ self.input_fd, self.encoding, self.terminfo
+ )
+ self.cursor_visible = 1
+
+ signal.signal(signal.SIGCONT, self._sigcont_handler)
+
+ def _sigcont_handler(self, signum, frame):
+ self.restore()
+ self.prepare()
+
+ def __read(self, n: int) -> bytes:
+ return os.read(self.input_fd, n)
+
+ def change_encoding(self, encoding: str) -> None:
+ """
+ Change the encoding used for I/O operations.
+
+ Parameters:
+ - encoding (str): New encoding to use.
+ """
+ self.encoding = encoding
+
+ def refresh(self, screen, c_xy):
+ """
+ Refresh the console screen.
+
+ Parameters:
+ - screen (list): List of strings representing the screen contents.
+ - c_xy (tuple): Cursor position (x, y) on the screen.
+ """
+ cx, cy = c_xy
+ if not self.__gone_tall:
+ while len(self.screen) < min(len(screen), self.height):
+ self.__hide_cursor()
+ if self.screen:
+ self.__move(0, len(self.screen) - 1)
+ self.__write("\n")
+ self.posxy = 0, len(self.screen)
+ self.screen.append("")
+ else:
+ while len(self.screen) < len(screen):
+ self.screen.append("")
+
+ if len(screen) > self.height:
+ self.__gone_tall = 1
+ self.__move = self.__move_tall
+
+ px, py = self.posxy
+ old_offset = offset = self.__offset
+ height = self.height
+
+ # we make sure the cursor is on the screen, and that we're
+ # using all of the screen if we can
+ if cy < offset:
+ offset = cy
+ elif cy >= offset + height:
+ offset = cy - height + 1
+ elif offset > 0 and len(screen) < offset + height:
+ offset = max(len(screen) - height, 0)
+ screen.append("")
+
+ oldscr = self.screen[old_offset : old_offset + height]
+ newscr = screen[offset : offset + height]
+
+ # use hardware scrolling if we have it.
+ if old_offset > offset and self._ri:
+ self.__hide_cursor()
+ self.__write_code(self._cup, 0, 0)
+ self.posxy = 0, old_offset
+ for i in range(old_offset - offset):
+ self.__write_code(self._ri)
+ oldscr.pop(-1)
+ oldscr.insert(0, "")
+ elif old_offset < offset and self._ind:
+ self.__hide_cursor()
+ self.__write_code(self._cup, self.height - 1, 0)
+ self.posxy = 0, old_offset + self.height - 1
+ for i in range(offset - old_offset):
+ self.__write_code(self._ind)
+ oldscr.pop(0)
+ oldscr.append("")
+
+ self.__offset = offset
+
+ for (
+ y,
+ oldline,
+ newline,
+ ) in zip(range(offset, offset + height), oldscr, newscr):
+ if oldline != newline:
+ self.__write_changed_line(y, oldline, newline, px)
+
+ y = len(newscr)
+ while y < len(oldscr):
+ self.__hide_cursor()
+ self.__move(0, y)
+ self.posxy = 0, y
+ self.__write_code(self._el)
+ y += 1
+
+ self.__show_cursor()
+
+ self.screen = screen.copy()
+ self.move_cursor(cx, cy)
+ self.flushoutput()
+
+ def move_cursor(self, x, y):
+ """
+ Move the cursor to the specified position on the screen.
+
+ Parameters:
+ - x (int): X coordinate.
+ - y (int): Y coordinate.
+ """
+ if y < self.__offset or y >= self.__offset + self.height:
+ self.event_queue.insert(Event("scroll", None))
+ else:
+ self.__move(x, y)
+ self.posxy = x, y
+ self.flushoutput()
+
+ def prepare(self):
+ """
+ Prepare the console for input/output operations.
+ """
+ self.__buffer = []
+
+ self.__svtermstate = tcgetattr(self.input_fd)
+ raw = self.__svtermstate.copy()
+ raw.iflag &= ~(termios.INPCK | termios.ISTRIP | termios.IXON)
+ raw.oflag &= ~(termios.OPOST)
+ raw.cflag &= ~(termios.CSIZE | termios.PARENB)
+ raw.cflag |= termios.CS8
+ raw.iflag |= termios.BRKINT
+ raw.lflag &= ~(termios.ICANON | termios.ECHO | termios.IEXTEN)
+ raw.lflag |= termios.ISIG
+ raw.cc[termios.VMIN] = 1
+ raw.cc[termios.VTIME] = 0
+ self.__input_fd_set(raw)
+
+ # In macOS terminal we need to deactivate line wrap via ANSI escape code
+ if self.is_apple_terminal:
+ os.write(self.output_fd, b"\033[?7l")
+
+ self.screen = []
+ self.height, self.width = self.getheightwidth()
+
+ self.posxy = 0, 0
+ self.__gone_tall = 0
+ self.__move = self.__move_short
+ self.__offset = 0
+
+ self.__maybe_write_code(self._smkx)
+
+ try:
+ self.old_sigwinch = signal.signal(signal.SIGWINCH, self.__sigwinch)
+ except ValueError:
+ pass
+
+ self.__enable_bracketed_paste()
+
+ def restore(self):
+ """
+ Restore the console to the default state
+ """
+ self.__disable_bracketed_paste()
+ self.__maybe_write_code(self._rmkx)
+ self.flushoutput()
+ self.__input_fd_set(self.__svtermstate)
+
+ if self.is_apple_terminal:
+ os.write(self.output_fd, b"\033[?7h")
+
+ if hasattr(self, "old_sigwinch"):
+ try:
+ signal.signal(signal.SIGWINCH, self.old_sigwinch)
+ except ValueError as e:
+ import threading
+ if threading.current_thread() is threading.main_thread():
+ raise e
+ del self.old_sigwinch
+
+ def push_char(self, char: int | bytes) -> None:
+ """
+ Push a character to the console event queue.
+ """
+ trace("push char {char!r}", char=char)
+ self.event_queue.push(char)
+
+ def get_event(self, block: bool = True) -> Event | None:
+ """
+ Get an event from the console event queue.
+
+ Parameters:
+ - block (bool): Whether to block until an event is available.
+
+ Returns:
+ - Event: Event object from the event queue.
+ """
+ if not block and not self.wait(timeout=0):
+ return None
+
+ while self.event_queue.empty():
+ while True:
+ try:
+ self.push_char(self.__read(1))
+ except OSError as err:
+ if err.errno == errno.EINTR:
+ if not self.event_queue.empty():
+ return self.event_queue.get()
+ else:
+ continue
+ elif err.errno == errno.EIO:
+ raise SystemExit(errno.EIO)
+ else:
+ raise
+ else:
+ break
+ return self.event_queue.get()
+
+ def wait(self, timeout: float | None = None) -> bool:
+ """
+ Wait for events on the console.
+ """
+ return (
+ not self.event_queue.empty()
+ or bool(self.pollob.poll(timeout))
+ )
+
+ def set_cursor_vis(self, visible):
+ """
+ Set the visibility of the cursor.
+
+ Parameters:
+ - visible (bool): Visibility flag.
+ """
+ if visible:
+ self.__show_cursor()
+ else:
+ self.__hide_cursor()
+
+ if TIOCGWINSZ:
+
+ def getheightwidth(self):
+ """
+ Get the height and width of the console.
+
+ Returns:
+ - tuple: Height and width of the console.
+ """
+ try:
+ return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+ except (KeyError, TypeError, ValueError):
+ try:
+ size = ioctl(self.input_fd, TIOCGWINSZ, b"\000" * 8)
+ except OSError:
+ return 25, 80
+ height, width = struct.unpack("hhhh", size)[0:2]
+ if not height:
+ return 25, 80
+ return height, width
+
+ else:
+
+ def getheightwidth(self):
+ """
+ Get the height and width of the console.
+
+ Returns:
+ - tuple: Height and width of the console.
+ """
+ try:
+ return int(os.environ["LINES"]), int(os.environ["COLUMNS"])
+ except (KeyError, TypeError, ValueError):
+ return 25, 80
+
+ def forgetinput(self):
+ """
+ Discard any pending input on the console.
+ """
+ termios.tcflush(self.input_fd, termios.TCIFLUSH)
+
+ def flushoutput(self):
+ """
+ Flush the output buffer.
+ """
+ for text, iscode in self.__buffer:
+ if iscode:
+ self.__tputs(text)
+ else:
+ os.write(self.output_fd, text.encode(self.encoding, "replace"))
+ del self.__buffer[:]
+
+ def finish(self):
+ """
+ Finish console operations and flush the output buffer.
+ """
+ y = len(self.screen) - 1
+ while y >= 0 and not self.screen[y]:
+ y -= 1
+ self.__move(0, min(y, self.height + self.__offset - 1))
+ self.__write("\n\r")
+ self.flushoutput()
+
+ def beep(self):
+ """
+ Emit a beep sound.
+ """
+ self.__maybe_write_code(self._bel)
+ self.flushoutput()
+
+ if FIONREAD:
+
+ def getpending(self):
+ """
+ Get pending events from the console event queue.
+
+ Returns:
+ - Event: Pending event from the event queue.
+ """
+ e = Event("key", "", b"")
+
+ while not self.event_queue.empty():
+ e2 = self.event_queue.get()
+ e.data += e2.data
+ e.raw += e.raw
+
+ amount = struct.unpack("i", ioctl(self.input_fd, FIONREAD, b"\0\0\0\0"))[0]
+ trace("getpending({a})", a=amount)
+ raw = self.__read(amount)
+ data = str(raw, self.encoding, "replace")
+ e.data += data
+ e.raw += raw
+ return e
+
+ else:
+
+ def getpending(self):
+ """
+ Get pending events from the console event queue.
+
+ Returns:
+ - Event: Pending event from the event queue.
+ """
+ e = Event("key", "", b"")
+
+ while not self.event_queue.empty():
+ e2 = self.event_queue.get()
+ e.data += e2.data
+ e.raw += e.raw
+
+ amount = 10000
+ raw = self.__read(amount)
+ data = str(raw, self.encoding, "replace")
+ e.data += data
+ e.raw += raw
+ return e
+
+ def clear(self):
+ """
+ Clear the console screen.
+ """
+ self.__write_code(self._clear)
+ self.__gone_tall = 1
+ self.__move = self.__move_tall
+ self.posxy = 0, 0
+ self.screen = []
+
+ @property
+ def input_hook(self):
+ # avoid inline imports here so the repl doesn't get flooded
+ # with import logging from -X importtime=2
+ if posix is not None and posix._is_inputhook_installed():
+ return posix._inputhook
+
+ def __enable_bracketed_paste(self) -> None:
+ os.write(self.output_fd, b"\x1b[?2004h")
+
+ def __disable_bracketed_paste(self) -> None:
+ os.write(self.output_fd, b"\x1b[?2004l")
+
+ def __setup_movement(self):
+ """
+ Set up the movement functions based on the terminal capabilities.
+ """
+ if 0 and self._hpa: # hpa don't work in windows telnet :-(
+ self.__move_x = self.__move_x_hpa
+ elif self._cub and self._cuf:
+ self.__move_x = self.__move_x_cub_cuf
+ elif self._cub1 and self._cuf1:
+ self.__move_x = self.__move_x_cub1_cuf1
+ else:
+ raise RuntimeError("insufficient terminal (horizontal)")
+
+ if self._cuu and self._cud:
+ self.__move_y = self.__move_y_cuu_cud
+ elif self._cuu1 and self._cud1:
+ self.__move_y = self.__move_y_cuu1_cud1
+ else:
+ raise RuntimeError("insufficient terminal (vertical)")
+
+ if self._dch1:
+ self.dch1 = self._dch1
+ elif self._dch:
+ self.dch1 = terminfo.tparm(self._dch, 1)
+ else:
+ self.dch1 = None
+
+ if self._ich1:
+ self.ich1 = self._ich1
+ elif self._ich:
+ self.ich1 = terminfo.tparm(self._ich, 1)
+ else:
+ self.ich1 = None
+
+ self.__move = self.__move_short
+
+ def __write_changed_line(self, y, oldline, newline, px_coord):
+ # this is frustrating; there's no reason to test (say)
+ # self.dch1 inside the loop -- but alternative ways of
+ # structuring this function are equally painful (I'm trying to
+ # avoid writing code generators these days...)
+ minlen = min(wlen(oldline), wlen(newline))
+ x_pos = 0
+ x_coord = 0
+
+ px_pos = 0
+ j = 0
+ for c in oldline:
+ if j >= px_coord:
+ break
+ j += wlen(c)
+ px_pos += 1
+
+ # reuse the oldline as much as possible, but stop as soon as we
+ # encounter an ESCAPE, because it might be the start of an escape
+ # sequence
+ while (
+ x_coord < minlen
+ and oldline[x_pos] == newline[x_pos]
+ and newline[x_pos] != "\x1b"
+ ):
+ x_coord += wlen(newline[x_pos])
+ x_pos += 1
+
+ # if we need to insert a single character right after the first detected change
+ if oldline[x_pos:] == newline[x_pos + 1 :] and self.ich1:
+ if (
+ y == self.posxy[1]
+ and x_coord > self.posxy[0]
+ and oldline[px_pos:x_pos] == newline[px_pos + 1 : x_pos + 1]
+ ):
+ x_pos = px_pos
+ x_coord = px_coord
+ character_width = wlen(newline[x_pos])
+ self.__move(x_coord, y)
+ self.__write_code(self.ich1)
+ self.__write(newline[x_pos])
+ self.posxy = x_coord + character_width, y
+
+ # if it's a single character change in the middle of the line
+ elif (
+ x_coord < minlen
+ and oldline[x_pos + 1 :] == newline[x_pos + 1 :]
+ and wlen(oldline[x_pos]) == wlen(newline[x_pos])
+ ):
+ character_width = wlen(newline[x_pos])
+ self.__move(x_coord, y)
+ self.__write(newline[x_pos])
+ self.posxy = x_coord + character_width, y
+
+ # if this is the last character to fit in the line and we edit in the middle of the line
+ elif (
+ self.dch1
+ and self.ich1
+ and wlen(newline) == self.width
+ and x_coord < wlen(newline) - 2
+ and newline[x_pos + 1 : -1] == oldline[x_pos:-2]
+ ):
+ self.__hide_cursor()
+ self.__move(self.width - 2, y)
+ self.posxy = self.width - 2, y
+ self.__write_code(self.dch1)
+
+ character_width = wlen(newline[x_pos])
+ self.__move(x_coord, y)
+ self.__write_code(self.ich1)
+ self.__write(newline[x_pos])
+ self.posxy = character_width + 1, y
+
+ else:
+ self.__hide_cursor()
+ self.__move(x_coord, y)
+ if wlen(oldline) > wlen(newline):
+ self.__write_code(self._el)
+ self.__write(newline[x_pos:])
+ self.posxy = wlen(newline), y
+
+ if "\x1b" in newline:
+ # ANSI escape characters are present, so we can't assume
+ # anything about the position of the cursor. Moving the cursor
+ # to the left margin should work to get to a known position.
+ self.move_cursor(0, y)
+
+ def __write(self, text):
+ self.__buffer.append((text, 0))
+
+ def __write_code(self, fmt, *args):
+ self.__buffer.append((terminfo.tparm(fmt, *args), 1))
+
+ def __maybe_write_code(self, fmt, *args):
+ if fmt:
+ self.__write_code(fmt, *args)
+
+ def __move_y_cuu1_cud1(self, y):
+ assert self._cud1 is not None
+ assert self._cuu1 is not None
+ dy = y - self.posxy[1]
+ if dy > 0:
+ self.__write_code(dy * self._cud1)
+ elif dy < 0:
+ self.__write_code((-dy) * self._cuu1)
+
+ def __move_y_cuu_cud(self, y):
+ dy = y - self.posxy[1]
+ if dy > 0:
+ self.__write_code(self._cud, dy)
+ elif dy < 0:
+ self.__write_code(self._cuu, -dy)
+
+ def __move_x_hpa(self, x: int) -> None:
+ if x != self.posxy[0]:
+ self.__write_code(self._hpa, x)
+
+ def __move_x_cub1_cuf1(self, x: int) -> None:
+ assert self._cuf1 is not None
+ assert self._cub1 is not None
+ dx = x - self.posxy[0]
+ if dx > 0:
+ self.__write_code(self._cuf1 * dx)
+ elif dx < 0:
+ self.__write_code(self._cub1 * (-dx))
+
+ def __move_x_cub_cuf(self, x: int) -> None:
+ dx = x - self.posxy[0]
+ if dx > 0:
+ self.__write_code(self._cuf, dx)
+ elif dx < 0:
+ self.__write_code(self._cub, -dx)
+
+ def __move_short(self, x, y):
+ self.__move_x(x)
+ self.__move_y(y)
+
+ def __move_tall(self, x, y):
+ assert 0 <= y - self.__offset < self.height, y - self.__offset
+ self.__write_code(self._cup, y - self.__offset, x)
+
+ def __sigwinch(self, signum, frame):
+ self.event_queue.insert(Event("resize", None))
+
+ def __hide_cursor(self):
+ if self.cursor_visible:
+ self.__maybe_write_code(self._civis)
+ self.cursor_visible = 0
+
+ def __show_cursor(self):
+ if not self.cursor_visible:
+ self.__maybe_write_code(self._cnorm)
+ self.cursor_visible = 1
+
+ def repaint(self):
+ if not self.__gone_tall:
+ self.posxy = 0, self.posxy[1]
+ self.__write("\r")
+ ns = len(self.screen) * ["\000" * self.width]
+ self.screen = ns
+ else:
+ self.posxy = 0, self.__offset
+ self.__move(0, self.__offset)
+ ns = self.height * ["\000" * self.width]
+ self.screen = ns
+
+ def __tputs(self, fmt, prog=delayprog):
+ """A Python implementation of the curses tputs function; the
+ curses one can't really be wrapped in a sane manner.
+
+ I have the strong suspicion that this is complexity that
+ will never do anyone any good."""
+ # using .get() means that things will blow up
+ # only if the bps is actually needed (which I'm
+ # betting is pretty unlikely)
+ bps = ratedict.get(self.__svtermstate.ospeed)
+ while True:
+ m = prog.search(fmt)
+ if not m:
+ os.write(self.output_fd, fmt)
+ break
+ x, y = m.span()
+ os.write(self.output_fd, fmt[:x])
+ fmt = fmt[y:]
+ delay = int(m.group(1))
+ if b"*" in m.group(2):
+ delay *= self.height
+ if self._pad and bps is not None:
+ nchars = (bps * delay) / 1000
+ os.write(self.output_fd, self._pad * nchars)
+ else:
+ time.sleep(float(delay) / 1000.0)
+
+ def __input_fd_set(
+ self,
+ state: TermState,
+ ignore: AbstractSet[int] = _error_codes_to_ignore,
+ ) -> bool:
+ try:
+ tcsetattr(self.input_fd, termios.TCSADRAIN, state)
+ except termios.error as te:
+ if te.args[0] not in ignore:
+ raise
+ return False
+ else:
+ return True
diff --git a/PythonLib/full/_pyrepl/unix_eventqueue.py b/PythonLib/full/_pyrepl/unix_eventqueue.py
new file mode 100644
index 000000000..2a9cca59e
--- /dev/null
+++ b/PythonLib/full/_pyrepl/unix_eventqueue.py
@@ -0,0 +1,77 @@
+# Copyright 2000-2008 Michael Hudson-Doyle
+# Armin Rigo
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from .terminfo import TermInfo
+from .trace import trace
+from .base_eventqueue import BaseEventQueue
+from termios import tcgetattr, VERASE
+import os
+
+
+# Mapping of human-readable key names to their terminal-specific codes
+TERMINAL_KEYNAMES = {
+ "delete": "kdch1",
+ "down": "kcud1",
+ "end": "kend",
+ "enter": "kent",
+ "home": "khome",
+ "insert": "kich1",
+ "left": "kcub1",
+ "page down": "knp",
+ "page up": "kpp",
+ "right": "kcuf1",
+ "up": "kcuu1",
+}
+
+
+# Function keys F1-F20 mapping
+TERMINAL_KEYNAMES.update(("f%d" % i, "kf%d" % i) for i in range(1, 21))
+
+# Known CTRL-arrow keycodes
+CTRL_ARROW_KEYCODES= {
+ # for xterm, gnome-terminal, xfce terminal, etc.
+ b'\033[1;5D': 'ctrl left',
+ b'\033[1;5C': 'ctrl right',
+ # for rxvt
+ b'\033Od': 'ctrl left',
+ b'\033Oc': 'ctrl right',
+}
+
+def get_terminal_keycodes(ti: TermInfo) -> dict[bytes, str]:
+ """
+ Generates a dictionary mapping terminal keycodes to human-readable names.
+ """
+ keycodes = {}
+ for key, terminal_code in TERMINAL_KEYNAMES.items():
+ keycode = ti.get(terminal_code)
+ trace('key {key} tiname {terminal_code} keycode {keycode!r}', **locals())
+ if keycode:
+ keycodes[keycode] = key
+ keycodes.update(CTRL_ARROW_KEYCODES)
+ return keycodes
+
+
+class EventQueue(BaseEventQueue):
+ def __init__(self, fd: int, encoding: str, ti: TermInfo) -> None:
+ keycodes = get_terminal_keycodes(ti)
+ if os.isatty(fd):
+ backspace = tcgetattr(fd)[6][VERASE]
+ keycodes[backspace] = "backspace"
+ BaseEventQueue.__init__(self, encoding, keycodes)
diff --git a/PythonLib/full/_pyrepl/utils.py b/PythonLib/full/_pyrepl/utils.py
new file mode 100644
index 000000000..06cddef85
--- /dev/null
+++ b/PythonLib/full/_pyrepl/utils.py
@@ -0,0 +1,391 @@
+from __future__ import annotations
+import builtins
+import functools
+import keyword
+import re
+import token as T
+import tokenize
+import unicodedata
+import _colorize
+
+from collections import deque
+from io import StringIO
+from tokenize import TokenInfo as TI
+from typing import Iterable, Iterator, Match, NamedTuple, Self
+
+from .types import CharBuffer, CharWidths
+from .trace import trace
+
+ANSI_ESCAPE_SEQUENCE = re.compile(r"\x1b\[[ -@]*[A-~]")
+ZERO_WIDTH_BRACKET = re.compile(r"\x01.*?\x02")
+ZERO_WIDTH_TRANS = str.maketrans({"\x01": "", "\x02": ""})
+IDENTIFIERS_AFTER = {"def", "class"}
+KEYWORD_CONSTANTS = {"True", "False", "None"}
+BUILTINS = {str(name) for name in dir(builtins) if not name.startswith('_')}
+
+
+def THEME(**kwargs):
+ # Not cached: the user can modify the theme inside the interactive session.
+ return _colorize.get_theme(**kwargs).syntax
+
+
+class Span(NamedTuple):
+ """Span indexing that's inclusive on both ends."""
+
+ start: int
+ end: int
+
+ @classmethod
+ def from_re(cls, m: Match[str], group: int | str) -> Self:
+ re_span = m.span(group)
+ return cls(re_span[0], re_span[1] - 1)
+
+ @classmethod
+ def from_token(cls, token: TI, line_len: list[int]) -> Self:
+ end_offset = -1
+ if (token.type in {T.FSTRING_MIDDLE, T.TSTRING_MIDDLE}
+ and token.string.endswith(("{", "}"))):
+ # gh-134158: a visible trailing brace comes from a double brace in input
+ end_offset += 1
+
+ return cls(
+ line_len[token.start[0] - 1] + token.start[1],
+ line_len[token.end[0] - 1] + token.end[1] + end_offset,
+ )
+
+
+class ColorSpan(NamedTuple):
+ span: Span
+ tag: str
+
+
+@functools.cache
+def str_width(c: str) -> int:
+ if ord(c) < 128:
+ return 1
+ # gh-139246 for zero-width joiner and combining characters
+ if unicodedata.combining(c):
+ return 0
+ category = unicodedata.category(c)
+ if category == "Cf" and c != "\u00ad":
+ return 0
+ w = unicodedata.east_asian_width(c)
+ if w in ("N", "Na", "H", "A"):
+ return 1
+ return 2
+
+
+def wlen(s: str) -> int:
+ if len(s) == 1 and s != "\x1a":
+ return str_width(s)
+ length = sum(str_width(i) for i in s)
+ # remove lengths of any escape sequences
+ sequence = ANSI_ESCAPE_SEQUENCE.findall(s)
+ ctrl_z_cnt = s.count("\x1a")
+ return length - sum(len(i) for i in sequence) + ctrl_z_cnt
+
+
+def unbracket(s: str, including_content: bool = False) -> str:
+ r"""Return `s` with \001 and \002 characters removed.
+
+ If `including_content` is True, content between \001 and \002 is also
+ stripped.
+ """
+ if including_content:
+ return ZERO_WIDTH_BRACKET.sub("", s)
+ return s.translate(ZERO_WIDTH_TRANS)
+
+
+def gen_colors(buffer: str) -> Iterator[ColorSpan]:
+ """Returns a list of index spans to color using the given color tag.
+
+ The input `buffer` should be a valid start of a Python code block, i.e.
+ it cannot be a block starting in the middle of a multiline string.
+ """
+ sio = StringIO(buffer)
+ line_lengths = [0] + [len(line) for line in sio.readlines()]
+ # make line_lengths cumulative
+ for i in range(1, len(line_lengths)):
+ line_lengths[i] += line_lengths[i-1]
+
+ sio.seek(0)
+ gen = tokenize.generate_tokens(sio.readline)
+ last_emitted: ColorSpan | None = None
+ try:
+ for color in gen_colors_from_token_stream(gen, line_lengths):
+ yield color
+ last_emitted = color
+ except SyntaxError:
+ return
+ except tokenize.TokenError as te:
+ yield from recover_unterminated_string(
+ te, line_lengths, last_emitted, buffer
+ )
+
+
+def recover_unterminated_string(
+ exc: tokenize.TokenError,
+ line_lengths: list[int],
+ last_emitted: ColorSpan | None,
+ buffer: str,
+) -> Iterator[ColorSpan]:
+ msg, loc = exc.args
+ if loc is None:
+ return
+
+ line_no, column = loc
+
+ if msg.startswith(
+ (
+ "unterminated string literal",
+ "unterminated f-string literal",
+ "unterminated t-string literal",
+ "EOF in multi-line string",
+ "unterminated triple-quoted f-string literal",
+ "unterminated triple-quoted t-string literal",
+ )
+ ):
+ start = line_lengths[line_no - 1] + column - 1
+ end = line_lengths[-1] - 1
+
+ # in case FSTRING_START was already emitted
+ if last_emitted and start <= last_emitted.span.start:
+ trace("before last emitted = {s}", s=start)
+ start = last_emitted.span.end + 1
+
+ span = Span(start, end)
+ trace("yielding span {a} -> {b}", a=span.start, b=span.end)
+ yield ColorSpan(span, "string")
+ else:
+ trace(
+ "unhandled token error({buffer}) = {te}",
+ buffer=repr(buffer),
+ te=str(exc),
+ )
+
+
+def gen_colors_from_token_stream(
+ token_generator: Iterator[TI],
+ line_lengths: list[int],
+) -> Iterator[ColorSpan]:
+ token_window = prev_next_window(token_generator)
+
+ is_def_name = False
+ bracket_level = 0
+ for prev_token, token, next_token in token_window:
+ assert token is not None
+ if token.start == token.end:
+ continue
+
+ match token.type:
+ case (
+ T.STRING
+ | T.FSTRING_START | T.FSTRING_MIDDLE | T.FSTRING_END
+ | T.TSTRING_START | T.TSTRING_MIDDLE | T.TSTRING_END
+ ):
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "string")
+ case T.COMMENT:
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "comment")
+ case T.NUMBER:
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "number")
+ case T.OP:
+ if token.string in "([{":
+ bracket_level += 1
+ elif token.string in ")]}":
+ bracket_level -= 1
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "op")
+ case T.NAME:
+ if is_def_name:
+ is_def_name = False
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "definition")
+ elif keyword.iskeyword(token.string):
+ span_cls = "keyword"
+ if token.string in KEYWORD_CONSTANTS:
+ span_cls = "keyword_constant"
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, span_cls)
+ if token.string in IDENTIFIERS_AFTER:
+ is_def_name = True
+ elif (
+ keyword.issoftkeyword(token.string)
+ and bracket_level == 0
+ and is_soft_keyword_used(prev_token, token, next_token)
+ ):
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "soft_keyword")
+ elif (
+ token.string in BUILTINS
+ and not (prev_token and prev_token.exact_type == T.DOT)
+ ):
+ span = Span.from_token(token, line_lengths)
+ yield ColorSpan(span, "builtin")
+
+
+keyword_first_sets_match = {"False", "None", "True", "await", "lambda", "not"}
+keyword_first_sets_case = {"False", "None", "True"}
+
+
+def is_soft_keyword_used(*tokens: TI | None) -> bool:
+ """Returns True if the current token is a keyword in this context.
+
+ For the `*tokens` to match anything, they have to be a three-tuple of
+ (previous, current, next).
+ """
+ trace("is_soft_keyword_used{t}", t=tokens)
+ match tokens:
+ case (
+ None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
+ TI(string="match"),
+ TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
+ | TI(T.OP, string="(" | "*" | "[" | "{" | "~" | "...")
+ ):
+ return True
+ case (
+ None | TI(T.NEWLINE) | TI(T.INDENT) | TI(string=":"),
+ TI(string="match"),
+ TI(T.NAME, string=s)
+ ):
+ if keyword.iskeyword(s):
+ return s in keyword_first_sets_match
+ return True
+ case (
+ None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
+ TI(string="case"),
+ TI(T.NUMBER | T.STRING | T.FSTRING_START | T.TSTRING_START)
+ | TI(T.OP, string="(" | "*" | "-" | "[" | "{")
+ ):
+ return True
+ case (
+ None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
+ TI(string="case"),
+ TI(T.NAME, string=s)
+ ):
+ if keyword.iskeyword(s):
+ return s in keyword_first_sets_case
+ return True
+ case (TI(string="case"), TI(string="_"), TI(string=":")):
+ return True
+ case (
+ None | TI(T.NEWLINE) | TI(T.INDENT) | TI(T.DEDENT) | TI(string=":"),
+ TI(string="type"),
+ TI(T.NAME, string=s)
+ ):
+ return not keyword.iskeyword(s)
+ case _:
+ return False
+
+
+def disp_str(
+ buffer: str,
+ colors: list[ColorSpan] | None = None,
+ start_index: int = 0,
+ force_color: bool = False,
+) -> tuple[CharBuffer, CharWidths]:
+ r"""Decompose the input buffer into a printable variant with applied colors.
+
+ Returns a tuple of two lists:
+ - the first list is the input buffer, character by character, with color
+ escape codes added (while those codes contain multiple ASCII characters,
+ each code is considered atomic *and is attached for the corresponding
+ visible character*);
+ - the second list is the visible width of each character in the input
+ buffer.
+
+ Note on colors:
+ - The `colors` list, if provided, is partially consumed within. We're using
+ a list and not a generator since we need to hold onto the current
+ unfinished span between calls to disp_str in case of multiline strings.
+ - The `colors` list is computed from the start of the input block. `buffer`
+ is only a subset of that input block, a single line within. This is why
+ we need `start_index` to inform us which position is the start of `buffer`
+ actually within user input. This allows us to match color spans correctly.
+
+ Examples:
+ >>> utils.disp_str("a = 9")
+ (['a', ' ', '=', ' ', '9'], [1, 1, 1, 1, 1])
+
+ >>> line = "while 1:"
+ >>> colors = list(utils.gen_colors(line))
+ >>> utils.disp_str(line, colors=colors)
+ (['\x1b[1;34mw', 'h', 'i', 'l', 'e\x1b[0m', ' ', '1', ':'], [1, 1, 1, 1, 1, 1, 1, 1])
+
+ """
+ chars: CharBuffer = []
+ char_widths: CharWidths = []
+
+ if not buffer:
+ return chars, char_widths
+
+ while colors and colors[0].span.end < start_index:
+ # move past irrelevant spans
+ colors.pop(0)
+
+ theme = THEME(force_color=force_color)
+ pre_color = ""
+ post_color = ""
+ if colors and colors[0].span.start < start_index:
+ # looks like we're continuing a previous color (e.g. a multiline str)
+ pre_color = theme[colors[0].tag]
+
+ for i, c in enumerate(buffer, start_index):
+ if colors and colors[0].span.start == i: # new color starts now
+ pre_color = theme[colors[0].tag]
+
+ if c == "\x1a": # CTRL-Z on Windows
+ chars.append(c)
+ char_widths.append(2)
+ elif ord(c) < 128:
+ chars.append(c)
+ char_widths.append(1)
+ elif unicodedata.category(c).startswith("C"):
+ c = r"\u%04x" % ord(c)
+ chars.append(c)
+ char_widths.append(len(c))
+ else:
+ chars.append(c)
+ char_widths.append(str_width(c))
+
+ if colors and colors[0].span.end == i: # current color ends now
+ post_color = theme.reset
+ colors.pop(0)
+
+ chars[-1] = pre_color + chars[-1] + post_color
+ pre_color = ""
+ post_color = ""
+
+ if colors and colors[0].span.start < i and colors[0].span.end > i:
+ # even though the current color should be continued, reset it for now.
+ # the next call to `disp_str()` will revive it.
+ chars[-1] += theme.reset
+
+ return chars, char_widths
+
+
+def prev_next_window[T](
+ iterable: Iterable[T]
+) -> Iterator[tuple[T | None, ...]]:
+ """Generates three-tuples of (previous, current, next) items.
+
+ On the first iteration previous is None. On the last iteration next
+ is None. In case of exception next is None and the exception is re-raised
+ on a subsequent next() call.
+
+ Inspired by `sliding_window` from `itertools` recipes.
+ """
+
+ iterator = iter(iterable)
+ window = deque((None, next(iterator)), maxlen=3)
+ try:
+ for x in iterator:
+ window.append(x)
+ yield tuple(window)
+ except Exception:
+ raise
+ finally:
+ window.append(None)
+ yield tuple(window)
diff --git a/PythonLib/full/_pyrepl/windows_console.py b/PythonLib/full/_pyrepl/windows_console.py
new file mode 100644
index 000000000..46c603074
--- /dev/null
+++ b/PythonLib/full/_pyrepl/windows_console.py
@@ -0,0 +1,730 @@
+# Copyright 2000-2004 Michael Hudson-Doyle
+#
+# All Rights Reserved
+#
+#
+# Permission to use, copy, modify, and distribute this software and
+# its documentation for any purpose is hereby granted without fee,
+# provided that the above copyright notice appear in all copies and
+# that both that copyright notice and this permission notice appear in
+# supporting documentation.
+#
+# THE AUTHOR MICHAEL HUDSON DISCLAIMS ALL WARRANTIES WITH REGARD TO
+# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+# AND FITNESS, IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL,
+# INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER
+# RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+# CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+from __future__ import annotations
+
+import io
+import os
+import sys
+
+import ctypes
+import types
+from ctypes.wintypes import (
+ _COORD,
+ WORD,
+ SMALL_RECT,
+ BOOL,
+ HANDLE,
+ CHAR,
+ DWORD,
+ WCHAR,
+ SHORT,
+)
+from ctypes import Structure, POINTER, Union
+from .console import Event, Console
+from .trace import trace
+from .utils import wlen
+from .windows_eventqueue import EventQueue
+
+try:
+ from ctypes import get_last_error, GetLastError, WinDLL, windll, WinError # type: ignore[attr-defined]
+except:
+ # Keep MyPy happy off Windows
+ from ctypes import CDLL as WinDLL, cdll as windll
+
+ def GetLastError() -> int:
+ return 42
+
+ def get_last_error() -> int:
+ return 42
+
+ class WinError(OSError): # type: ignore[no-redef]
+ def __init__(self, err: int | None, descr: str | None = None) -> None:
+ self.err = err
+ self.descr = descr
+
+# declare nt optional to allow None assignment on other platforms
+nt: types.ModuleType | None
+try:
+ import nt
+except ImportError:
+ nt = None
+
+TYPE_CHECKING = False
+
+if TYPE_CHECKING:
+ from typing import IO
+
+# Virtual-Key Codes: https://learn.microsoft.com/en-us/windows/win32/inputdev/virtual-key-codes
+VK_MAP: dict[int, str] = {
+ 0x23: "end", # VK_END
+ 0x24: "home", # VK_HOME
+ 0x25: "left", # VK_LEFT
+ 0x26: "up", # VK_UP
+ 0x27: "right", # VK_RIGHT
+ 0x28: "down", # VK_DOWN
+ 0x2E: "delete", # VK_DELETE
+ 0x70: "f1", # VK_F1
+ 0x71: "f2", # VK_F2
+ 0x72: "f3", # VK_F3
+ 0x73: "f4", # VK_F4
+ 0x74: "f5", # VK_F5
+ 0x75: "f6", # VK_F6
+ 0x76: "f7", # VK_F7
+ 0x77: "f8", # VK_F8
+ 0x78: "f9", # VK_F9
+ 0x79: "f10", # VK_F10
+ 0x7A: "f11", # VK_F11
+ 0x7B: "f12", # VK_F12
+ 0x7C: "f13", # VK_F13
+ 0x7D: "f14", # VK_F14
+ 0x7E: "f15", # VK_F15
+ 0x7F: "f16", # VK_F16
+ 0x80: "f17", # VK_F17
+ 0x81: "f18", # VK_F18
+ 0x82: "f19", # VK_F19
+ 0x83: "f20", # VK_F20
+}
+
+# Virtual terminal output sequences
+# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#output-sequences
+# Check `windows_eventqueue.py` for input sequences
+ERASE_IN_LINE = "\x1b[K"
+MOVE_LEFT = "\x1b[{}D"
+MOVE_RIGHT = "\x1b[{}C"
+MOVE_UP = "\x1b[{}A"
+MOVE_DOWN = "\x1b[{}B"
+CLEAR = "\x1b[H\x1b[J"
+
+# State of control keys: https://learn.microsoft.com/en-us/windows/console/key-event-record-str
+ALT_ACTIVE = 0x01 | 0x02
+CTRL_ACTIVE = 0x04 | 0x08
+
+WAIT_TIMEOUT = 0x102
+WAIT_FAILED = 0xFFFFFFFF
+
+# from winbase.h
+INFINITE = 0xFFFFFFFF
+
+
+class _error(Exception):
+ pass
+
+def _supports_vt():
+ try:
+ return nt._supports_virtual_terminal()
+ except AttributeError:
+ return False
+
+class WindowsConsole(Console):
+ def __init__(
+ self,
+ f_in: IO[bytes] | int = 0,
+ f_out: IO[bytes] | int = 1,
+ term: str = "",
+ encoding: str = "",
+ ):
+ super().__init__(f_in, f_out, term, encoding)
+
+ self.__vt_support = _supports_vt()
+
+ if self.__vt_support:
+ trace('console supports virtual terminal')
+
+ # Save original console modes so we can recover on cleanup.
+ original_input_mode = DWORD()
+ GetConsoleMode(InHandle, original_input_mode)
+ trace(f'saved original input mode 0x{original_input_mode.value:x}')
+ self.__original_input_mode = original_input_mode.value
+
+ SetConsoleMode(
+ OutHandle,
+ ENABLE_WRAP_AT_EOL_OUTPUT
+ | ENABLE_PROCESSED_OUTPUT
+ | ENABLE_VIRTUAL_TERMINAL_PROCESSING,
+ )
+
+ self.screen: list[str] = []
+ self.width = 80
+ self.height = 25
+ self.__offset = 0
+ self.event_queue = EventQueue(encoding)
+ try:
+ self.out = io._WindowsConsoleIO(self.output_fd, "w") # type: ignore[attr-defined]
+ except ValueError:
+ # Console I/O is redirected, fallback...
+ self.out = None
+
+ def refresh(self, screen: list[str], c_xy: tuple[int, int]) -> None:
+ """
+ Refresh the console screen.
+
+ Parameters:
+ - screen (list): List of strings representing the screen contents.
+ - c_xy (tuple): Cursor position (x, y) on the screen.
+ """
+ cx, cy = c_xy
+
+ while len(self.screen) < min(len(screen), self.height):
+ self._hide_cursor()
+ if self.screen:
+ self._move_relative(0, len(self.screen) - 1)
+ self.__write("\n")
+ self.posxy = 0, len(self.screen)
+ self.screen.append("")
+
+ px, py = self.posxy
+ old_offset = offset = self.__offset
+ height = self.height
+
+ # we make sure the cursor is on the screen, and that we're
+ # using all of the screen if we can
+ if cy < offset:
+ offset = cy
+ elif cy >= offset + height:
+ offset = cy - height + 1
+ scroll_lines = offset - old_offset
+
+ # Scrolling the buffer as the current input is greater than the visible
+ # portion of the window. We need to scroll the visible portion and the
+ # entire history
+ self._scroll(scroll_lines, self._getscrollbacksize())
+ self.posxy = self.posxy[0], self.posxy[1] + scroll_lines
+ self.__offset += scroll_lines
+
+ for i in range(scroll_lines):
+ self.screen.append("")
+ elif offset > 0 and len(screen) < offset + height:
+ offset = max(len(screen) - height, 0)
+ screen.append("")
+
+ oldscr = self.screen[old_offset : old_offset + height]
+ newscr = screen[offset : offset + height]
+
+ self.__offset = offset
+
+ self._hide_cursor()
+ for (
+ y,
+ oldline,
+ newline,
+ ) in zip(range(offset, offset + height), oldscr, newscr):
+ if oldline != newline:
+ self.__write_changed_line(y, oldline, newline, px)
+
+ y = len(newscr)
+ while y < len(oldscr):
+ self._move_relative(0, y)
+ self.posxy = 0, y
+ self._erase_to_end()
+ y += 1
+
+ self._show_cursor()
+
+ self.screen = screen
+ self.move_cursor(cx, cy)
+
+ @property
+ def input_hook(self):
+ # avoid inline imports here so the repl doesn't get flooded
+ # with import logging from -X importtime=2
+ if nt is not None and nt._is_inputhook_installed():
+ return nt._inputhook
+
+ def __write_changed_line(
+ self, y: int, oldline: str, newline: str, px_coord: int
+ ) -> None:
+ # this is frustrating; there's no reason to test (say)
+ # self.dch1 inside the loop -- but alternative ways of
+ # structuring this function are equally painful (I'm trying to
+ # avoid writing code generators these days...)
+ minlen = min(wlen(oldline), wlen(newline))
+ x_pos = 0
+ x_coord = 0
+
+ px_pos = 0
+ j = 0
+ for c in oldline:
+ if j >= px_coord:
+ break
+ j += wlen(c)
+ px_pos += 1
+
+ # reuse the oldline as much as possible, but stop as soon as we
+ # encounter an ESCAPE, because it might be the start of an escape
+ # sequence
+ while (
+ x_coord < minlen
+ and oldline[x_pos] == newline[x_pos]
+ and newline[x_pos] != "\x1b"
+ ):
+ x_coord += wlen(newline[x_pos])
+ x_pos += 1
+
+ self._hide_cursor()
+ self._move_relative(x_coord, y)
+ if wlen(oldline) > wlen(newline):
+ self._erase_to_end()
+
+ self.__write(newline[x_pos:])
+ self.posxy = min(wlen(newline), self.width - 1), y
+
+ if "\x1b" in newline or y != self.posxy[1] or '\x1a' in newline:
+ # ANSI escape characters are present, so we can't assume
+ # anything about the position of the cursor. Moving the cursor
+ # to the left margin should work to get to a known position.
+ self.move_cursor(0, y)
+
+ def _scroll(
+ self, top: int, bottom: int, left: int | None = None, right: int | None = None
+ ) -> None:
+ scroll_rect = SMALL_RECT()
+ scroll_rect.Top = SHORT(top)
+ scroll_rect.Bottom = SHORT(bottom)
+ scroll_rect.Left = SHORT(0 if left is None else left)
+ scroll_rect.Right = SHORT(
+ self.getheightwidth()[1] - 1 if right is None else right
+ )
+ destination_origin = _COORD()
+ fill_info = CHAR_INFO()
+ fill_info.UnicodeChar = " "
+
+ if not ScrollConsoleScreenBuffer(
+ OutHandle, scroll_rect, None, destination_origin, fill_info
+ ):
+ raise WinError(GetLastError())
+
+ def _hide_cursor(self):
+ self.__write("\x1b[?25l")
+
+ def _show_cursor(self):
+ self.__write("\x1b[?25h")
+
+ def _enable_blinking(self):
+ self.__write("\x1b[?12h")
+
+ def _disable_blinking(self):
+ self.__write("\x1b[?12l")
+
+ def _enable_bracketed_paste(self) -> None:
+ self.__write("\x1b[?2004h")
+
+ def _disable_bracketed_paste(self) -> None:
+ self.__write("\x1b[?2004l")
+
+ def __write(self, text: str) -> None:
+ if "\x1a" in text:
+ text = ''.join(["^Z" if x == '\x1a' else x for x in text])
+
+ if self.out is not None:
+ self.out.write(text.encode(self.encoding, "replace"))
+ self.out.flush()
+ else:
+ os.write(self.output_fd, text.encode(self.encoding, "replace"))
+
+ @property
+ def screen_xy(self) -> tuple[int, int]:
+ info = CONSOLE_SCREEN_BUFFER_INFO()
+ if not GetConsoleScreenBufferInfo(OutHandle, info):
+ raise WinError(GetLastError())
+ return info.dwCursorPosition.X, info.dwCursorPosition.Y
+
+ def _erase_to_end(self) -> None:
+ self.__write(ERASE_IN_LINE)
+
+ def prepare(self) -> None:
+ trace("prepare")
+ self.screen = []
+ self.height, self.width = self.getheightwidth()
+
+ self.posxy = 0, 0
+ self.__gone_tall = 0
+ self.__offset = 0
+
+ if self.__vt_support:
+ SetConsoleMode(InHandle, self.__original_input_mode | ENABLE_VIRTUAL_TERMINAL_INPUT)
+ self._enable_bracketed_paste()
+
+ def restore(self) -> None:
+ if self.__vt_support:
+ # Recover to original mode before running REPL
+ self._disable_bracketed_paste()
+ SetConsoleMode(InHandle, self.__original_input_mode)
+
+ def _move_relative(self, x: int, y: int) -> None:
+ """Moves relative to the current posxy"""
+ dx = x - self.posxy[0]
+ dy = y - self.posxy[1]
+ if dx < 0:
+ self.__write(MOVE_LEFT.format(-dx))
+ elif dx > 0:
+ self.__write(MOVE_RIGHT.format(dx))
+
+ if dy < 0:
+ self.__write(MOVE_UP.format(-dy))
+ elif dy > 0:
+ self.__write(MOVE_DOWN.format(dy))
+
+ def move_cursor(self, x: int, y: int) -> None:
+ if x < 0 or y < 0:
+ raise ValueError(f"Bad cursor position {x}, {y}")
+
+ if y < self.__offset or y >= self.__offset + self.height:
+ self.event_queue.insert(Event("scroll", ""))
+ else:
+ self._move_relative(x, y)
+ self.posxy = x, y
+
+ def set_cursor_vis(self, visible: bool) -> None:
+ if visible:
+ self._show_cursor()
+ else:
+ self._hide_cursor()
+
+ def getheightwidth(self) -> tuple[int, int]:
+ """Return (height, width) where height and width are the height
+ and width of the terminal window in characters."""
+ info = CONSOLE_SCREEN_BUFFER_INFO()
+ if not GetConsoleScreenBufferInfo(OutHandle, info):
+ raise WinError(GetLastError())
+ return (
+ info.srWindow.Bottom - info.srWindow.Top + 1,
+ info.srWindow.Right - info.srWindow.Left + 1,
+ )
+
+ def _getscrollbacksize(self) -> int:
+ info = CONSOLE_SCREEN_BUFFER_INFO()
+ if not GetConsoleScreenBufferInfo(OutHandle, info):
+ raise WinError(GetLastError())
+
+ return info.srWindow.Bottom # type: ignore[no-any-return]
+
+ def _read_input(self) -> INPUT_RECORD | None:
+ rec = INPUT_RECORD()
+ read = DWORD()
+ if not ReadConsoleInput(InHandle, rec, 1, read):
+ raise WinError(GetLastError())
+
+ return rec
+
+ def _read_input_bulk(
+ self, n: int
+ ) -> tuple[ctypes.Array[INPUT_RECORD], int]:
+ rec = (n * INPUT_RECORD)()
+ read = DWORD()
+ if not ReadConsoleInput(InHandle, rec, n, read):
+ raise WinError(GetLastError())
+
+ return rec, read.value
+
+ def get_event(self, block: bool = True) -> Event | None:
+ """Return an Event instance. Returns None if |block| is false
+ and there is no event pending, otherwise waits for the
+ completion of an event."""
+
+ if not block and not self.wait(timeout=0):
+ return None
+
+ while self.event_queue.empty():
+ rec = self._read_input()
+ if rec is None:
+ return None
+
+ if rec.EventType == WINDOW_BUFFER_SIZE_EVENT:
+ return Event("resize", "")
+
+ if rec.EventType != KEY_EVENT or not rec.Event.KeyEvent.bKeyDown:
+ # Only process keys and keydown events
+ if block:
+ continue
+ return None
+
+ key_event = rec.Event.KeyEvent
+ raw_key = key = key_event.uChar.UnicodeChar
+
+ if key == "\r":
+ # Make enter unix-like
+ return Event(evt="key", data="\n")
+ elif key_event.wVirtualKeyCode == 8:
+ # Turn backspace directly into the command
+ key = "backspace"
+ elif key == "\x00":
+ # Handle special keys like arrow keys and translate them into the appropriate command
+ key = VK_MAP.get(key_event.wVirtualKeyCode)
+ if key:
+ if key_event.dwControlKeyState & CTRL_ACTIVE:
+ key = f"ctrl {key}"
+ elif key_event.dwControlKeyState & ALT_ACTIVE:
+ # queue the key, return the meta command
+ self.event_queue.insert(Event(evt="key", data=key))
+ return Event(evt="key", data="\033") # keymap.py uses this for meta
+ return Event(evt="key", data=key)
+ if block:
+ continue
+
+ return None
+ elif self.__vt_support:
+ # If virtual terminal is enabled, scanning VT sequences
+ for char in raw_key.encode(self.event_queue.encoding, "replace"):
+ self.event_queue.push(char)
+ continue
+
+ if key_event.dwControlKeyState & ALT_ACTIVE:
+ # Do not swallow characters that have been entered via AltGr:
+ # Windows internally converts AltGr to CTRL+ALT, see
+ # https://learn.microsoft.com/en-us/windows/win32/api/winuser/nf-winuser-vkkeyscanw
+ if not key_event.dwControlKeyState & CTRL_ACTIVE:
+ # queue the key, return the meta command
+ self.event_queue.insert(Event(evt="key", data=key))
+ return Event(evt="key", data="\033") # keymap.py uses this for meta
+
+ return Event(evt="key", data=key)
+ return self.event_queue.get()
+
+ def push_char(self, char: int | bytes) -> None:
+ """
+ Push a character to the console event queue.
+ """
+ raise NotImplementedError("push_char not supported on Windows")
+
+ def beep(self) -> None:
+ self.__write("\x07")
+
+ def clear(self) -> None:
+ """Wipe the screen"""
+ self.__write(CLEAR)
+ self.posxy = 0, 0
+ self.screen = []
+
+ def finish(self) -> None:
+ """Move the cursor to the end of the display and otherwise get
+ ready for end. XXX could be merged with restore? Hmm."""
+ y = len(self.screen) - 1
+ while y >= 0 and not self.screen[y]:
+ y -= 1
+ self._move_relative(0, min(y, self.height + self.__offset - 1))
+ self.__write("\r\n")
+
+ def flushoutput(self) -> None:
+ """Flush all output to the screen (assuming there's some
+ buffering going on somewhere).
+
+ All output on Windows is unbuffered so this is a nop"""
+ pass
+
+ def forgetinput(self) -> None:
+ """Forget all pending, but not yet processed input."""
+ if not FlushConsoleInputBuffer(InHandle):
+ raise WinError(GetLastError())
+
+ def getpending(self) -> Event:
+ """Return the characters that have been typed but not yet
+ processed."""
+ e = Event("key", "", b"")
+
+ while not self.event_queue.empty():
+ e2 = self.event_queue.get()
+ if e2:
+ e.data += e2.data
+
+ recs, rec_count = self._read_input_bulk(1024)
+ for i in range(rec_count):
+ rec = recs[i]
+ # In case of a legacy console, we do not only receive a keydown
+ # event, but also a keyup event - and for uppercase letters
+ # an additional SHIFT_PRESSED event.
+ if rec and rec.EventType == KEY_EVENT:
+ key_event = rec.Event.KeyEvent
+ if not key_event.bKeyDown:
+ continue
+ ch = key_event.uChar.UnicodeChar
+ if ch == "\x00":
+ # ignore SHIFT_PRESSED and special keys
+ continue
+ if ch == "\r":
+ ch += "\n"
+ e.data += ch
+ return e
+
+ def wait_for_event(self, timeout: float | None) -> bool:
+ """Wait for an event."""
+ if timeout is None:
+ timeout = INFINITE
+ else:
+ timeout = int(timeout)
+ ret = WaitForSingleObject(InHandle, timeout)
+ if ret == WAIT_FAILED:
+ raise WinError(get_last_error())
+ elif ret == WAIT_TIMEOUT:
+ return False
+ return True
+
+ def wait(self, timeout: float | None) -> bool:
+ """
+ Wait for events on the console.
+ """
+ return (
+ not self.event_queue.empty()
+ or self.wait_for_event(timeout)
+ )
+
+ def repaint(self) -> None:
+ raise NotImplementedError("No repaint support")
+
+
+# Windows interop
+class CONSOLE_SCREEN_BUFFER_INFO(Structure):
+ _fields_ = [
+ ("dwSize", _COORD),
+ ("dwCursorPosition", _COORD),
+ ("wAttributes", WORD),
+ ("srWindow", SMALL_RECT),
+ ("dwMaximumWindowSize", _COORD),
+ ]
+
+
+class CONSOLE_CURSOR_INFO(Structure):
+ _fields_ = [
+ ("dwSize", DWORD),
+ ("bVisible", BOOL),
+ ]
+
+
+class CHAR_INFO(Structure):
+ _fields_ = [
+ ("UnicodeChar", WCHAR),
+ ("Attributes", WORD),
+ ]
+
+
+class Char(Union):
+ _fields_ = [
+ ("UnicodeChar", WCHAR),
+ ("Char", CHAR),
+ ]
+
+
+class KeyEvent(ctypes.Structure):
+ _fields_ = [
+ ("bKeyDown", BOOL),
+ ("wRepeatCount", WORD),
+ ("wVirtualKeyCode", WORD),
+ ("wVirtualScanCode", WORD),
+ ("uChar", Char),
+ ("dwControlKeyState", DWORD),
+ ]
+
+
+class WindowsBufferSizeEvent(ctypes.Structure):
+ _fields_ = [("dwSize", _COORD)]
+
+
+class ConsoleEvent(ctypes.Union):
+ _fields_ = [
+ ("KeyEvent", KeyEvent),
+ ("WindowsBufferSizeEvent", WindowsBufferSizeEvent),
+ ]
+
+
+class INPUT_RECORD(Structure):
+ _fields_ = [("EventType", WORD), ("Event", ConsoleEvent)]
+
+
+KEY_EVENT = 0x01
+FOCUS_EVENT = 0x10
+MENU_EVENT = 0x08
+MOUSE_EVENT = 0x02
+WINDOW_BUFFER_SIZE_EVENT = 0x04
+
+ENABLE_PROCESSED_INPUT = 0x0001
+ENABLE_LINE_INPUT = 0x0002
+ENABLE_ECHO_INPUT = 0x0004
+ENABLE_MOUSE_INPUT = 0x0010
+ENABLE_INSERT_MODE = 0x0020
+ENABLE_VIRTUAL_TERMINAL_INPUT = 0x0200
+
+ENABLE_PROCESSED_OUTPUT = 0x01
+ENABLE_WRAP_AT_EOL_OUTPUT = 0x02
+ENABLE_VIRTUAL_TERMINAL_PROCESSING = 0x04
+
+STD_INPUT_HANDLE = -10
+STD_OUTPUT_HANDLE = -11
+
+if sys.platform == "win32":
+ _KERNEL32 = WinDLL("kernel32", use_last_error=True)
+
+ GetStdHandle = windll.kernel32.GetStdHandle
+ GetStdHandle.argtypes = [DWORD]
+ GetStdHandle.restype = HANDLE
+
+ GetConsoleScreenBufferInfo = _KERNEL32.GetConsoleScreenBufferInfo
+ GetConsoleScreenBufferInfo.argtypes = [
+ HANDLE,
+ ctypes.POINTER(CONSOLE_SCREEN_BUFFER_INFO),
+ ]
+ GetConsoleScreenBufferInfo.restype = BOOL
+
+ ScrollConsoleScreenBuffer = _KERNEL32.ScrollConsoleScreenBufferW
+ ScrollConsoleScreenBuffer.argtypes = [
+ HANDLE,
+ POINTER(SMALL_RECT),
+ POINTER(SMALL_RECT),
+ _COORD,
+ POINTER(CHAR_INFO),
+ ]
+ ScrollConsoleScreenBuffer.restype = BOOL
+
+ GetConsoleMode = _KERNEL32.GetConsoleMode
+ GetConsoleMode.argtypes = [HANDLE, POINTER(DWORD)]
+ GetConsoleMode.restype = BOOL
+
+ SetConsoleMode = _KERNEL32.SetConsoleMode
+ SetConsoleMode.argtypes = [HANDLE, DWORD]
+ SetConsoleMode.restype = BOOL
+
+ ReadConsoleInput = _KERNEL32.ReadConsoleInputW
+ ReadConsoleInput.argtypes = [HANDLE, POINTER(INPUT_RECORD), DWORD, POINTER(DWORD)]
+ ReadConsoleInput.restype = BOOL
+
+
+ FlushConsoleInputBuffer = _KERNEL32.FlushConsoleInputBuffer
+ FlushConsoleInputBuffer.argtypes = [HANDLE]
+ FlushConsoleInputBuffer.restype = BOOL
+
+ WaitForSingleObject = _KERNEL32.WaitForSingleObject
+ WaitForSingleObject.argtypes = [HANDLE, DWORD]
+ WaitForSingleObject.restype = DWORD
+
+ OutHandle = GetStdHandle(STD_OUTPUT_HANDLE)
+ InHandle = GetStdHandle(STD_INPUT_HANDLE)
+else:
+
+ def _win_only(*args, **kwargs):
+ raise NotImplementedError("Windows only")
+
+ GetStdHandle = _win_only
+ GetConsoleScreenBufferInfo = _win_only
+ ScrollConsoleScreenBuffer = _win_only
+ GetConsoleMode = _win_only
+ SetConsoleMode = _win_only
+ ReadConsoleInput = _win_only
+ FlushConsoleInputBuffer = _win_only
+ WaitForSingleObject = _win_only
+ OutHandle = 0
+ InHandle = 0
diff --git a/PythonLib/full/_pyrepl/windows_eventqueue.py b/PythonLib/full/_pyrepl/windows_eventqueue.py
new file mode 100644
index 000000000..d99722f9a
--- /dev/null
+++ b/PythonLib/full/_pyrepl/windows_eventqueue.py
@@ -0,0 +1,42 @@
+"""
+Windows event and VT sequence scanner
+"""
+
+from .base_eventqueue import BaseEventQueue
+
+
+# Reference: https://learn.microsoft.com/en-us/windows/console/console-virtual-terminal-sequences#input-sequences
+VT_MAP: dict[bytes, str] = {
+ b'\x1b[A': 'up',
+ b'\x1b[B': 'down',
+ b'\x1b[C': 'right',
+ b'\x1b[D': 'left',
+ b'\x1b[1;5D': 'ctrl left',
+ b'\x1b[1;5C': 'ctrl right',
+
+ b'\x1b[H': 'home',
+ b'\x1b[F': 'end',
+
+ b'\x7f': 'backspace',
+ b'\x1b[2~': 'insert',
+ b'\x1b[3~': 'delete',
+ b'\x1b[5~': 'page up',
+ b'\x1b[6~': 'page down',
+
+ b'\x1bOP': 'f1',
+ b'\x1bOQ': 'f2',
+ b'\x1bOR': 'f3',
+ b'\x1bOS': 'f4',
+ b'\x1b[15~': 'f5',
+ b'\x1b[17~': 'f6',
+ b'\x1b[18~': 'f7',
+ b'\x1b[19~': 'f8',
+ b'\x1b[20~': 'f9',
+ b'\x1b[21~': 'f10',
+ b'\x1b[23~': 'f11',
+ b'\x1b[24~': 'f12',
+}
+
+class EventQueue(BaseEventQueue):
+ def __init__(self, encoding: str) -> None:
+ BaseEventQueue.__init__(self, encoding, VT_MAP)
diff --git a/PythonLib/full/_sitebuiltins.py b/PythonLib/full/_sitebuiltins.py
index c66269a57..81b36efc6 100644
--- a/PythonLib/full/_sitebuiltins.py
+++ b/PythonLib/full/_sitebuiltins.py
@@ -36,7 +36,7 @@ def __init__(self, name, data, files=(), dirs=()):
import os
self.__name = name
self.__data = data
- self.__lines = None
+ self.__lines = []
self.__filenames = [os.path.join(dir, filename)
for dir in dirs
for filename in files]
@@ -65,24 +65,12 @@ def __repr__(self):
return "Type %s() to see the full %s text" % ((self.__name,)*2)
def __call__(self):
+ from _pyrepl.pager import get_pager
self.__setup()
- prompt = 'Hit Return for more, or q (and Return) to quit: '
- lineno = 0
- while 1:
- try:
- for i in range(lineno, lineno + self.MAXLINES):
- print(self.__lines[i])
- except IndexError:
- break
- else:
- lineno += self.MAXLINES
- key = None
- while key is None:
- key = input(prompt)
- if key not in ('', 'q'):
- key = None
- if key == 'q':
- break
+
+ pager = get_pager()
+ text = "\n".join(self.__lines)
+ pager(text, title=self.__name)
class _Helper(object):
diff --git a/PythonLib/full/_strptime.py b/PythonLib/full/_strptime.py
index dfd2bc5d8..fc7e369c3 100644
--- a/PythonLib/full/_strptime.py
+++ b/PythonLib/full/_strptime.py
@@ -10,9 +10,11 @@
strptime -- Calculates the time struct represented by the passed-in string
"""
+import os
import time
import locale
import calendar
+import re
from re import compile as re_compile
from re import sub as re_sub
from re import IGNORECASE
@@ -40,6 +42,29 @@ def _findall(haystack, needle):
yield i
i += len(needle)
+def _fixmonths(months):
+ yield from months
+ # The lower case of 'İ' ('\u0130') is 'i\u0307'.
+ # The re module only supports 1-to-1 character matching in
+ # case-insensitive mode.
+ for s in months:
+ if 'i\u0307' in s:
+ yield s.replace('i\u0307', '\u0130')
+
+lzh_TW_alt_digits = (
+ # 〇:一:二:三:四:五:六:七:八:九
+ '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db',
+ '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d',
+ # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九
+ '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db',
+ '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d',
+ # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九
+ '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db',
+ '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d',
+ # 卅:卅一
+ '\u5345', '\u5345\u4e00')
+
+
class LocaleTime(object):
"""Stores and handles locale-specific information related to time.
@@ -83,6 +108,7 @@ def __init__(self):
self.__calc_weekday()
self.__calc_month()
self.__calc_am_pm()
+ self.__calc_alt_digits()
self.__calc_timezone()
self.__calc_date_time()
if _getlang() != self.lang:
@@ -118,9 +144,43 @@ def __calc_am_pm(self):
am_pm.append(time.strftime("%p", time_tuple).lower().strip())
self.am_pm = am_pm
+ def __calc_alt_digits(self):
+ # Set self.LC_alt_digits by using time.strftime().
+
+ # The magic data should contain all decimal digits.
+ time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0))
+ s = time.strftime("%x%X", time_tuple)
+ if s.isascii():
+ # Fast path -- all digits are ASCII.
+ self.LC_alt_digits = ()
+ return
+
+ digits = ''.join(sorted(set(re.findall(r'\d', s))))
+ if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9:
+ # All 10 decimal digits from the same set.
+ if digits.isascii():
+ # All digits are ASCII.
+ self.LC_alt_digits = ()
+ return
+
+ self.LC_alt_digits = [a + b for a in digits for b in digits]
+ # Test whether the numbers contain leading zero.
+ time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0))
+ if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2):
+ self.LC_alt_digits[:10] = digits
+ return
+
+ # Either non-Gregorian calendar or non-decimal numbers.
+ if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s):
+ # lzh_TW
+ self.LC_alt_digits = lzh_TW_alt_digits
+ return
+
+ self.LC_alt_digits = None
+
def __calc_date_time(self):
- # Set self.date_time, self.date, & self.time by using
- # time.strftime().
+ # Set self.LC_date_time, self.LC_date, self.LC_time and
+ # self.LC_time_ampm by using time.strftime().
# Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of
# overloaded numbers is minimized. The order in which searches for
@@ -128,26 +188,32 @@ def __calc_date_time(self):
# possible ambiguity for what something represents.
time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0))
time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0))
- replacement_pairs = [
+ replacement_pairs = []
+
+ # Non-ASCII digits
+ if self.LC_alt_digits or self.LC_alt_digits is None:
+ for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'),
+ (44, '%OM'), (55, '%OS'), (17, '%Od'),
+ (3, '%Om'), (2, '%Ow'), (10, '%OI')]:
+ if self.LC_alt_digits is None:
+ s = chr(0x660 + n // 10) + chr(0x660 + n % 10)
+ replacement_pairs.append((s, d))
+ if n < 10:
+ replacement_pairs.append((s[1], d))
+ elif len(self.LC_alt_digits) > n:
+ replacement_pairs.append((self.LC_alt_digits[n], d))
+ else:
+ replacement_pairs.append((time.strftime(d, time_tuple), d))
+ replacement_pairs += [
('1999', '%Y'), ('99', '%y'), ('22', '%H'),
('44', '%M'), ('55', '%S'), ('76', '%j'),
('17', '%d'), ('03', '%m'), ('3', '%m'),
# '3' needed for when no leading zero.
('2', '%w'), ('10', '%I'),
- # Non-ASCII digits
- ('\u0661\u0669\u0669\u0669', '%Y'),
- ('\u0669\u0669', '%Oy'),
- ('\u0662\u0662', '%OH'),
- ('\u0664\u0664', '%OM'),
- ('\u0665\u0665', '%OS'),
- ('\u0661\u0667', '%Od'),
- ('\u0660\u0663', '%Om'),
- ('\u0663', '%Om'),
- ('\u0662', '%Ow'),
- ('\u0661\u0660', '%OI'),
]
+
date_time = []
- for directive in ('%c', '%x', '%X'):
+ for directive in ('%c', '%x', '%X', '%r'):
current_format = time.strftime(directive, time_tuple).lower()
current_format = current_format.replace('%', '%%')
# The month and the day of the week formats are treated specially
@@ -171,9 +237,10 @@ def __calc_date_time(self):
if tz:
current_format = current_format.replace(tz, "%Z")
# Transform all non-ASCII digits to digits in range U+0660 to U+0669.
- current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])",
'f': r"(?P[0-9]{1,6})",
- 'H': r"(?P2[0-3]|[0-1]\d|\d)",
+ 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)",
+ 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)",
'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])",
+ 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])",
'G': r"(?P\d\d\d\d)",
'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])",
'm': r"(?P1[0-2]|0[1-9]|[1-9])",
@@ -300,29 +370,60 @@ def __init__(self, locale_time=None):
'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)",
# W is set below by using 'U'
'y': r"(?P\d\d)",
- #XXX: Does 'Y' need to worry about having less or more than
- # 4 digits?
'Y': r"(?P\d\d\d\d)",
'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))",
'A': self.__seqToRE(self.locale_time.f_weekday, 'A'),
'a': self.__seqToRE(self.locale_time.a_weekday, 'a'),
- 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'),
- 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'),
+ 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'),
+ 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'),
'p': self.__seqToRE(self.locale_time.am_pm, 'p'),
'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone
for tz in tz_names),
'Z'),
'%': '%'}
- for d in 'dmyHIMS':
- mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d
- mapping['Ow'] = r'(?P\d)'
+ if self.locale_time.LC_alt_digits is None:
+ for d in 'dmyCHIMS':
+ mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d
+ mapping['Ow'] = r'(?P\d)'
+ else:
+ mapping.update({
+ 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd',
+ '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'),
+ 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm',
+ '1[0-2]|0[1-9]|[1-9]'),
+ 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w',
+ '[0-6]'),
+ 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y',
+ '[0-9][0-9]'),
+ 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C',
+ '[0-9][0-9]'),
+ 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H',
+ '2[0-3]|[0-1][0-9]|[0-9]'),
+ 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I',
+ '1[0-2]|0[1-9]|[1-9]'),
+ 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M',
+ '[0-5][0-9]|[0-9]'),
+ 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S',
+ '6[0-1]|[0-5][0-9]|[0-9]'),
+ })
+ mapping.update({
+ 'e': mapping['d'],
+ 'Oe': mapping['Od'],
+ 'P': mapping['p'],
+ 'Op': mapping['p'],
+ 'W': mapping['U'].replace('U', 'W'),
+ })
mapping['W'] = mapping['U'].replace('U', 'W')
+
base.__init__(mapping)
+ base.__setitem__('T', self.pattern('%H:%M:%S'))
+ base.__setitem__('R', self.pattern('%H:%M'))
+ base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm))
base.__setitem__('X', self.pattern(self.locale_time.LC_time))
base.__setitem__('x', self.pattern(self.locale_time.LC_date))
base.__setitem__('c', self.pattern(self.locale_time.LC_date_time))
- def __seqToRE(self, to_convert, directive):
+ def __seqToRE(self, to_convert, directive, altregex=None):
"""Convert a list to a regex string for matching a directive.
Want possible matching values to be from longest to shortest. This
@@ -338,8 +439,9 @@ def __seqToRE(self, to_convert, directive):
else:
return ''
regex = '|'.join(re_escape(stuff) for stuff in to_convert)
- regex = '(?P<%s>%s' % (directive, regex)
- return '%s)' % regex
+ if altregex is not None:
+ regex += '|' + altregex
+ return '(?P<%s>%s)' % (directive, regex)
def pattern(self, format):
"""Return regex pattern for the format string.
@@ -354,9 +456,29 @@ def pattern(self, format):
format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format)
format = re_sub(r'\s+', r'\\s+', format)
format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR
+ year_in_format = False
+ day_of_month_in_format = False
def repl(m):
- return self[m[1]]
- format = re_sub(r'%(O?.)', repl, format)
+ format_char = m[1]
+ match format_char:
+ case 'Y' | 'y' | 'G':
+ nonlocal year_in_format
+ year_in_format = True
+ case 'd':
+ nonlocal day_of_month_in_format
+ day_of_month_in_format = True
+ return self[format_char]
+ format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format)
+ if day_of_month_in_format and not year_in_format:
+ import warnings
+ warnings.warn("""\
+Parsing dates involving a day of month without a year specified is ambiguous
+and fails to parse leap day. The default behavior will change in Python 3.15
+to either always raise an exception or to use a different default year (TBD).
+To avoid trouble, add a specific year to the input & format.
+See https://github.com/python/cpython/issues/70647.""",
+ DeprecationWarning,
+ skip_file_prefixes=(os.path.dirname(__file__),))
return format
def compile(self, format):
@@ -420,14 +542,13 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
# \\, in which case it was a stray % but with a space after it
except KeyError as err:
bad_directive = err.args[0]
- if bad_directive == "\\":
- bad_directive = "%"
del err
+ bad_directive = bad_directive.replace('\\s', '')
+ if not bad_directive:
+ raise ValueError("stray %% in format '%s'" % format) from None
+ bad_directive = bad_directive.replace('\\', '', 1)
raise ValueError("'%s' is a bad directive in format '%s'" %
(bad_directive, format)) from None
- # IndexError only occurs when the format string is "%"
- except IndexError:
- raise ValueError("stray %% in format '%s'" % format) from None
_regex_cache[format] = format_regex
found = format_regex.match(data_string)
if not found:
@@ -449,6 +570,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
# values
weekday = julian = None
found_dict = found.groupdict()
+ if locale_time.LC_alt_digits:
+ def parse_int(s):
+ try:
+ return locale_time.LC_alt_digits.index(s)
+ except ValueError:
+ return int(s)
+ else:
+ parse_int = int
+
for group_key in found_dict.keys():
# Directives not explicitly handled below:
# c, x, X
@@ -456,30 +586,34 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
# U, W
# worthless without day of the week
if group_key == 'y':
- year = int(found_dict['y'])
- # Open Group specification for strptime() states that a %y
- #value in the range of [00, 68] is in the century 2000, while
- #[69,99] is in the century 1900
- if year <= 68:
- year += 2000
+ year = parse_int(found_dict['y'])
+ if 'C' in found_dict:
+ century = parse_int(found_dict['C'])
+ year += century * 100
else:
- year += 1900
+ # Open Group specification for strptime() states that a %y
+ #value in the range of [00, 68] is in the century 2000, while
+ #[69,99] is in the century 1900
+ if year <= 68:
+ year += 2000
+ else:
+ year += 1900
elif group_key == 'Y':
year = int(found_dict['Y'])
elif group_key == 'G':
iso_year = int(found_dict['G'])
elif group_key == 'm':
- month = int(found_dict['m'])
+ month = parse_int(found_dict['m'])
elif group_key == 'B':
month = locale_time.f_month.index(found_dict['B'].lower())
elif group_key == 'b':
month = locale_time.a_month.index(found_dict['b'].lower())
elif group_key == 'd':
- day = int(found_dict['d'])
+ day = parse_int(found_dict['d'])
elif group_key == 'H':
- hour = int(found_dict['H'])
+ hour = parse_int(found_dict['H'])
elif group_key == 'I':
- hour = int(found_dict['I'])
+ hour = parse_int(found_dict['I'])
ampm = found_dict.get('p', '').lower()
# If there was no AM/PM indicator, we'll treat this like AM
if ampm in ('', locale_time.am_pm[0]):
@@ -495,9 +629,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"):
if hour != 12:
hour += 12
elif group_key == 'M':
- minute = int(found_dict['M'])
+ minute = parse_int(found_dict['M'])
elif group_key == 'S':
- second = int(found_dict['S'])
+ second = parse_int(found_dict['S'])
elif group_key == 'f':
s = found_dict['f']
# Pad to always return microseconds.
@@ -649,18 +783,40 @@ def _strptime_time(data_string, format="%a %b %d %H:%M:%S %Y"):
tt = _strptime(data_string, format)[0]
return time.struct_time(tt[:time._STRUCT_TM_ITEMS])
-def _strptime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
- """Return a class cls instance based on the input string and the
+def _strptime_datetime_date(cls, data_string, format="%a %b %d %Y"):
+ """Return a date instance based on the input string and the
+ format string."""
+ tt, _, _ = _strptime(data_string, format)
+ args = tt[:3]
+ return cls(*args)
+
+def _parse_tz(tzname, gmtoff, gmtoff_fraction):
+ tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction)
+ if tzname:
+ return datetime_timezone(tzdelta, tzname)
+ else:
+ return datetime_timezone(tzdelta)
+
+def _strptime_datetime_time(cls, data_string, format="%H:%M:%S"):
+ """Return a time instance based on the input string and the
format string."""
tt, fraction, gmtoff_fraction = _strptime(data_string, format)
tzname, gmtoff = tt[-2:]
- args = tt[:6] + (fraction,)
- if gmtoff is not None:
- tzdelta = datetime_timedelta(seconds=gmtoff, microseconds=gmtoff_fraction)
- if tzname:
- tz = datetime_timezone(tzdelta, tzname)
- else:
- tz = datetime_timezone(tzdelta)
- args += (tz,)
+ args = tt[3:6] + (fraction,)
+ if gmtoff is None:
+ return cls(*args)
+ else:
+ tz = _parse_tz(tzname, gmtoff, gmtoff_fraction)
+ return cls(*args, tz)
- return cls(*args)
+def _strptime_datetime_datetime(cls, data_string, format="%a %b %d %H:%M:%S %Y"):
+ """Return a datetime instance based on the input string and the
+ format string."""
+ tt, fraction, gmtoff_fraction = _strptime(data_string, format)
+ tzname, gmtoff = tt[-2:]
+ args = tt[:6] + (fraction,)
+ if gmtoff is None:
+ return cls(*args)
+ else:
+ tz = _parse_tz(tzname, gmtoff, gmtoff_fraction)
+ return cls(*args, tz)
diff --git a/PythonLib/full/_threading_local.py b/PythonLib/full/_threading_local.py
index b006d76c4..0b9e5d3bb 100644
--- a/PythonLib/full/_threading_local.py
+++ b/PythonLib/full/_threading_local.py
@@ -4,128 +4,6 @@
class. Depending on the version of Python you're using, there may be a
faster one available. You should always import the `local` class from
`threading`.)
-
-Thread-local objects support the management of thread-local data.
-If you have data that you want to be local to a thread, simply create
-a thread-local object and use its attributes:
-
- >>> mydata = local()
- >>> mydata.number = 42
- >>> mydata.number
- 42
-
-You can also access the local-object's dictionary:
-
- >>> mydata.__dict__
- {'number': 42}
- >>> mydata.__dict__.setdefault('widgets', [])
- []
- >>> mydata.widgets
- []
-
-What's important about thread-local objects is that their data are
-local to a thread. If we access the data in a different thread:
-
- >>> log = []
- >>> def f():
- ... items = sorted(mydata.__dict__.items())
- ... log.append(items)
- ... mydata.number = 11
- ... log.append(mydata.number)
-
- >>> import threading
- >>> thread = threading.Thread(target=f)
- >>> thread.start()
- >>> thread.join()
- >>> log
- [[], 11]
-
-we get different data. Furthermore, changes made in the other thread
-don't affect data seen in this thread:
-
- >>> mydata.number
- 42
-
-Of course, values you get from a local object, including a __dict__
-attribute, are for whatever thread was current at the time the
-attribute was read. For that reason, you generally don't want to save
-these values across threads, as they apply only to the thread they
-came from.
-
-You can create custom local objects by subclassing the local class:
-
- >>> class MyLocal(local):
- ... number = 2
- ... def __init__(self, /, **kw):
- ... self.__dict__.update(kw)
- ... def squared(self):
- ... return self.number ** 2
-
-This can be useful to support default values, methods and
-initialization. Note that if you define an __init__ method, it will be
-called each time the local object is used in a separate thread. This
-is necessary to initialize each thread's dictionary.
-
-Now if we create a local object:
-
- >>> mydata = MyLocal(color='red')
-
-Now we have a default number:
-
- >>> mydata.number
- 2
-
-an initial color:
-
- >>> mydata.color
- 'red'
- >>> del mydata.color
-
-And a method that operates on the data:
-
- >>> mydata.squared()
- 4
-
-As before, we can access the data in a separate thread:
-
- >>> log = []
- >>> thread = threading.Thread(target=f)
- >>> thread.start()
- >>> thread.join()
- >>> log
- [[('color', 'red')], 11]
-
-without affecting this thread's data:
-
- >>> mydata.number
- 2
- >>> mydata.color
- Traceback (most recent call last):
- ...
- AttributeError: 'MyLocal' object has no attribute 'color'
-
-Note that subclasses can define slots, but they are not thread
-local. They are shared across threads:
-
- >>> class MyLocal(local):
- ... __slots__ = 'number'
-
- >>> mydata = MyLocal()
- >>> mydata.number = 42
- >>> mydata.color = 'red'
-
-So, the separate thread:
-
- >>> thread = threading.Thread(target=f)
- >>> thread.start()
- >>> thread.join()
-
-affects what we see:
-
- >>> mydata.number
- 11
-
->>> del mydata
"""
from weakref import ref
diff --git a/PythonLib/full/_weakrefset.py b/PythonLib/full/_weakrefset.py
index 489eec714..d1c7fcaee 100644
--- a/PythonLib/full/_weakrefset.py
+++ b/PythonLib/full/_weakrefset.py
@@ -8,69 +8,29 @@
__all__ = ['WeakSet']
-class _IterationGuard:
- # This context manager registers itself in the current iterators of the
- # weak container, such as to delay all removals until the context manager
- # exits.
- # This technique should be relatively thread-safe (since sets are).
-
- def __init__(self, weakcontainer):
- # Don't create cycles
- self.weakcontainer = ref(weakcontainer)
-
- def __enter__(self):
- w = self.weakcontainer()
- if w is not None:
- w._iterating.add(self)
- return self
-
- def __exit__(self, e, t, b):
- w = self.weakcontainer()
- if w is not None:
- s = w._iterating
- s.remove(self)
- if not s:
- w._commit_removals()
-
-
class WeakSet:
def __init__(self, data=None):
self.data = set()
+
def _remove(item, selfref=ref(self)):
self = selfref()
if self is not None:
- if self._iterating:
- self._pending_removals.append(item)
- else:
- self.data.discard(item)
+ self.data.discard(item)
+
self._remove = _remove
- # A list of keys to be removed
- self._pending_removals = []
- self._iterating = set()
if data is not None:
self.update(data)
- def _commit_removals(self):
- pop = self._pending_removals.pop
- discard = self.data.discard
- while True:
- try:
- item = pop()
- except IndexError:
- return
- discard(item)
-
def __iter__(self):
- with _IterationGuard(self):
- for itemref in self.data:
- item = itemref()
- if item is not None:
- # Caveat: the iterator will keep a strong reference to
- # `item` until it is resumed or closed.
- yield item
+ for itemref in self.data.copy():
+ item = itemref()
+ if item is not None:
+ # Caveat: the iterator will keep a strong reference to
+ # `item` until it is resumed or closed.
+ yield item
def __len__(self):
- return len(self.data) - len(self._pending_removals)
+ return len(self.data)
def __contains__(self, item):
try:
@@ -83,21 +43,15 @@ def __reduce__(self):
return self.__class__, (list(self),), self.__getstate__()
def add(self, item):
- if self._pending_removals:
- self._commit_removals()
self.data.add(ref(item, self._remove))
def clear(self):
- if self._pending_removals:
- self._commit_removals()
self.data.clear()
def copy(self):
return self.__class__(self)
def pop(self):
- if self._pending_removals:
- self._commit_removals()
while True:
try:
itemref = self.data.pop()
@@ -108,18 +62,12 @@ def pop(self):
return item
def remove(self, item):
- if self._pending_removals:
- self._commit_removals()
self.data.remove(ref(item))
def discard(self, item):
- if self._pending_removals:
- self._commit_removals()
self.data.discard(ref(item))
def update(self, other):
- if self._pending_removals:
- self._commit_removals()
for element in other:
self.add(element)
@@ -136,8 +84,6 @@ def difference(self, other):
def difference_update(self, other):
self.__isub__(other)
def __isub__(self, other):
- if self._pending_removals:
- self._commit_removals()
if self is other:
self.data.clear()
else:
@@ -151,8 +97,6 @@ def intersection(self, other):
def intersection_update(self, other):
self.__iand__(other)
def __iand__(self, other):
- if self._pending_removals:
- self._commit_removals()
self.data.intersection_update(ref(item) for item in other)
return self
@@ -184,8 +128,6 @@ def symmetric_difference(self, other):
def symmetric_difference_update(self, other):
self.__ixor__(other)
def __ixor__(self, other):
- if self._pending_removals:
- self._commit_removals()
if self is other:
self.data.clear()
else:
diff --git a/PythonLib/full/aifc.py b/PythonLib/full/aifc.py
deleted file mode 100644
index 5254987e2..000000000
--- a/PythonLib/full/aifc.py
+++ /dev/null
@@ -1,984 +0,0 @@
-"""Stuff to parse AIFF-C and AIFF files.
-
-Unless explicitly stated otherwise, the description below is true
-both for AIFF-C files and AIFF files.
-
-An AIFF-C file has the following structure.
-
- +-----------------+
- | FORM |
- +-----------------+
- | |
- +----+------------+
- | | AIFC |
- | +------------+
- | | |
- | | . |
- | | . |
- | | . |
- +----+------------+
-
-An AIFF file has the string "AIFF" instead of "AIFC".
-
-A chunk consists of an identifier (4 bytes) followed by a size (4 bytes,
-big endian order), followed by the data. The size field does not include
-the size of the 8 byte header.
-
-The following chunk types are recognized.
-
- FVER
- (AIFF-C only).
- MARK
- <# of markers> (2 bytes)
- list of markers:
- (2 bytes, must be > 0)
- (4 bytes)
- ("pstring")
- COMM
- <# of channels> (2 bytes)
- <# of sound frames> (4 bytes)
- (2 bytes)
- (10 bytes, IEEE 80-bit extended
- floating point)
- in AIFF-C files only:
- (4 bytes)
- ("pstring")
- SSND
- (4 bytes, not used by this program)
- (4 bytes, not used by this program)
-
-
-A pstring consists of 1 byte length, a string of characters, and 0 or 1
-byte pad to make the total length even.
-
-Usage.
-
-Reading AIFF files:
- f = aifc.open(file, 'r')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods read(), seek(), and close().
-In some types of audio files, if the setpos() method is not used,
-the seek() method is not necessary.
-
-This returns an instance of a class with the following public methods:
- getnchannels() -- returns number of audio channels (1 for
- mono, 2 for stereo)
- getsampwidth() -- returns sample width in bytes
- getframerate() -- returns sampling frequency
- getnframes() -- returns number of audio frames
- getcomptype() -- returns compression type ('NONE' for AIFF files)
- getcompname() -- returns human-readable version of
- compression type ('not compressed' for AIFF files)
- getparams() -- returns a namedtuple consisting of all of the
- above in the above order
- getmarkers() -- get the list of marks in the audio file or None
- if there are no marks
- getmark(id) -- get mark with the specified id (raises an error
- if the mark does not exist)
- readframes(n) -- returns at most n frames of audio
- rewind() -- rewind to the beginning of the audio stream
- setpos(pos) -- seek to the specified position
- tell() -- return the current position
- close() -- close the instance (make it unusable)
-The position returned by tell(), the position given to setpos() and
-the position of marks are all compatible and have nothing to do with
-the actual position in the file.
-The close() method is called automatically when the class instance
-is destroyed.
-
-Writing AIFF files:
- f = aifc.open(file, 'w')
-where file is either the name of a file or an open file pointer.
-The open file pointer must have methods write(), tell(), seek(), and
-close().
-
-This returns an instance of a class with the following public methods:
- aiff() -- create an AIFF file (AIFF-C default)
- aifc() -- create an AIFF-C file
- setnchannels(n) -- set the number of channels
- setsampwidth(n) -- set the sample width
- setframerate(n) -- set the frame rate
- setnframes(n) -- set the number of frames
- setcomptype(type, name)
- -- set the compression type and the
- human-readable compression type
- setparams(tuple)
- -- set all parameters at once
- setmark(id, pos, name)
- -- add specified mark to the list of marks
- tell() -- return current position in output file (useful
- in combination with setmark())
- writeframesraw(data)
- -- write audio frames without pathing up the
- file header
- writeframes(data)
- -- write audio frames and patch up the file header
- close() -- patch up the file header and close the
- output file
-You should set the parameters before the first writeframesraw or
-writeframes. The total number of frames does not need to be set,
-but when it is set to the correct value, the header does not have to
-be patched up.
-It is best to first set all parameters, perhaps possibly the
-compression type, and then write audio frames using writeframesraw.
-When all frames have been written, either call writeframes(b'') or
-close() to patch up the sizes in the header.
-Marks can be added anytime. If there are any marks, you must call
-close() after all frames have been written.
-The close() method is called automatically when the class instance
-is destroyed.
-
-When a file is opened with the extension '.aiff', an AIFF file is
-written, otherwise an AIFF-C file is written. This default can be
-changed by calling aiff() or aifc() before the first writeframes or
-writeframesraw.
-"""
-
-import struct
-import builtins
-import warnings
-
-__all__ = ["Error", "open"]
-
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-class Error(Exception):
- pass
-
-_AIFC_version = 0xA2805140 # Version 1 of AIFF-C
-
-def _read_long(file):
- try:
- return struct.unpack('>l', file.read(4))[0]
- except struct.error:
- raise EOFError from None
-
-def _read_ulong(file):
- try:
- return struct.unpack('>L', file.read(4))[0]
- except struct.error:
- raise EOFError from None
-
-def _read_short(file):
- try:
- return struct.unpack('>h', file.read(2))[0]
- except struct.error:
- raise EOFError from None
-
-def _read_ushort(file):
- try:
- return struct.unpack('>H', file.read(2))[0]
- except struct.error:
- raise EOFError from None
-
-def _read_string(file):
- length = ord(file.read(1))
- if length == 0:
- data = b''
- else:
- data = file.read(length)
- if length & 1 == 0:
- dummy = file.read(1)
- return data
-
-_HUGE_VAL = 1.79769313486231e+308 # See
-
-def _read_float(f): # 10 bytes
- expon = _read_short(f) # 2 bytes
- sign = 1
- if expon < 0:
- sign = -1
- expon = expon + 0x8000
- himant = _read_ulong(f) # 4 bytes
- lomant = _read_ulong(f) # 4 bytes
- if expon == himant == lomant == 0:
- f = 0.0
- elif expon == 0x7FFF:
- f = _HUGE_VAL
- else:
- expon = expon - 16383
- f = (himant * 0x100000000 + lomant) * pow(2.0, expon - 63)
- return sign * f
-
-def _write_short(f, x):
- f.write(struct.pack('>h', x))
-
-def _write_ushort(f, x):
- f.write(struct.pack('>H', x))
-
-def _write_long(f, x):
- f.write(struct.pack('>l', x))
-
-def _write_ulong(f, x):
- f.write(struct.pack('>L', x))
-
-def _write_string(f, s):
- if len(s) > 255:
- raise ValueError("string exceeds maximum pstring length")
- f.write(struct.pack('B', len(s)))
- f.write(s)
- if len(s) & 1 == 0:
- f.write(b'\x00')
-
-def _write_float(f, x):
- import math
- if x < 0:
- sign = 0x8000
- x = x * -1
- else:
- sign = 0
- if x == 0:
- expon = 0
- himant = 0
- lomant = 0
- else:
- fmant, expon = math.frexp(x)
- if expon > 16384 or fmant >= 1 or fmant != fmant: # Infinity or NaN
- expon = sign|0x7FFF
- himant = 0
- lomant = 0
- else: # Finite
- expon = expon + 16382
- if expon < 0: # denormalized
- fmant = math.ldexp(fmant, expon)
- expon = 0
- expon = expon | sign
- fmant = math.ldexp(fmant, 32)
- fsmant = math.floor(fmant)
- himant = int(fsmant)
- fmant = math.ldexp(fmant - fsmant, 32)
- fsmant = math.floor(fmant)
- lomant = int(fsmant)
- _write_ushort(f, expon)
- _write_ulong(f, himant)
- _write_ulong(f, lomant)
-
-with warnings.catch_warnings():
- warnings.simplefilter("ignore", DeprecationWarning)
- from chunk import Chunk
-from collections import namedtuple
-
-_aifc_params = namedtuple('_aifc_params',
- 'nchannels sampwidth framerate nframes comptype compname')
-
-_aifc_params.nchannels.__doc__ = 'Number of audio channels (1 for mono, 2 for stereo)'
-_aifc_params.sampwidth.__doc__ = 'Sample width in bytes'
-_aifc_params.framerate.__doc__ = 'Sampling frequency'
-_aifc_params.nframes.__doc__ = 'Number of audio frames'
-_aifc_params.comptype.__doc__ = 'Compression type ("NONE" for AIFF files)'
-_aifc_params.compname.__doc__ = ("""\
-A human-readable version of the compression type
-('not compressed' for AIFF files)""")
-
-
-class Aifc_read:
- # Variables used in this class:
- #
- # These variables are available to the user though appropriate
- # methods of this class:
- # _file -- the open file with methods read(), close(), and seek()
- # set through the __init__() method
- # _nchannels -- the number of audio channels
- # available through the getnchannels() method
- # _nframes -- the number of audio frames
- # available through the getnframes() method
- # _sampwidth -- the number of bytes per audio sample
- # available through the getsampwidth() method
- # _framerate -- the sampling frequency
- # available through the getframerate() method
- # _comptype -- the AIFF-C compression type ('NONE' if AIFF)
- # available through the getcomptype() method
- # _compname -- the human-readable AIFF-C compression type
- # available through the getcomptype() method
- # _markers -- the marks in the audio file
- # available through the getmarkers() and getmark()
- # methods
- # _soundpos -- the position in the audio stream
- # available through the tell() method, set through the
- # setpos() method
- #
- # These variables are used internally only:
- # _version -- the AIFF-C version number
- # _decomp -- the decompressor from builtin module cl
- # _comm_chunk_read -- 1 iff the COMM chunk has been read
- # _aifc -- 1 iff reading an AIFF-C file
- # _ssnd_seek_needed -- 1 iff positioned correctly in audio
- # file for readframes()
- # _ssnd_chunk -- instantiation of a chunk class for the SSND chunk
- # _framesize -- size of one frame in the file
-
- _file = None # Set here since __del__ checks it
-
- def initfp(self, file):
- self._version = 0
- self._convert = None
- self._markers = []
- self._soundpos = 0
- self._file = file
- chunk = Chunk(file)
- if chunk.getname() != b'FORM':
- raise Error('file does not start with FORM id')
- formdata = chunk.read(4)
- if formdata == b'AIFF':
- self._aifc = 0
- elif formdata == b'AIFC':
- self._aifc = 1
- else:
- raise Error('not an AIFF or AIFF-C file')
- self._comm_chunk_read = 0
- self._ssnd_chunk = None
- while 1:
- self._ssnd_seek_needed = 1
- try:
- chunk = Chunk(self._file)
- except EOFError:
- break
- chunkname = chunk.getname()
- if chunkname == b'COMM':
- self._read_comm_chunk(chunk)
- self._comm_chunk_read = 1
- elif chunkname == b'SSND':
- self._ssnd_chunk = chunk
- dummy = chunk.read(8)
- self._ssnd_seek_needed = 0
- elif chunkname == b'FVER':
- self._version = _read_ulong(chunk)
- elif chunkname == b'MARK':
- self._readmark(chunk)
- chunk.skip()
- if not self._comm_chunk_read or not self._ssnd_chunk:
- raise Error('COMM chunk and/or SSND chunk missing')
-
- def __init__(self, f):
- if isinstance(f, str):
- file_object = builtins.open(f, 'rb')
- try:
- self.initfp(file_object)
- except:
- file_object.close()
- raise
- else:
- # assume it is an open file object already
- self.initfp(f)
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- self.close()
-
- #
- # User visible methods.
- #
- def getfp(self):
- return self._file
-
- def rewind(self):
- self._ssnd_seek_needed = 1
- self._soundpos = 0
-
- def close(self):
- file = self._file
- if file is not None:
- self._file = None
- file.close()
-
- def tell(self):
- return self._soundpos
-
- def getnchannels(self):
- return self._nchannels
-
- def getnframes(self):
- return self._nframes
-
- def getsampwidth(self):
- return self._sampwidth
-
- def getframerate(self):
- return self._framerate
-
- def getcomptype(self):
- return self._comptype
-
- def getcompname(self):
- return self._compname
-
-## def getversion(self):
-## return self._version
-
- def getparams(self):
- return _aifc_params(self.getnchannels(), self.getsampwidth(),
- self.getframerate(), self.getnframes(),
- self.getcomptype(), self.getcompname())
-
- def getmarkers(self):
- if len(self._markers) == 0:
- return None
- return self._markers
-
- def getmark(self, id):
- for marker in self._markers:
- if id == marker[0]:
- return marker
- raise Error('marker {0!r} does not exist'.format(id))
-
- def setpos(self, pos):
- if pos < 0 or pos > self._nframes:
- raise Error('position not in range')
- self._soundpos = pos
- self._ssnd_seek_needed = 1
-
- def readframes(self, nframes):
- if self._ssnd_seek_needed:
- self._ssnd_chunk.seek(0)
- dummy = self._ssnd_chunk.read(8)
- pos = self._soundpos * self._framesize
- if pos:
- self._ssnd_chunk.seek(pos + 8)
- self._ssnd_seek_needed = 0
- if nframes == 0:
- return b''
- data = self._ssnd_chunk.read(nframes * self._framesize)
- if self._convert and data:
- data = self._convert(data)
- self._soundpos = self._soundpos + len(data) // (self._nchannels
- * self._sampwidth)
- return data
-
- #
- # Internal methods.
- #
-
- def _alaw2lin(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.alaw2lin(data, 2)
-
- def _ulaw2lin(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.ulaw2lin(data, 2)
-
- def _adpcm2lin(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- if not hasattr(self, '_adpcmstate'):
- # first time
- self._adpcmstate = None
- data, self._adpcmstate = audioop.adpcm2lin(data, 2, self._adpcmstate)
- return data
-
- def _sowt2lin(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.byteswap(data, 2)
-
- def _read_comm_chunk(self, chunk):
- self._nchannels = _read_short(chunk)
- self._nframes = _read_long(chunk)
- self._sampwidth = (_read_short(chunk) + 7) // 8
- self._framerate = int(_read_float(chunk))
- if self._sampwidth <= 0:
- raise Error('bad sample width')
- if self._nchannels <= 0:
- raise Error('bad # of channels')
- self._framesize = self._nchannels * self._sampwidth
- if self._aifc:
- #DEBUG: SGI's soundeditor produces a bad size :-(
- kludge = 0
- if chunk.chunksize == 18:
- kludge = 1
- warnings.warn('Warning: bad COMM chunk size')
- chunk.chunksize = 23
- #DEBUG end
- self._comptype = chunk.read(4)
- #DEBUG start
- if kludge:
- length = ord(chunk.file.read(1))
- if length & 1 == 0:
- length = length + 1
- chunk.chunksize = chunk.chunksize + length
- chunk.file.seek(-1, 1)
- #DEBUG end
- self._compname = _read_string(chunk)
- if self._comptype != b'NONE':
- if self._comptype == b'G722':
- self._convert = self._adpcm2lin
- elif self._comptype in (b'ulaw', b'ULAW'):
- self._convert = self._ulaw2lin
- elif self._comptype in (b'alaw', b'ALAW'):
- self._convert = self._alaw2lin
- elif self._comptype in (b'sowt', b'SOWT'):
- self._convert = self._sowt2lin
- else:
- raise Error('unsupported compression type')
- self._sampwidth = 2
- else:
- self._comptype = b'NONE'
- self._compname = b'not compressed'
-
- def _readmark(self, chunk):
- nmarkers = _read_short(chunk)
- # Some files appear to contain invalid counts.
- # Cope with this by testing for EOF.
- try:
- for i in range(nmarkers):
- id = _read_short(chunk)
- pos = _read_long(chunk)
- name = _read_string(chunk)
- if pos or name:
- # some files appear to have
- # dummy markers consisting of
- # a position 0 and name ''
- self._markers.append((id, pos, name))
- except EOFError:
- w = ('Warning: MARK chunk contains only %s marker%s instead of %s' %
- (len(self._markers), '' if len(self._markers) == 1 else 's',
- nmarkers))
- warnings.warn(w)
-
-class Aifc_write:
- # Variables used in this class:
- #
- # These variables are user settable through appropriate methods
- # of this class:
- # _file -- the open file with methods write(), close(), tell(), seek()
- # set through the __init__() method
- # _comptype -- the AIFF-C compression type ('NONE' in AIFF)
- # set through the setcomptype() or setparams() method
- # _compname -- the human-readable AIFF-C compression type
- # set through the setcomptype() or setparams() method
- # _nchannels -- the number of audio channels
- # set through the setnchannels() or setparams() method
- # _sampwidth -- the number of bytes per audio sample
- # set through the setsampwidth() or setparams() method
- # _framerate -- the sampling frequency
- # set through the setframerate() or setparams() method
- # _nframes -- the number of audio frames written to the header
- # set through the setnframes() or setparams() method
- # _aifc -- whether we're writing an AIFF-C file or an AIFF file
- # set through the aifc() method, reset through the
- # aiff() method
- #
- # These variables are used internally only:
- # _version -- the AIFF-C version number
- # _comp -- the compressor from builtin module cl
- # _nframeswritten -- the number of audio frames actually written
- # _datalength -- the size of the audio samples written to the header
- # _datawritten -- the size of the audio samples actually written
-
- _file = None # Set here since __del__ checks it
-
- def __init__(self, f):
- if isinstance(f, str):
- file_object = builtins.open(f, 'wb')
- try:
- self.initfp(file_object)
- except:
- file_object.close()
- raise
-
- # treat .aiff file extensions as non-compressed audio
- if f.endswith('.aiff'):
- self._aifc = 0
- else:
- # assume it is an open file object already
- self.initfp(f)
-
- def initfp(self, file):
- self._file = file
- self._version = _AIFC_version
- self._comptype = b'NONE'
- self._compname = b'not compressed'
- self._convert = None
- self._nchannels = 0
- self._sampwidth = 0
- self._framerate = 0
- self._nframes = 0
- self._nframeswritten = 0
- self._datawritten = 0
- self._datalength = 0
- self._markers = []
- self._marklength = 0
- self._aifc = 1 # AIFF-C is default
-
- def __del__(self):
- self.close()
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- self.close()
-
- #
- # User visible methods.
- #
- def aiff(self):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- self._aifc = 0
-
- def aifc(self):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- self._aifc = 1
-
- def setnchannels(self, nchannels):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- if nchannels < 1:
- raise Error('bad # of channels')
- self._nchannels = nchannels
-
- def getnchannels(self):
- if not self._nchannels:
- raise Error('number of channels not set')
- return self._nchannels
-
- def setsampwidth(self, sampwidth):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- if sampwidth < 1 or sampwidth > 4:
- raise Error('bad sample width')
- self._sampwidth = sampwidth
-
- def getsampwidth(self):
- if not self._sampwidth:
- raise Error('sample width not set')
- return self._sampwidth
-
- def setframerate(self, framerate):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- if framerate <= 0:
- raise Error('bad frame rate')
- self._framerate = framerate
-
- def getframerate(self):
- if not self._framerate:
- raise Error('frame rate not set')
- return self._framerate
-
- def setnframes(self, nframes):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- self._nframes = nframes
-
- def getnframes(self):
- return self._nframeswritten
-
- def setcomptype(self, comptype, compname):
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- if comptype not in (b'NONE', b'ulaw', b'ULAW',
- b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
- raise Error('unsupported compression type')
- self._comptype = comptype
- self._compname = compname
-
- def getcomptype(self):
- return self._comptype
-
- def getcompname(self):
- return self._compname
-
-## def setversion(self, version):
-## if self._nframeswritten:
-## raise Error, 'cannot change parameters after starting to write'
-## self._version = version
-
- def setparams(self, params):
- nchannels, sampwidth, framerate, nframes, comptype, compname = params
- if self._nframeswritten:
- raise Error('cannot change parameters after starting to write')
- if comptype not in (b'NONE', b'ulaw', b'ULAW',
- b'alaw', b'ALAW', b'G722', b'sowt', b'SOWT'):
- raise Error('unsupported compression type')
- self.setnchannels(nchannels)
- self.setsampwidth(sampwidth)
- self.setframerate(framerate)
- self.setnframes(nframes)
- self.setcomptype(comptype, compname)
-
- def getparams(self):
- if not self._nchannels or not self._sampwidth or not self._framerate:
- raise Error('not all parameters set')
- return _aifc_params(self._nchannels, self._sampwidth, self._framerate,
- self._nframes, self._comptype, self._compname)
-
- def setmark(self, id, pos, name):
- if id <= 0:
- raise Error('marker ID must be > 0')
- if pos < 0:
- raise Error('marker position must be >= 0')
- if not isinstance(name, bytes):
- raise Error('marker name must be bytes')
- for i in range(len(self._markers)):
- if id == self._markers[i][0]:
- self._markers[i] = id, pos, name
- return
- self._markers.append((id, pos, name))
-
- def getmark(self, id):
- for marker in self._markers:
- if id == marker[0]:
- return marker
- raise Error('marker {0!r} does not exist'.format(id))
-
- def getmarkers(self):
- if len(self._markers) == 0:
- return None
- return self._markers
-
- def tell(self):
- return self._nframeswritten
-
- def writeframesraw(self, data):
- if not isinstance(data, (bytes, bytearray)):
- data = memoryview(data).cast('B')
- self._ensure_header_written(len(data))
- nframes = len(data) // (self._sampwidth * self._nchannels)
- if self._convert:
- data = self._convert(data)
- self._file.write(data)
- self._nframeswritten = self._nframeswritten + nframes
- self._datawritten = self._datawritten + len(data)
-
- def writeframes(self, data):
- self.writeframesraw(data)
- if self._nframeswritten != self._nframes or \
- self._datalength != self._datawritten:
- self._patchheader()
-
- def close(self):
- if self._file is None:
- return
- try:
- self._ensure_header_written(0)
- if self._datawritten & 1:
- # quick pad to even size
- self._file.write(b'\x00')
- self._datawritten = self._datawritten + 1
- self._writemarkers()
- if self._nframeswritten != self._nframes or \
- self._datalength != self._datawritten or \
- self._marklength:
- self._patchheader()
- finally:
- # Prevent ref cycles
- self._convert = None
- f = self._file
- self._file = None
- f.close()
-
- #
- # Internal methods.
- #
-
- def _lin2alaw(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.lin2alaw(data, 2)
-
- def _lin2ulaw(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.lin2ulaw(data, 2)
-
- def _lin2adpcm(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- if not hasattr(self, '_adpcmstate'):
- self._adpcmstate = None
- data, self._adpcmstate = audioop.lin2adpcm(data, 2, self._adpcmstate)
- return data
-
- def _lin2sowt(self, data):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', category=DeprecationWarning)
- import audioop
- return audioop.byteswap(data, 2)
-
- def _ensure_header_written(self, datasize):
- if not self._nframeswritten:
- if self._comptype in (b'ULAW', b'ulaw',
- b'ALAW', b'alaw', b'G722',
- b'sowt', b'SOWT'):
- if not self._sampwidth:
- self._sampwidth = 2
- if self._sampwidth != 2:
- raise Error('sample width must be 2 when compressing '
- 'with ulaw/ULAW, alaw/ALAW, sowt/SOWT '
- 'or G7.22 (ADPCM)')
- if not self._nchannels:
- raise Error('# channels not specified')
- if not self._sampwidth:
- raise Error('sample width not specified')
- if not self._framerate:
- raise Error('sampling rate not specified')
- self._write_header(datasize)
-
- def _init_compression(self):
- if self._comptype == b'G722':
- self._convert = self._lin2adpcm
- elif self._comptype in (b'ulaw', b'ULAW'):
- self._convert = self._lin2ulaw
- elif self._comptype in (b'alaw', b'ALAW'):
- self._convert = self._lin2alaw
- elif self._comptype in (b'sowt', b'SOWT'):
- self._convert = self._lin2sowt
-
- def _write_header(self, initlength):
- if self._aifc and self._comptype != b'NONE':
- self._init_compression()
- self._file.write(b'FORM')
- if not self._nframes:
- self._nframes = initlength // (self._nchannels * self._sampwidth)
- self._datalength = self._nframes * self._nchannels * self._sampwidth
- if self._datalength & 1:
- self._datalength = self._datalength + 1
- if self._aifc:
- if self._comptype in (b'ulaw', b'ULAW', b'alaw', b'ALAW'):
- self._datalength = self._datalength // 2
- if self._datalength & 1:
- self._datalength = self._datalength + 1
- elif self._comptype == b'G722':
- self._datalength = (self._datalength + 3) // 4
- if self._datalength & 1:
- self._datalength = self._datalength + 1
- try:
- self._form_length_pos = self._file.tell()
- except (AttributeError, OSError):
- self._form_length_pos = None
- commlength = self._write_form_length(self._datalength)
- if self._aifc:
- self._file.write(b'AIFC')
- self._file.write(b'FVER')
- _write_ulong(self._file, 4)
- _write_ulong(self._file, self._version)
- else:
- self._file.write(b'AIFF')
- self._file.write(b'COMM')
- _write_ulong(self._file, commlength)
- _write_short(self._file, self._nchannels)
- if self._form_length_pos is not None:
- self._nframes_pos = self._file.tell()
- _write_ulong(self._file, self._nframes)
- if self._comptype in (b'ULAW', b'ulaw', b'ALAW', b'alaw', b'G722'):
- _write_short(self._file, 8)
- else:
- _write_short(self._file, self._sampwidth * 8)
- _write_float(self._file, self._framerate)
- if self._aifc:
- self._file.write(self._comptype)
- _write_string(self._file, self._compname)
- self._file.write(b'SSND')
- if self._form_length_pos is not None:
- self._ssnd_length_pos = self._file.tell()
- _write_ulong(self._file, self._datalength + 8)
- _write_ulong(self._file, 0)
- _write_ulong(self._file, 0)
-
- def _write_form_length(self, datalength):
- if self._aifc:
- commlength = 18 + 5 + len(self._compname)
- if commlength & 1:
- commlength = commlength + 1
- verslength = 12
- else:
- commlength = 18
- verslength = 0
- _write_ulong(self._file, 4 + verslength + self._marklength + \
- 8 + commlength + 16 + datalength)
- return commlength
-
- def _patchheader(self):
- curpos = self._file.tell()
- if self._datawritten & 1:
- datalength = self._datawritten + 1
- self._file.write(b'\x00')
- else:
- datalength = self._datawritten
- if datalength == self._datalength and \
- self._nframes == self._nframeswritten and \
- self._marklength == 0:
- self._file.seek(curpos, 0)
- return
- self._file.seek(self._form_length_pos, 0)
- dummy = self._write_form_length(datalength)
- self._file.seek(self._nframes_pos, 0)
- _write_ulong(self._file, self._nframeswritten)
- self._file.seek(self._ssnd_length_pos, 0)
- _write_ulong(self._file, datalength + 8)
- self._file.seek(curpos, 0)
- self._nframes = self._nframeswritten
- self._datalength = datalength
-
- def _writemarkers(self):
- if len(self._markers) == 0:
- return
- self._file.write(b'MARK')
- length = 2
- for marker in self._markers:
- id, pos, name = marker
- length = length + len(name) + 1 + 6
- if len(name) & 1 == 0:
- length = length + 1
- _write_ulong(self._file, length)
- self._marklength = length + 8
- _write_short(self._file, len(self._markers))
- for marker in self._markers:
- id, pos, name = marker
- _write_short(self._file, id)
- _write_ulong(self._file, pos)
- _write_string(self._file, name)
-
-def open(f, mode=None):
- if mode is None:
- if hasattr(f, 'mode'):
- mode = f.mode
- else:
- mode = 'rb'
- if mode in ('r', 'rb'):
- return Aifc_read(f)
- elif mode in ('w', 'wb'):
- return Aifc_write(f)
- else:
- raise Error("mode must be 'r', 'rb', 'w', or 'wb'")
-
-
-if __name__ == '__main__':
- import sys
- if not sys.argv[1:]:
- sys.argv.append('/usr/demos/data/audio/bach.aiff')
- fn = sys.argv[1]
- with open(fn, 'r') as f:
- print("Reading", fn)
- print("nchannels =", f.getnchannels())
- print("nframes =", f.getnframes())
- print("sampwidth =", f.getsampwidth())
- print("framerate =", f.getframerate())
- print("comptype =", f.getcomptype())
- print("compname =", f.getcompname())
- if sys.argv[2:]:
- gn = sys.argv[2]
- print("Writing", gn)
- with open(gn, 'w') as g:
- g.setparams(f.getparams())
- while 1:
- data = f.readframes(1024)
- if not data:
- break
- g.writeframes(data)
- print("Done.")
diff --git a/PythonLib/full/annotationlib.py b/PythonLib/full/annotationlib.py
new file mode 100644
index 000000000..5c9a08126
--- /dev/null
+++ b/PythonLib/full/annotationlib.py
@@ -0,0 +1,1197 @@
+"""Helpers for introspecting and wrapping annotations."""
+
+import ast
+import builtins
+import enum
+import keyword
+import sys
+import types
+
+__all__ = [
+ "Format",
+ "ForwardRef",
+ "call_annotate_function",
+ "call_evaluate_function",
+ "get_annotate_from_class_namespace",
+ "get_annotations",
+ "annotations_to_string",
+ "type_repr",
+]
+
+
+class Format(enum.IntEnum):
+ VALUE = 1
+ VALUE_WITH_FAKE_GLOBALS = 2
+ FORWARDREF = 3
+ STRING = 4
+
+
+_sentinel = object()
+# Following `NAME_ERROR_MSG` in `ceval_macros.h`:
+_NAME_ERROR_MSG = "name '{name:.200}' is not defined"
+
+
+# Slots shared by ForwardRef and _Stringifier. The __forward__ names must be
+# preserved for compatibility with the old typing.ForwardRef class. The remaining
+# names are private.
+_SLOTS = (
+ "__forward_is_argument__",
+ "__forward_is_class__",
+ "__forward_module__",
+ "__weakref__",
+ "__arg__",
+ "__globals__",
+ "__extra_names__",
+ "__code__",
+ "__ast_node__",
+ "__cell__",
+ "__owner__",
+ "__stringifier_dict__",
+ "__resolved_str_cache__",
+)
+
+
+class ForwardRef:
+ """Wrapper that holds a forward reference.
+
+ Constructor arguments:
+ * arg: a string representing the code to be evaluated.
+ * module: the module where the forward reference was created.
+ Must be a string, not a module object.
+ * owner: The owning object (module, class, or function).
+ * is_argument: Does nothing, retained for compatibility.
+ * is_class: True if the forward reference was created in class scope.
+
+ """
+
+ __slots__ = _SLOTS
+
+ def __init__(
+ self,
+ arg,
+ *,
+ module=None,
+ owner=None,
+ is_argument=True,
+ is_class=False,
+ ):
+ if not isinstance(arg, str):
+ raise TypeError(f"Forward reference must be a string -- got {arg!r}")
+
+ self.__arg__ = arg
+ self.__forward_is_argument__ = is_argument
+ self.__forward_is_class__ = is_class
+ self.__forward_module__ = module
+ self.__owner__ = owner
+ # These are always set to None here but may be non-None if a ForwardRef
+ # is created through __class__ assignment on a _Stringifier object.
+ self.__globals__ = None
+ # This may be either a cell object (for a ForwardRef referring to a single name)
+ # or a dict mapping cell names to cell objects (for a ForwardRef containing references
+ # to multiple names).
+ self.__cell__ = None
+ self.__extra_names__ = None
+ # These are initially None but serve as a cache and may be set to a non-None
+ # value later.
+ self.__code__ = None
+ self.__ast_node__ = None
+ self.__resolved_str_cache__ = None
+
+ def __init_subclass__(cls, /, *args, **kwds):
+ raise TypeError("Cannot subclass ForwardRef")
+
+ def evaluate(
+ self,
+ *,
+ globals=None,
+ locals=None,
+ type_params=None,
+ owner=None,
+ format=Format.VALUE,
+ ):
+ """Evaluate the forward reference and return the value.
+
+ If the forward reference cannot be evaluated, raise an exception.
+ """
+ match format:
+ case Format.STRING:
+ return self.__resolved_str__
+ case Format.VALUE:
+ is_forwardref_format = False
+ case Format.FORWARDREF:
+ is_forwardref_format = True
+ case _:
+ raise NotImplementedError(format)
+ if isinstance(self.__cell__, types.CellType):
+ try:
+ return self.__cell__.cell_contents
+ except ValueError:
+ pass
+ if owner is None:
+ owner = self.__owner__
+
+ if globals is None and self.__forward_module__ is not None:
+ globals = getattr(
+ sys.modules.get(self.__forward_module__, None), "__dict__", None
+ )
+ if globals is None:
+ globals = self.__globals__
+ if globals is None:
+ if isinstance(owner, type):
+ module_name = getattr(owner, "__module__", None)
+ if module_name:
+ module = sys.modules.get(module_name, None)
+ if module:
+ globals = getattr(module, "__dict__", None)
+ elif isinstance(owner, types.ModuleType):
+ globals = getattr(owner, "__dict__", None)
+ elif callable(owner):
+ globals = getattr(owner, "__globals__", None)
+
+ # If we pass None to eval() below, the globals of this module are used.
+ if globals is None:
+ globals = {}
+
+ if type_params is None and owner is not None:
+ type_params = getattr(owner, "__type_params__", None)
+
+ if locals is None:
+ locals = {}
+ if isinstance(owner, type):
+ locals.update(vars(owner))
+ elif (
+ type_params is not None
+ or isinstance(self.__cell__, dict)
+ or self.__extra_names__
+ ):
+ # Create a new locals dict if necessary,
+ # to avoid mutating the argument.
+ locals = dict(locals)
+
+ # "Inject" type parameters into the local namespace
+ # (unless they are shadowed by assignments *in* the local namespace),
+ # as a way of emulating annotation scopes when calling `eval()`
+ if type_params is not None:
+ for param in type_params:
+ locals.setdefault(param.__name__, param)
+
+ # Similar logic can be used for nonlocals, which should not
+ # override locals.
+ if isinstance(self.__cell__, dict):
+ for cell_name, cell in self.__cell__.items():
+ try:
+ cell_value = cell.cell_contents
+ except ValueError:
+ pass
+ else:
+ locals.setdefault(cell_name, cell_value)
+
+ if self.__extra_names__:
+ locals.update(self.__extra_names__)
+
+ arg = self.__forward_arg__
+ if arg.isidentifier() and not keyword.iskeyword(arg):
+ if arg in locals:
+ return locals[arg]
+ elif arg in globals:
+ return globals[arg]
+ elif hasattr(builtins, arg):
+ return getattr(builtins, arg)
+ elif is_forwardref_format:
+ return self
+ else:
+ raise NameError(_NAME_ERROR_MSG.format(name=arg), name=arg)
+ else:
+ code = self.__forward_code__
+ try:
+ return eval(code, globals=globals, locals=locals)
+ except Exception:
+ if not is_forwardref_format:
+ raise
+
+ # All variables, in scoping order, should be checked before
+ # triggering __missing__ to create a _Stringifier.
+ new_locals = _StringifierDict(
+ {**builtins.__dict__, **globals, **locals},
+ globals=globals,
+ owner=owner,
+ is_class=self.__forward_is_class__,
+ format=format,
+ )
+ try:
+ result = eval(code, globals=globals, locals=new_locals)
+ except Exception:
+ return self
+ else:
+ new_locals.transmogrify(self.__cell__)
+ return result
+
+ def _evaluate(self, globalns, localns, type_params=_sentinel, *, recursive_guard):
+ import typing
+ import warnings
+
+ if type_params is _sentinel:
+ typing._deprecation_warning_for_no_type_params_passed(
+ "typing.ForwardRef._evaluate"
+ )
+ type_params = ()
+ warnings._deprecated(
+ "ForwardRef._evaluate",
+ "{name} is a private API and is retained for compatibility, but will be removed"
+ " in Python 3.16. Use ForwardRef.evaluate() or typing.evaluate_forward_ref() instead.",
+ remove=(3, 16),
+ )
+ return typing.evaluate_forward_ref(
+ self,
+ globals=globalns,
+ locals=localns,
+ type_params=type_params,
+ _recursive_guard=recursive_guard,
+ )
+
+ @property
+ def __forward_arg__(self):
+ if self.__arg__ is not None:
+ return self.__arg__
+ if self.__ast_node__ is not None:
+ self.__arg__ = ast.unparse(self.__ast_node__)
+ return self.__arg__
+ raise AssertionError(
+ "Attempted to access '__forward_arg__' on an uninitialized ForwardRef"
+ )
+
+ @property
+ def __resolved_str__(self):
+ # __forward_arg__ with any names from __extra_names__ replaced
+ # with the type_repr of the value they represent
+ if self.__resolved_str_cache__ is None:
+ resolved_str = self.__forward_arg__
+ names = self.__extra_names__
+
+ if names:
+ visitor = _ExtraNameFixer(names)
+ ast_expr = ast.parse(resolved_str, mode="eval").body
+ node = visitor.visit(ast_expr)
+ resolved_str = ast.unparse(node)
+
+ self.__resolved_str_cache__ = resolved_str
+
+ return self.__resolved_str_cache__
+
+ @property
+ def __forward_code__(self):
+ if self.__code__ is not None:
+ return self.__code__
+ arg = self.__forward_arg__
+ try:
+ self.__code__ = compile(_rewrite_star_unpack(arg), "", "eval")
+ except SyntaxError:
+ raise SyntaxError(f"Forward reference must be an expression -- got {arg!r}")
+ return self.__code__
+
+ def __eq__(self, other):
+ if not isinstance(other, ForwardRef):
+ return NotImplemented
+ return (
+ self.__forward_arg__ == other.__forward_arg__
+ and self.__forward_module__ == other.__forward_module__
+ # Use "is" here because we use id() for this in __hash__
+ # because dictionaries are not hashable.
+ and self.__globals__ is other.__globals__
+ and self.__forward_is_class__ == other.__forward_is_class__
+ # Two separate cells are always considered unequal in forward refs.
+ and (
+ {name: id(cell) for name, cell in self.__cell__.items()}
+ == {name: id(cell) for name, cell in other.__cell__.items()}
+ if isinstance(self.__cell__, dict) and isinstance(other.__cell__, dict)
+ else self.__cell__ is other.__cell__
+ )
+ and self.__owner__ == other.__owner__
+ and (
+ (tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None) ==
+ (tuple(sorted(other.__extra_names__.items())) if other.__extra_names__ else None)
+ )
+ )
+
+ def __hash__(self):
+ return hash((
+ self.__forward_arg__,
+ self.__forward_module__,
+ id(self.__globals__), # dictionaries are not hashable, so hash by identity
+ self.__forward_is_class__,
+ ( # cells are not hashable as well
+ tuple(sorted([(name, id(cell)) for name, cell in self.__cell__.items()]))
+ if isinstance(self.__cell__, dict) else id(self.__cell__),
+ ),
+ self.__owner__,
+ tuple(sorted(self.__extra_names__.items())) if self.__extra_names__ else None,
+ ))
+
+ def __or__(self, other):
+ return types.UnionType[self, other]
+
+ def __ror__(self, other):
+ return types.UnionType[other, self]
+
+ def __repr__(self):
+ extra = []
+ if self.__forward_module__ is not None:
+ extra.append(f", module={self.__forward_module__!r}")
+ if self.__forward_is_class__:
+ extra.append(", is_class=True")
+ if self.__owner__ is not None:
+ extra.append(f", owner={self.__owner__!r}")
+ return f"ForwardRef({self.__resolved_str__!r}{''.join(extra)})"
+
+
+_Template = type(t"")
+
+
+class _Stringifier:
+ # Must match the slots on ForwardRef, so we can turn an instance of one into an
+ # instance of the other in place.
+ __slots__ = _SLOTS
+
+ def __init__(
+ self,
+ node,
+ globals=None,
+ owner=None,
+ is_class=False,
+ cell=None,
+ *,
+ stringifier_dict,
+ extra_names=None,
+ ):
+ # Either an AST node or a simple str (for the common case where a ForwardRef
+ # represent a single name).
+ assert isinstance(node, (ast.AST, str))
+ self.__arg__ = None
+ self.__forward_is_argument__ = False
+ self.__forward_is_class__ = is_class
+ self.__forward_module__ = None
+ self.__code__ = None
+ self.__ast_node__ = node
+ self.__globals__ = globals
+ self.__extra_names__ = extra_names
+ self.__cell__ = cell
+ self.__owner__ = owner
+ self.__stringifier_dict__ = stringifier_dict
+ self.__resolved_str_cache__ = None # Needed for ForwardRef
+
+ def __convert_to_ast(self, other):
+ if isinstance(other, _Stringifier):
+ if isinstance(other.__ast_node__, str):
+ return ast.Name(id=other.__ast_node__), other.__extra_names__
+ return other.__ast_node__, other.__extra_names__
+ elif type(other) is _Template:
+ return _template_to_ast(other), None
+ elif (
+ # In STRING format we don't bother with the create_unique_name() dance;
+ # it's better to emit the repr() of the object instead of an opaque name.
+ self.__stringifier_dict__.format == Format.STRING
+ or other is None
+ or type(other) in (str, int, float, bool, complex)
+ ):
+ return ast.Constant(value=other), None
+ elif type(other) is dict:
+ extra_names = {}
+ keys = []
+ values = []
+ for key, value in other.items():
+ new_key, new_extra_names = self.__convert_to_ast(key)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ keys.append(new_key)
+ new_value, new_extra_names = self.__convert_to_ast(value)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ values.append(new_value)
+ return ast.Dict(keys, values), extra_names
+ elif type(other) in (list, tuple, set):
+ extra_names = {}
+ elts = []
+ for elt in other:
+ new_elt, new_extra_names = self.__convert_to_ast(elt)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ elts.append(new_elt)
+ ast_class = {list: ast.List, tuple: ast.Tuple, set: ast.Set}[type(other)]
+ return ast_class(elts), extra_names
+ else:
+ name = self.__stringifier_dict__.create_unique_name()
+ return ast.Name(id=name), {name: other}
+
+ def __convert_to_ast_getitem(self, other):
+ if isinstance(other, slice):
+ extra_names = {}
+
+ def conv(obj):
+ if obj is None:
+ return None
+ new_obj, new_extra_names = self.__convert_to_ast(obj)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ return new_obj
+
+ return ast.Slice(
+ lower=conv(other.start),
+ upper=conv(other.stop),
+ step=conv(other.step),
+ ), extra_names
+ else:
+ return self.__convert_to_ast(other)
+
+ def __get_ast(self):
+ node = self.__ast_node__
+ if isinstance(node, str):
+ return ast.Name(id=node)
+ return node
+
+ def __make_new(self, node, extra_names=None):
+ new_extra_names = {}
+ if self.__extra_names__ is not None:
+ new_extra_names.update(self.__extra_names__)
+ if extra_names is not None:
+ new_extra_names.update(extra_names)
+ stringifier = _Stringifier(
+ node,
+ self.__globals__,
+ self.__owner__,
+ self.__forward_is_class__,
+ stringifier_dict=self.__stringifier_dict__,
+ extra_names=new_extra_names or None,
+ )
+ self.__stringifier_dict__.stringifiers.append(stringifier)
+ return stringifier
+
+ # Must implement this since we set __eq__. We hash by identity so that
+ # stringifiers in dict keys are kept separate.
+ def __hash__(self):
+ return id(self)
+
+ def __getitem__(self, other):
+ # Special case, to avoid stringifying references to class-scoped variables
+ # as '__classdict__["x"]'.
+ if self.__ast_node__ == "__classdict__":
+ raise KeyError
+ if isinstance(other, tuple):
+ extra_names = {}
+ elts = []
+ for elt in other:
+ new_elt, new_extra_names = self.__convert_to_ast_getitem(elt)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ elts.append(new_elt)
+ other = ast.Tuple(elts)
+ else:
+ other, extra_names = self.__convert_to_ast_getitem(other)
+ assert isinstance(other, ast.AST), repr(other)
+ return self.__make_new(ast.Subscript(self.__get_ast(), other), extra_names)
+
+ def __getattr__(self, attr):
+ return self.__make_new(ast.Attribute(self.__get_ast(), attr))
+
+ def __call__(self, *args, **kwargs):
+ extra_names = {}
+ ast_args = []
+ for arg in args:
+ new_arg, new_extra_names = self.__convert_to_ast(arg)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ ast_args.append(new_arg)
+ ast_kwargs = []
+ for key, value in kwargs.items():
+ new_value, new_extra_names = self.__convert_to_ast(value)
+ if new_extra_names is not None:
+ extra_names.update(new_extra_names)
+ ast_kwargs.append(ast.keyword(key, new_value))
+ return self.__make_new(ast.Call(self.__get_ast(), ast_args, ast_kwargs), extra_names)
+
+ def __iter__(self):
+ yield self.__make_new(ast.Starred(self.__get_ast()))
+
+ def __repr__(self):
+ if isinstance(self.__ast_node__, str):
+ return self.__ast_node__
+ return ast.unparse(self.__ast_node__)
+
+ def __format__(self, format_spec):
+ raise TypeError("Cannot stringify annotation containing string formatting")
+
+ def _make_binop(op: ast.AST):
+ def binop(self, other):
+ rhs, extra_names = self.__convert_to_ast(other)
+ return self.__make_new(
+ ast.BinOp(self.__get_ast(), op, rhs), extra_names
+ )
+
+ return binop
+
+ __add__ = _make_binop(ast.Add())
+ __sub__ = _make_binop(ast.Sub())
+ __mul__ = _make_binop(ast.Mult())
+ __matmul__ = _make_binop(ast.MatMult())
+ __truediv__ = _make_binop(ast.Div())
+ __mod__ = _make_binop(ast.Mod())
+ __lshift__ = _make_binop(ast.LShift())
+ __rshift__ = _make_binop(ast.RShift())
+ __or__ = _make_binop(ast.BitOr())
+ __xor__ = _make_binop(ast.BitXor())
+ __and__ = _make_binop(ast.BitAnd())
+ __floordiv__ = _make_binop(ast.FloorDiv())
+ __pow__ = _make_binop(ast.Pow())
+
+ del _make_binop
+
+ def _make_rbinop(op: ast.AST):
+ def rbinop(self, other):
+ new_other, extra_names = self.__convert_to_ast(other)
+ return self.__make_new(
+ ast.BinOp(new_other, op, self.__get_ast()), extra_names
+ )
+
+ return rbinop
+
+ __radd__ = _make_rbinop(ast.Add())
+ __rsub__ = _make_rbinop(ast.Sub())
+ __rmul__ = _make_rbinop(ast.Mult())
+ __rmatmul__ = _make_rbinop(ast.MatMult())
+ __rtruediv__ = _make_rbinop(ast.Div())
+ __rmod__ = _make_rbinop(ast.Mod())
+ __rlshift__ = _make_rbinop(ast.LShift())
+ __rrshift__ = _make_rbinop(ast.RShift())
+ __ror__ = _make_rbinop(ast.BitOr())
+ __rxor__ = _make_rbinop(ast.BitXor())
+ __rand__ = _make_rbinop(ast.BitAnd())
+ __rfloordiv__ = _make_rbinop(ast.FloorDiv())
+ __rpow__ = _make_rbinop(ast.Pow())
+
+ del _make_rbinop
+
+ def _make_compare(op):
+ def compare(self, other):
+ rhs, extra_names = self.__convert_to_ast(other)
+ return self.__make_new(
+ ast.Compare(
+ left=self.__get_ast(),
+ ops=[op],
+ comparators=[rhs],
+ ),
+ extra_names,
+ )
+
+ return compare
+
+ __lt__ = _make_compare(ast.Lt())
+ __le__ = _make_compare(ast.LtE())
+ __eq__ = _make_compare(ast.Eq())
+ __ne__ = _make_compare(ast.NotEq())
+ __gt__ = _make_compare(ast.Gt())
+ __ge__ = _make_compare(ast.GtE())
+
+ del _make_compare
+
+ def _make_unary_op(op):
+ def unary_op(self):
+ return self.__make_new(ast.UnaryOp(op, self.__get_ast()))
+
+ return unary_op
+
+ __invert__ = _make_unary_op(ast.Invert())
+ __pos__ = _make_unary_op(ast.UAdd())
+ __neg__ = _make_unary_op(ast.USub())
+
+ del _make_unary_op
+
+
+def _template_to_ast_constructor(template):
+ """Convert a `template` instance to a non-literal AST."""
+ args = []
+ for part in template:
+ match part:
+ case str():
+ args.append(ast.Constant(value=part))
+ case _:
+ interp = ast.Call(
+ func=ast.Name(id="Interpolation"),
+ args=[
+ ast.Constant(value=part.value),
+ ast.Constant(value=part.expression),
+ ast.Constant(value=part.conversion),
+ ast.Constant(value=part.format_spec),
+ ]
+ )
+ args.append(interp)
+ return ast.Call(func=ast.Name(id="Template"), args=args, keywords=[])
+
+
+def _template_to_ast_literal(template, parsed):
+ """Convert a `template` instance to a t-string literal AST."""
+ values = []
+ interp_count = 0
+ for part in template:
+ match part:
+ case str():
+ values.append(ast.Constant(value=part))
+ case _:
+ interp = ast.Interpolation(
+ str=part.expression,
+ value=parsed[interp_count],
+ conversion=ord(part.conversion) if part.conversion else -1,
+ format_spec=ast.Constant(value=part.format_spec)
+ if part.format_spec
+ else None,
+ )
+ values.append(interp)
+ interp_count += 1
+ return ast.TemplateStr(values=values)
+
+
+def _template_to_ast(template):
+ """Make a best-effort conversion of a `template` instance to an AST."""
+ # gh-138558: Not all Template instances can be represented as t-string
+ # literals. Return the most accurate AST we can. See issue for details.
+
+ # If any expr is empty or whitespace only, we cannot convert to a literal.
+ if any(part.expression.strip() == "" for part in template.interpolations):
+ return _template_to_ast_constructor(template)
+
+ try:
+ # Wrap in parens to allow whitespace inside interpolation curly braces
+ parsed = tuple(
+ ast.parse(f"({part.expression})", mode="eval").body
+ for part in template.interpolations
+ )
+ except SyntaxError:
+ return _template_to_ast_constructor(template)
+
+ return _template_to_ast_literal(template, parsed)
+
+
+class _StringifierDict(dict):
+ def __init__(self, namespace, *, globals=None, owner=None, is_class=False, format):
+ super().__init__(namespace)
+ self.namespace = namespace
+ self.globals = globals
+ self.owner = owner
+ self.is_class = is_class
+ self.stringifiers = []
+ self.next_id = 1
+ self.format = format
+
+ def __missing__(self, key):
+ fwdref = _Stringifier(
+ key,
+ globals=self.globals,
+ owner=self.owner,
+ is_class=self.is_class,
+ stringifier_dict=self,
+ )
+ self.stringifiers.append(fwdref)
+ return fwdref
+
+ def transmogrify(self, cell_dict):
+ for obj in self.stringifiers:
+ obj.__class__ = ForwardRef
+ obj.__stringifier_dict__ = None # not needed for ForwardRef
+ if isinstance(obj.__ast_node__, str):
+ obj.__arg__ = obj.__ast_node__
+ obj.__ast_node__ = None
+ if cell_dict is not None and obj.__cell__ is None:
+ obj.__cell__ = cell_dict
+
+ def create_unique_name(self):
+ name = f"__annotationlib_name_{self.next_id}__"
+ self.next_id += 1
+ return name
+
+
+def call_evaluate_function(evaluate, format, *, owner=None):
+ """Call an evaluate function. Evaluate functions are normally generated for
+ the value of type aliases and the bounds, constraints, and defaults of
+ type parameter objects.
+ """
+ return call_annotate_function(evaluate, format, owner=owner, _is_evaluate=True)
+
+
+def call_annotate_function(annotate, format, *, owner=None, _is_evaluate=False):
+ """Call an __annotate__ function. __annotate__ functions are normally
+ generated by the compiler to defer the evaluation of annotations. They
+ can be called with any of the format arguments in the Format enum, but
+ compiler-generated __annotate__ functions only support the VALUE format.
+ This function provides additional functionality to call __annotate__
+ functions with the FORWARDREF and STRING formats.
+
+ *annotate* must be an __annotate__ function, which takes a single argument
+ and returns a dict of annotations.
+
+ *format* must be a member of the Format enum or one of the corresponding
+ integer values.
+
+ *owner* can be the object that owns the annotations (i.e., the module,
+ class, or function that the __annotate__ function derives from). With the
+ FORWARDREF format, it is used to provide better evaluation capabilities
+ on the generated ForwardRef objects.
+
+ """
+ if format == Format.VALUE_WITH_FAKE_GLOBALS:
+ raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only")
+ try:
+ return annotate(format)
+ except NotImplementedError:
+ pass
+ if format == Format.STRING:
+ # STRING is implemented by calling the annotate function in a special
+ # environment where every name lookup results in an instance of _Stringifier.
+ # _Stringifier supports every dunder operation and returns a new _Stringifier.
+ # At the end, we get a dictionary that mostly contains _Stringifier objects (or
+ # possibly constants if the annotate function uses them directly). We then
+ # convert each of those into a string to get an approximation of the
+ # original source.
+
+ # Attempt to call with VALUE_WITH_FAKE_GLOBALS to check if it is implemented
+ # See: https://github.com/python/cpython/issues/138764
+ # Only fail on NotImplementedError
+ try:
+ annotate(Format.VALUE_WITH_FAKE_GLOBALS)
+ except NotImplementedError:
+ # Both STRING and VALUE_WITH_FAKE_GLOBALS are not implemented: fallback to VALUE
+ return annotations_to_string(annotate(Format.VALUE))
+ except Exception:
+ pass
+
+ globals = _StringifierDict({}, format=format)
+ is_class = isinstance(owner, type)
+ closure, _ = _build_closure(
+ annotate, owner, is_class, globals, allow_evaluation=False
+ )
+ func = types.FunctionType(
+ annotate.__code__,
+ globals,
+ closure=closure,
+ argdefs=annotate.__defaults__,
+ kwdefaults=annotate.__kwdefaults__,
+ )
+ annos = func(Format.VALUE_WITH_FAKE_GLOBALS)
+ if _is_evaluate:
+ return _stringify_single(annos)
+ return {
+ key: _stringify_single(val)
+ for key, val in annos.items()
+ }
+ elif format == Format.FORWARDREF:
+ # FORWARDREF is implemented similarly to STRING, but there are two changes,
+ # at the beginning and the end of the process.
+ # First, while STRING uses an empty dictionary as the namespace, so that all
+ # name lookups result in _Stringifier objects, FORWARDREF uses the globals
+ # and builtins, so that defined names map to their real values.
+ # Second, instead of returning strings, we want to return either real values
+ # or ForwardRef objects. To do this, we keep track of all _Stringifier objects
+ # created while the annotation is being evaluated, and at the end we convert
+ # them all to ForwardRef objects by assigning to __class__. To make this
+ # technique work, we have to ensure that the _Stringifier and ForwardRef
+ # classes share the same attributes.
+ # We use this technique because while the annotations are being evaluated,
+ # we want to support all operations that the language allows, including even
+ # __getattr__ and __eq__, and return new _Stringifier objects so we can accurately
+ # reconstruct the source. But in the dictionary that we eventually return, we
+ # want to return objects with more user-friendly behavior, such as an __eq__
+ # that returns a bool and an defined set of attributes.
+ namespace = {**annotate.__builtins__, **annotate.__globals__}
+ is_class = isinstance(owner, type)
+ globals = _StringifierDict(
+ namespace,
+ globals=annotate.__globals__,
+ owner=owner,
+ is_class=is_class,
+ format=format,
+ )
+ closure, cell_dict = _build_closure(
+ annotate, owner, is_class, globals, allow_evaluation=True
+ )
+ func = types.FunctionType(
+ annotate.__code__,
+ globals,
+ closure=closure,
+ argdefs=annotate.__defaults__,
+ kwdefaults=annotate.__kwdefaults__,
+ )
+ try:
+ result = func(Format.VALUE_WITH_FAKE_GLOBALS)
+ except NotImplementedError:
+ # FORWARDREF and VALUE_WITH_FAKE_GLOBALS not supported, fall back to VALUE
+ return annotate(Format.VALUE)
+ except Exception:
+ pass
+ else:
+ globals.transmogrify(cell_dict)
+ return result
+
+ # Try again, but do not provide any globals. This allows us to return
+ # a value in certain cases where an exception gets raised during evaluation.
+ globals = _StringifierDict(
+ {},
+ globals=annotate.__globals__,
+ owner=owner,
+ is_class=is_class,
+ format=format,
+ )
+ closure, cell_dict = _build_closure(
+ annotate, owner, is_class, globals, allow_evaluation=False
+ )
+ func = types.FunctionType(
+ annotate.__code__,
+ globals,
+ closure=closure,
+ argdefs=annotate.__defaults__,
+ kwdefaults=annotate.__kwdefaults__,
+ )
+ result = func(Format.VALUE_WITH_FAKE_GLOBALS)
+ globals.transmogrify(cell_dict)
+ if _is_evaluate:
+ if isinstance(result, ForwardRef):
+ return result.evaluate(format=Format.FORWARDREF)
+ else:
+ return result
+ else:
+ return {
+ key: (
+ val.evaluate(format=Format.FORWARDREF)
+ if isinstance(val, ForwardRef)
+ else val
+ )
+ for key, val in result.items()
+ }
+ elif format == Format.VALUE:
+ # Should be impossible because __annotate__ functions must not raise
+ # NotImplementedError for this format.
+ raise RuntimeError("annotate function does not support VALUE format")
+ else:
+ raise ValueError(f"Invalid format: {format!r}")
+
+
+def _build_closure(annotate, owner, is_class, stringifier_dict, *, allow_evaluation):
+ if not annotate.__closure__:
+ return None, None
+ new_closure = []
+ cell_dict = {}
+ for name, cell in zip(annotate.__code__.co_freevars, annotate.__closure__, strict=True):
+ cell_dict[name] = cell
+ new_cell = None
+ if allow_evaluation:
+ try:
+ cell.cell_contents
+ except ValueError:
+ pass
+ else:
+ new_cell = cell
+ if new_cell is None:
+ fwdref = _Stringifier(
+ name,
+ cell=cell,
+ owner=owner,
+ globals=annotate.__globals__,
+ is_class=is_class,
+ stringifier_dict=stringifier_dict,
+ )
+ stringifier_dict.stringifiers.append(fwdref)
+ new_cell = types.CellType(fwdref)
+ new_closure.append(new_cell)
+ return tuple(new_closure), cell_dict
+
+
+def _stringify_single(anno):
+ if anno is ...:
+ return "..."
+ # We have to handle str specially to support PEP 563 stringified annotations.
+ elif isinstance(anno, str):
+ return anno
+ elif isinstance(anno, _Template):
+ return ast.unparse(_template_to_ast(anno))
+ else:
+ return repr(anno)
+
+
+def get_annotate_from_class_namespace(obj):
+ """Retrieve the annotate function from a class namespace dictionary.
+
+ Return None if the namespace does not contain an annotate function.
+ This is useful in metaclass ``__new__`` methods to retrieve the annotate function.
+ """
+ try:
+ return obj["__annotate__"]
+ except KeyError:
+ return obj.get("__annotate_func__", None)
+
+
+def get_annotations(
+ obj, *, globals=None, locals=None, eval_str=False, format=Format.VALUE
+):
+ """Compute the annotations dict for an object.
+
+ obj may be a callable, class, module, or other object with
+ __annotate__ or __annotations__ attributes.
+ Passing any other object raises TypeError.
+
+ The *format* parameter controls the format in which annotations are returned,
+ and must be a member of the Format enum or its integer equivalent.
+ For the VALUE format, the __annotations__ is tried first; if it
+ does not exist, the __annotate__ function is called. The
+ FORWARDREF format uses __annotations__ if it exists and can be
+ evaluated, and otherwise falls back to calling the __annotate__ function.
+ The STRING format tries __annotate__ first, and falls back to
+ using __annotations__, stringified using annotations_to_string().
+
+ This function handles several details for you:
+
+ * If eval_str is true, values of type str will
+ be un-stringized using eval(). This is intended
+ for use with stringized annotations
+ ("from __future__ import annotations").
+ * If obj doesn't have an annotations dict, returns an
+ empty dict. (Functions and methods always have an
+ annotations dict; classes, modules, and other types of
+ callables may not.)
+ * Ignores inherited annotations on classes. If a class
+ doesn't have its own annotations dict, returns an empty dict.
+ * All accesses to object members and dict values are done
+ using getattr() and dict.get() for safety.
+ * Always, always, always returns a freshly-created dict.
+
+ eval_str controls whether or not values of type str are replaced
+ with the result of calling eval() on those values:
+
+ * If eval_str is true, eval() is called on values of type str.
+ * If eval_str is false (the default), values of type str are unchanged.
+
+ globals and locals are passed in to eval(); see the documentation
+ for eval() for more information. If either globals or locals is
+ None, this function may replace that value with a context-specific
+ default, contingent on type(obj):
+
+ * If obj is a module, globals defaults to obj.__dict__.
+ * If obj is a class, globals defaults to
+ sys.modules[obj.__module__].__dict__ and locals
+ defaults to the obj class namespace.
+ * If obj is a callable, globals defaults to obj.__globals__,
+ although if obj is a wrapped function (using
+ functools.update_wrapper()) it is first unwrapped.
+ """
+ if eval_str and format != Format.VALUE:
+ raise ValueError("eval_str=True is only supported with format=Format.VALUE")
+
+ match format:
+ case Format.VALUE:
+ # For VALUE, we first look at __annotations__
+ ann = _get_dunder_annotations(obj)
+
+ # If it's not there, try __annotate__ instead
+ if ann is None:
+ ann = _get_and_call_annotate(obj, format)
+ case Format.FORWARDREF:
+ # For FORWARDREF, we use __annotations__ if it exists
+ try:
+ ann = _get_dunder_annotations(obj)
+ except Exception:
+ pass
+ else:
+ if ann is not None:
+ return dict(ann)
+
+ # But if __annotations__ threw a NameError, we try calling __annotate__
+ ann = _get_and_call_annotate(obj, format)
+ if ann is None:
+ # If that didn't work either, we have a very weird object: evaluating
+ # __annotations__ threw NameError and there is no __annotate__. In that case,
+ # we fall back to trying __annotations__ again.
+ ann = _get_dunder_annotations(obj)
+ case Format.STRING:
+ # For STRING, we try to call __annotate__
+ ann = _get_and_call_annotate(obj, format)
+ if ann is not None:
+ return dict(ann)
+ # But if we didn't get it, we use __annotations__ instead.
+ ann = _get_dunder_annotations(obj)
+ if ann is not None:
+ return annotations_to_string(ann)
+ case Format.VALUE_WITH_FAKE_GLOBALS:
+ raise ValueError("The VALUE_WITH_FAKE_GLOBALS format is for internal use only")
+ case _:
+ raise ValueError(f"Unsupported format {format!r}")
+
+ if ann is None:
+ if isinstance(obj, type) or callable(obj):
+ return {}
+ raise TypeError(f"{obj!r} does not have annotations")
+
+ if not ann:
+ return {}
+
+ if not eval_str:
+ return dict(ann)
+
+ if globals is None or locals is None:
+ if isinstance(obj, type):
+ # class
+ obj_globals = None
+ module_name = getattr(obj, "__module__", None)
+ if module_name:
+ module = sys.modules.get(module_name, None)
+ if module:
+ obj_globals = getattr(module, "__dict__", None)
+ obj_locals = dict(vars(obj))
+ unwrap = obj
+ elif isinstance(obj, types.ModuleType):
+ # module
+ obj_globals = getattr(obj, "__dict__")
+ obj_locals = None
+ unwrap = None
+ elif callable(obj):
+ # this includes types.Function, types.BuiltinFunctionType,
+ # types.BuiltinMethodType, functools.partial, functools.singledispatch,
+ # "class funclike" from Lib/test/test_inspect... on and on it goes.
+ obj_globals = getattr(obj, "__globals__", None)
+ obj_locals = None
+ unwrap = obj
+ else:
+ obj_globals = obj_locals = unwrap = None
+
+ if unwrap is not None:
+ # Use an id-based visited set to detect cycles in the __wrapped__
+ # and functools.partial.func chain (e.g. f.__wrapped__ = f).
+ # On cycle detection we stop and use whatever __globals__ we have
+ # found so far, mirroring the approach of inspect.unwrap().
+ _seen_ids = {id(unwrap)}
+ while True:
+ if hasattr(unwrap, "__wrapped__"):
+ candidate = unwrap.__wrapped__
+ if id(candidate) in _seen_ids:
+ break
+ _seen_ids.add(id(candidate))
+ unwrap = candidate
+ continue
+ if functools := sys.modules.get("functools"):
+ if isinstance(unwrap, functools.partial):
+ candidate = unwrap.func
+ if id(candidate) in _seen_ids:
+ break
+ _seen_ids.add(id(candidate))
+ unwrap = candidate
+ continue
+ break
+ if hasattr(unwrap, "__globals__"):
+ obj_globals = unwrap.__globals__
+
+ if globals is None:
+ globals = obj_globals
+ if locals is None:
+ locals = obj_locals
+
+ # "Inject" type parameters into the local namespace
+ # (unless they are shadowed by assignments *in* the local namespace),
+ # as a way of emulating annotation scopes when calling `eval()`
+ if type_params := getattr(obj, "__type_params__", ()):
+ if locals is None:
+ locals = {}
+ locals = {param.__name__: param for param in type_params} | locals
+
+ return_value = {
+ key: value if not isinstance(value, str)
+ else eval(_rewrite_star_unpack(value), globals, locals)
+ for key, value in ann.items()
+ }
+ return return_value
+
+
+def type_repr(value):
+ """Convert a Python value to a format suitable for use with the STRING format.
+
+ This is intended as a helper for tools that support the STRING format but do
+ not have access to the code that originally produced the annotations. It uses
+ repr() for most objects.
+
+ """
+ if isinstance(value, (type, types.FunctionType, types.BuiltinFunctionType)):
+ if value.__module__ == "builtins":
+ return value.__qualname__
+ return f"{value.__module__}.{value.__qualname__}"
+ elif isinstance(value, _Template):
+ tree = _template_to_ast(value)
+ return ast.unparse(tree)
+ if value is ...:
+ return "..."
+ return repr(value)
+
+
+def annotations_to_string(annotations):
+ """Convert an annotation dict containing values to approximately the STRING format.
+
+ Always returns a fresh a dictionary.
+ """
+ return {
+ n: t if isinstance(t, str) else type_repr(t)
+ for n, t in annotations.items()
+ }
+
+
+def _rewrite_star_unpack(arg):
+ """If the given argument annotation expression is a star unpack e.g. `'*Ts'`
+ rewrite it to a valid expression.
+ """
+ if arg.lstrip().startswith("*"):
+ return f"({arg},)[0]" # E.g. (*Ts,)[0] or (*tuple[int, int],)[0]
+ else:
+ return arg
+
+
+def _get_and_call_annotate(obj, format):
+ """Get the __annotate__ function and call it.
+
+ May not return a fresh dictionary.
+ """
+ annotate = getattr(obj, "__annotate__", None)
+ if annotate is not None:
+ ann = call_annotate_function(annotate, format, owner=obj)
+ if not isinstance(ann, dict):
+ raise ValueError(f"{obj!r}.__annotate__ returned a non-dict")
+ return ann
+ return None
+
+
+_BASE_GET_ANNOTATIONS = type.__dict__["__annotations__"].__get__
+
+
+def _get_dunder_annotations(obj):
+ """Return the annotations for an object, checking that it is a dictionary.
+
+ Does not return a fresh dictionary.
+ """
+ # This special case is needed to support types defined under
+ # from __future__ import annotations, where accessing the __annotations__
+ # attribute directly might return annotations for the wrong class.
+ if isinstance(obj, type):
+ try:
+ ann = _BASE_GET_ANNOTATIONS(obj)
+ except AttributeError:
+ # For static types, the descriptor raises AttributeError.
+ return None
+ else:
+ ann = getattr(obj, "__annotations__", None)
+ if ann is None:
+ return None
+
+ if not isinstance(ann, dict):
+ raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None")
+ return ann
+
+
+class _ExtraNameFixer(ast.NodeTransformer):
+ """Fixer for __extra_names__ items in ForwardRef __repr__ and string evaluation"""
+ def __init__(self, extra_names):
+ self.extra_names = extra_names
+
+ def visit_Name(self, node: ast.Name):
+ if (new_name := self.extra_names.get(node.id, _sentinel)) is not _sentinel:
+ node = ast.Name(id=type_repr(new_name))
+ return node
diff --git a/PythonLib/full/argparse.py b/PythonLib/full/argparse.py
index 2a8b501a4..24f474056 100644
--- a/PythonLib/full/argparse.py
+++ b/PythonLib/full/argparse.py
@@ -18,11 +18,12 @@
'integers', metavar='int', nargs='+', type=int,
help='an integer to be summed')
parser.add_argument(
- '--log', default=sys.stdout, type=argparse.FileType('w'),
+ '--log',
help='the file where the sum should be written')
args = parser.parse_args()
- args.log.write('%s' % sum(args.integers))
- args.log.close()
+ with (open(args.log, 'w') if args.log is not None
+ else contextlib.nullcontext(sys.stdout)) as log:
+ log.write('%s' % sum(args.integers))
The module contains the following public classes:
@@ -39,7 +40,8 @@
- FileType -- A factory for defining types of files to be created. As the
example above shows, instances of FileType are typically passed as
- the type= argument of add_argument() calls.
+ the type= argument of add_argument() calls. Deprecated since
+ Python 3.14.
- Action -- The base class for parser actions. Typically actions are
selected by passing strings like 'store_true' or 'append_const' to
@@ -89,8 +91,6 @@
import re as _re
import sys as _sys
-import warnings
-
from gettext import gettext as _, ngettext
SUPPRESS = '==SUPPRESS=='
@@ -149,6 +149,10 @@ def _copy_items(items):
return copy.copy(items)
+def _identity(value):
+ return value
+
+
# ===============
# Formatting Help
# ===============
@@ -161,18 +165,21 @@ class HelpFormatter(object):
provided by the class are considered an implementation detail.
"""
- def __init__(self,
- prog,
- indent_increment=2,
- max_help_position=24,
- width=None):
-
+ def __init__(
+ self,
+ prog,
+ indent_increment=2,
+ max_help_position=24,
+ width=None,
+ color=True,
+ ):
# default setting for width
if width is None:
import shutil
width = shutil.get_terminal_size().columns
width -= 2
+ self._set_color(color)
self._prog = prog
self._indent_increment = indent_increment
self._max_help_position = min(max_help_position,
@@ -189,9 +196,20 @@ def __init__(self,
self._whitespace_matcher = _re.compile(r'\s+', _re.ASCII)
self._long_break_matcher = _re.compile(r'\n\n\n+')
+ def _set_color(self, color):
+ from _colorize import can_colorize, decolor, get_theme
+
+ if color and can_colorize():
+ self._theme = get_theme(force_color=True).argparse
+ self._decolor = decolor
+ else:
+ self._theme = get_theme(force_no_color=True).argparse
+ self._decolor = _identity
+
# ===============================
# Section and indentation methods
# ===============================
+
def _indent(self):
self._current_indent += self._indent_increment
self._level += 1
@@ -226,7 +244,11 @@ def format_help(self):
if self.heading is not SUPPRESS and self.heading is not None:
current_indent = self.formatter._current_indent
heading_text = _('%(heading)s:') % dict(heading=self.heading)
- heading = '%*s%s\n' % (current_indent, '', heading_text)
+ t = self.formatter._theme
+ heading = (
+ f'{" " * current_indent}'
+ f'{t.heading}{heading_text}{t.reset}\n'
+ )
else:
heading = ''
@@ -239,6 +261,7 @@ def _add_item(self, func, args):
# ========================
# Message building methods
# ========================
+
def start_section(self, heading):
self._indent()
section = self._Section(self, self._current_section, heading)
@@ -262,7 +285,7 @@ def add_argument(self, action):
if action.help is not SUPPRESS:
# find all invocations
- get_invocation = self._format_action_invocation
+ get_invocation = lambda x: self._decolor(self._format_action_invocation(x))
invocation_lengths = [len(get_invocation(action)) + self._current_indent]
for subaction in self._iter_indented_subactions(action):
invocation_lengths.append(len(get_invocation(subaction)) + self._current_indent)
@@ -282,6 +305,7 @@ def add_arguments(self, actions):
# =======================
# Help-formatting methods
# =======================
+
def format_help(self):
help = self._root_section.format_help()
if help:
@@ -295,51 +319,39 @@ def _join_parts(self, part_strings):
if part and part is not SUPPRESS])
def _format_usage(self, usage, actions, groups, prefix):
+ t = self._theme
+
if prefix is None:
prefix = _('usage: ')
# if usage is specified, use that
if usage is not None:
- usage = usage % dict(prog=self._prog)
+ usage = (
+ t.prog_extra
+ + usage
+ % {"prog": f"{t.prog}{self._prog}{t.reset}{t.prog_extra}"}
+ + t.reset
+ )
# if no optionals or positionals are available, usage is just prog
elif usage is None and not actions:
- usage = '%(prog)s' % dict(prog=self._prog)
+ usage = f"{t.prog}{self._prog}{t.reset}"
# if optionals and positionals are available, calculate usage
elif usage is None:
prog = '%(prog)s' % dict(prog=self._prog)
- # split optionals from positionals
- optionals = []
- positionals = []
- for action in actions:
- if action.option_strings:
- optionals.append(action)
- else:
- positionals.append(action)
-
+ parts, pos_start = self._get_actions_usage_parts(actions, groups)
# build full usage string
- format = self._format_actions_usage
- action_usage = format(optionals + positionals, groups)
- usage = ' '.join([s for s in [prog, action_usage] if s])
+ usage = ' '.join(filter(None, [prog, *parts]))
# wrap the usage parts if it's too long
text_width = self._width - self._current_indent
- if len(prefix) + len(usage) > text_width:
+ if len(prefix) + len(self._decolor(usage)) > text_width:
# break usage into wrappable parts
- part_regexp = (
- r'\(.*?\)+(?=\s|$)|'
- r'\[.*?\]+(?=\s|$)|'
- r'\S+'
- )
- opt_usage = format(optionals, groups)
- pos_usage = format(positionals, groups)
- opt_parts = _re.findall(part_regexp, opt_usage)
- pos_parts = _re.findall(part_regexp, pos_usage)
- assert ' '.join(opt_parts) == opt_usage
- assert ' '.join(pos_parts) == pos_usage
+ opt_parts = parts[:pos_start]
+ pos_parts = parts[pos_start:]
# helper for wrapping lines
def get_lines(parts, indent, prefix=None):
@@ -351,12 +363,13 @@ def get_lines(parts, indent, prefix=None):
else:
line_len = indent_length - 1
for part in parts:
- if line_len + 1 + len(part) > text_width and line:
+ part_len = len(self._decolor(part))
+ if line_len + 1 + part_len > text_width and line:
lines.append(indent + ' '.join(line))
line = []
line_len = indent_length - 1
line.append(part)
- line_len += len(part) + 1
+ line_len += part_len + 1
if line:
lines.append(indent + ' '.join(line))
if prefix is not None:
@@ -364,8 +377,9 @@ def get_lines(parts, indent, prefix=None):
return lines
# if prog is short, follow it with optionals or positionals
- if len(prefix) + len(prog) <= 0.75 * text_width:
- indent = ' ' * (len(prefix) + len(prog) + 1)
+ prog_len = len(self._decolor(prog))
+ if len(prefix) + prog_len <= 0.75 * text_width:
+ indent = ' ' * (len(prefix) + prog_len + 1)
if opt_parts:
lines = get_lines([prog] + opt_parts, indent, prefix)
lines.extend(get_lines(pos_parts, indent))
@@ -388,123 +402,120 @@ def get_lines(parts, indent, prefix=None):
# join lines into usage
usage = '\n'.join(lines)
+ usage = usage.removeprefix(prog)
+ usage = f"{t.prog}{prog}{t.reset}{usage}"
+
# prefix with 'usage:'
- return '%s%s\n\n' % (prefix, usage)
+ return f'{t.usage}{prefix}{t.reset}{usage}\n\n'
- def _format_actions_usage(self, actions, groups):
- # find group indices and identify actions in groups
- group_actions = set()
- inserts = {}
- for group in groups:
- if not group._group_actions:
- raise ValueError(f'empty group {group}')
+ def _is_long_option(self, string):
+ return len(string) > 2
- try:
- start = actions.index(group._group_actions[0])
- except ValueError:
- continue
- else:
- group_action_count = len(group._group_actions)
- end = start + group_action_count
- if actions[start:end] == group._group_actions:
-
- suppressed_actions_count = 0
- for action in group._group_actions:
- group_actions.add(action)
- if action.help is SUPPRESS:
- suppressed_actions_count += 1
-
- exposed_actions_count = group_action_count - suppressed_actions_count
- if not exposed_actions_count:
- continue
-
- if not group.required:
- if start in inserts:
- inserts[start] += ' ['
- else:
- inserts[start] = '['
- if end in inserts:
- inserts[end] += ']'
- else:
- inserts[end] = ']'
- elif exposed_actions_count > 1:
- if start in inserts:
- inserts[start] += ' ('
- else:
- inserts[start] = '('
- if end in inserts:
- inserts[end] += ')'
- else:
- inserts[end] = ')'
- for i in range(start + 1, end):
- inserts[i] = '|'
+ def _get_actions_usage_parts(self, actions, groups):
+ """Get usage parts with split index for optionals/positionals.
+
+ Returns (parts, pos_start) where pos_start is the index in parts
+ where positionals begin.
+ This preserves mutually exclusive group formatting across the
+ optionals/positionals boundary (gh-75949).
+ """
+ actions = [action for action in actions if action.help is not SUPPRESS]
+ # group actions by mutually exclusive groups
+ action_groups = dict.fromkeys(actions)
+ for group in groups:
+ for action in group._group_actions:
+ if action in action_groups:
+ action_groups[action] = group
+ # positional arguments keep their position
+ positionals = []
+ for action in actions:
+ if not action.option_strings:
+ group = action_groups.pop(action)
+ if group:
+ group_actions = [
+ action2 for action2 in group._group_actions
+ if action2.option_strings and
+ action_groups.pop(action2, None)
+ ] + [action]
+ positionals.append((group.required, group_actions))
+ else:
+ positionals.append((None, [action]))
+ # the remaining optional arguments are sorted by the position of
+ # the first option in the group
+ optionals = []
+ for action in actions:
+ if action.option_strings and action in action_groups:
+ group = action_groups.pop(action)
+ if group:
+ group_actions = [action] + [
+ action2 for action2 in group._group_actions
+ if action2.option_strings and
+ action_groups.pop(action2, None)
+ ]
+ optionals.append((group.required, group_actions))
+ else:
+ optionals.append((None, [action]))
# collect all actions format strings
parts = []
- for i, action in enumerate(actions):
-
- # suppressed arguments are marked with None
- # remove | separators for suppressed arguments
- if action.help is SUPPRESS:
- parts.append(None)
- if inserts.get(i) == '|':
- inserts.pop(i)
- elif inserts.get(i + 1) == '|':
- inserts.pop(i + 1)
-
- # produce all arg strings
- elif not action.option_strings:
- default = self._get_default_metavar_for_positional(action)
- part = self._format_args(action, default)
-
- # if it's in a group, strip the outer []
- if action in group_actions:
- if part[0] == '[' and part[-1] == ']':
- part = part[1:-1]
-
- # add the action string to the list
- parts.append(part)
-
- # produce the first way to invoke the option in brackets
- else:
- option_string = action.option_strings[0]
+ t = self._theme
+ pos_start = None
+ for i, (required, group) in enumerate(optionals + positionals):
+ start = len(parts)
+ if i == len(optionals):
+ pos_start = start
+ in_group = len(group) > 1
+ for action in group:
+ # produce all arg strings
+ if not action.option_strings:
+ default = self._get_default_metavar_for_positional(action)
+ part = self._format_args(action, default)
+ # if it's in a group, strip the outer []
+ if in_group:
+ if part[0] == '[' and part[-1] == ']':
+ part = part[1:-1]
+ part = t.summary_action + part + t.reset
+
+ # produce the first way to invoke the option in brackets
+ else:
+ option_string = action.option_strings[0]
+ if self._is_long_option(option_string):
+ option_color = t.summary_long_option
+ else:
+ option_color = t.summary_short_option
- # if the Optional doesn't take a value, format is:
- # -s or --long
- if action.nargs == 0:
- part = action.format_usage()
+ # if the Optional doesn't take a value, format is:
+ # -s or --long
+ if action.nargs == 0:
+ part = action.format_usage()
+ part = f"{option_color}{part}{t.reset}"
- # if the Optional takes a value, format is:
- # -s ARGS or --long ARGS
- else:
- default = self._get_default_metavar_for_optional(action)
- args_string = self._format_args(action, default)
- part = '%s %s' % (option_string, args_string)
+ # if the Optional takes a value, format is:
+ # -s ARGS or --long ARGS
+ else:
+ default = self._get_default_metavar_for_optional(action)
+ args_string = self._format_args(action, default)
+ part = (
+ f"{option_color}{option_string} "
+ f"{t.summary_label}{args_string}{t.reset}"
+ )
- # make it look optional if it's not required or in a group
- if not action.required and action not in group_actions:
- part = '[%s]' % part
+ # make it look optional if it's not required or in a group
+ if not (action.required or required or in_group):
+ part = '[%s]' % part
# add the action string to the list
parts.append(part)
- # insert things at the necessary indices
- for i in sorted(inserts, reverse=True):
- parts[i:i] = [inserts[i]]
-
- # join all the action items with spaces
- text = ' '.join([item for item in parts if item is not None])
+ if in_group:
+ parts[start] = ('(' if required else '[') + parts[start]
+ for i in range(start, len(parts) - 1):
+ parts[i] += ' |'
+ parts[-1] += ')' if required else ']'
- # clean up separators for mutually exclusive groups
- open = r'[\[(]'
- close = r'[\])]'
- text = _re.sub(r'(%s) ' % open, r'\1', text)
- text = _re.sub(r' (%s)' % close, r'\1', text)
- text = _re.sub(r'%s *%s' % (open, close), r'', text)
- text = text.strip()
-
- # return the text
- return text
+ if pos_start is None:
+ pos_start = len(parts)
+ return parts, pos_start
def _format_text(self, text):
if '%(prog)' in text:
@@ -520,6 +531,7 @@ def _format_action(self, action):
help_width = max(self._width - help_position, 11)
action_width = help_position - self._current_indent - 2
action_header = self._format_action_invocation(action)
+ action_header_no_color = self._decolor(action_header)
# no help; start on same line and add a final newline
if not action.help:
@@ -527,9 +539,15 @@ def _format_action(self, action):
action_header = '%*s%s\n' % tup
# short action name; start on the same line and pad two spaces
- elif len(action_header) <= action_width:
- tup = self._current_indent, '', action_width, action_header
+ elif len(action_header_no_color) <= action_width:
+ # calculate widths without color codes
+ action_header_color = action_header
+ tup = self._current_indent, '', action_width, action_header_no_color
action_header = '%*s%-*s ' % tup
+ # swap in the colored header
+ action_header = action_header.replace(
+ action_header_no_color, action_header_color
+ )
indent_first = 0
# long action name; start on the next line
@@ -562,27 +580,42 @@ def _format_action(self, action):
return self._join_parts(parts)
def _format_action_invocation(self, action):
+ t = self._theme
+
if not action.option_strings:
default = self._get_default_metavar_for_positional(action)
- return ' '.join(self._metavar_formatter(action, default)(1))
+ return (
+ t.action
+ + ' '.join(self._metavar_formatter(action, default)(1))
+ + t.reset
+ )
else:
- parts = []
+
+ def color_option_strings(strings):
+ parts = []
+ for s in strings:
+ if self._is_long_option(s):
+ parts.append(f"{t.long_option}{s}{t.reset}")
+ else:
+ parts.append(f"{t.short_option}{s}{t.reset}")
+ return parts
# if the Optional doesn't take a value, format is:
# -s, --long
if action.nargs == 0:
- parts.extend(action.option_strings)
+ option_strings = color_option_strings(action.option_strings)
+ return ', '.join(option_strings)
# if the Optional takes a value, format is:
- # -s ARGS, --long ARGS
+ # -s, --long ARGS
else:
default = self._get_default_metavar_for_optional(action)
- args_string = self._format_args(action, default)
- for option_string in action.option_strings:
- parts.append('%s %s' % (option_string, args_string))
-
- return ', '.join(parts)
+ option_strings = color_option_strings(action.option_strings)
+ args_string = (
+ f"{t.label}{self._format_args(action, default)}{t.reset}"
+ )
+ return ', '.join(option_strings) + ' ' + args_string
def _metavar_formatter(self, action, default_metavar):
if action.metavar is not None:
@@ -628,16 +661,19 @@ def _format_args(self, action, default_metavar):
return result
def _expand_help(self, action):
+ help_string = self._get_help_string(action)
+ if '%' not in help_string:
+ return help_string
params = dict(vars(action), prog=self._prog)
for name in list(params):
- if params[name] is SUPPRESS:
+ value = params[name]
+ if value is SUPPRESS:
del params[name]
- for name in list(params):
- if hasattr(params[name], '__name__'):
- params[name] = params[name].__name__
+ elif hasattr(value, '__name__'):
+ params[name] = value.__name__
if params.get('choices') is not None:
params['choices'] = ', '.join(map(str, params['choices']))
- return self._get_help_string(action) % params
+ return help_string % params
def _iter_indented_subactions(self, action):
try:
@@ -703,23 +739,18 @@ class ArgumentDefaultsHelpFormatter(HelpFormatter):
"""
def _get_help_string(self, action):
- """
- Add the default value to the option help message.
-
- ArgumentDefaultsHelpFormatter and BooleanOptionalAction when it isn't
- already present. This code will do that, detecting cornercases to
- prevent duplicates or cases where it wouldn't make sense to the end
- user.
- """
help = action.help
if help is None:
help = ''
- if '%(default)' not in help:
- if action.default is not SUPPRESS:
- defaulting_nargs = [OPTIONAL, ZERO_OR_MORE]
- if action.option_strings or action.nargs in defaulting_nargs:
- help += _(' (default: %(default)s)')
+ if (
+ '%(default)' not in help
+ and action.default is not SUPPRESS
+ and not action.required
+ ):
+ defaulting_nargs = (OPTIONAL, ZERO_OR_MORE)
+ if action.option_strings or action.nargs in defaulting_nargs:
+ help += _(' (default: %(default)s)')
return help
@@ -856,7 +887,8 @@ def __init__(self,
choices=None,
required=False,
help=None,
- metavar=None):
+ metavar=None,
+ deprecated=False):
self.option_strings = option_strings
self.dest = dest
self.nargs = nargs
@@ -867,6 +899,7 @@ def __init__(self,
self.required = required
self.help = help
self.metavar = metavar
+ self.deprecated = deprecated
def _get_kwargs(self):
names = [
@@ -880,6 +913,7 @@ def _get_kwargs(self):
'required',
'help',
'metavar',
+ 'deprecated',
]
return [(name, getattr(self, name)) for name in names]
@@ -887,59 +921,37 @@ def format_usage(self):
return self.option_strings[0]
def __call__(self, parser, namespace, values, option_string=None):
- raise NotImplementedError(_('.__call__() not defined'))
-
+ raise NotImplementedError('.__call__() not defined')
-# FIXME: remove together with `BooleanOptionalAction` deprecated arguments.
-_deprecated_default = object()
class BooleanOptionalAction(Action):
def __init__(self,
option_strings,
dest,
default=None,
- type=_deprecated_default,
- choices=_deprecated_default,
required=False,
help=None,
- metavar=_deprecated_default):
+ deprecated=False):
_option_strings = []
for option_string in option_strings:
_option_strings.append(option_string)
if option_string.startswith('--'):
+ if option_string.startswith('--no-'):
+ raise ValueError(f'invalid option name {option_string!r} '
+ f'for BooleanOptionalAction')
option_string = '--no-' + option_string[2:]
_option_strings.append(option_string)
- # We need `_deprecated` special value to ban explicit arguments that
- # match default value. Like:
- # parser.add_argument('-f', action=BooleanOptionalAction, type=int)
- for field_name in ('type', 'choices', 'metavar'):
- if locals()[field_name] is not _deprecated_default:
- warnings._deprecated(
- field_name,
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.",
- remove=(3, 14))
-
- if type is _deprecated_default:
- type = None
- if choices is _deprecated_default:
- choices = None
- if metavar is _deprecated_default:
- metavar = None
-
super().__init__(
option_strings=_option_strings,
dest=dest,
nargs=0,
default=default,
- type=type,
- choices=choices,
required=required,
help=help,
- metavar=metavar)
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
@@ -962,7 +974,8 @@ def __init__(self,
choices=None,
required=False,
help=None,
- metavar=None):
+ metavar=None,
+ deprecated=False):
if nargs == 0:
raise ValueError('nargs for store actions must be != 0; if you '
'have nothing to store, actions such as store '
@@ -979,7 +992,8 @@ def __init__(self,
choices=choices,
required=required,
help=help,
- metavar=metavar)
+ metavar=metavar,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values)
@@ -994,7 +1008,8 @@ def __init__(self,
default=None,
required=False,
help=None,
- metavar=None):
+ metavar=None,
+ deprecated=False):
super(_StoreConstAction, self).__init__(
option_strings=option_strings,
dest=dest,
@@ -1002,7 +1017,8 @@ def __init__(self,
const=const,
default=default,
required=required,
- help=help)
+ help=help,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, self.const)
@@ -1015,14 +1031,16 @@ def __init__(self,
dest,
default=False,
required=False,
- help=None):
+ help=None,
+ deprecated=False):
super(_StoreTrueAction, self).__init__(
option_strings=option_strings,
dest=dest,
const=True,
- default=default,
+ deprecated=deprecated,
required=required,
- help=help)
+ help=help,
+ default=default)
class _StoreFalseAction(_StoreConstAction):
@@ -1032,14 +1050,16 @@ def __init__(self,
dest,
default=True,
required=False,
- help=None):
+ help=None,
+ deprecated=False):
super(_StoreFalseAction, self).__init__(
option_strings=option_strings,
dest=dest,
const=False,
default=default,
required=required,
- help=help)
+ help=help,
+ deprecated=deprecated)
class _AppendAction(Action):
@@ -1054,7 +1074,8 @@ def __init__(self,
choices=None,
required=False,
help=None,
- metavar=None):
+ metavar=None,
+ deprecated=False):
if nargs == 0:
raise ValueError('nargs for append actions must be != 0; if arg '
'strings are not supplying the value to append, '
@@ -1071,7 +1092,8 @@ def __init__(self,
choices=choices,
required=required,
help=help,
- metavar=metavar)
+ metavar=metavar,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None)
@@ -1089,7 +1111,8 @@ def __init__(self,
default=None,
required=False,
help=None,
- metavar=None):
+ metavar=None,
+ deprecated=False):
super(_AppendConstAction, self).__init__(
option_strings=option_strings,
dest=dest,
@@ -1098,7 +1121,8 @@ def __init__(self,
default=default,
required=required,
help=help,
- metavar=metavar)
+ metavar=metavar,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
items = getattr(namespace, self.dest, None)
@@ -1114,14 +1138,16 @@ def __init__(self,
dest,
default=None,
required=False,
- help=None):
+ help=None,
+ deprecated=False):
super(_CountAction, self).__init__(
option_strings=option_strings,
dest=dest,
nargs=0,
default=default,
required=required,
- help=help)
+ help=help,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
count = getattr(namespace, self.dest, None)
@@ -1136,13 +1162,15 @@ def __init__(self,
option_strings,
dest=SUPPRESS,
default=SUPPRESS,
- help=None):
+ help=None,
+ deprecated=False):
super(_HelpAction, self).__init__(
option_strings=option_strings,
dest=dest,
default=default,
nargs=0,
- help=help)
+ help=help,
+ deprecated=deprecated)
def __call__(self, parser, namespace, values, option_string=None):
parser.print_help()
@@ -1156,7 +1184,8 @@ def __init__(self,
version=None,
dest=SUPPRESS,
default=SUPPRESS,
- help=None):
+ help=None,
+ deprecated=False):
if help is None:
help = _("show program's version number and exit")
super(_VersionAction, self).__init__(
@@ -1202,6 +1231,8 @@ def __init__(self,
self._parser_class = parser_class
self._name_parser_map = {}
self._choices_actions = []
+ self._deprecated = set()
+ self._color = True
super(_SubParsersAction, self).__init__(
option_strings=option_strings,
@@ -1212,34 +1243,45 @@ def __init__(self,
help=help,
metavar=metavar)
- def add_parser(self, name, **kwargs):
+ def add_parser(self, name, *, deprecated=False, **kwargs):
# set prog from the existing prefix
if kwargs.get('prog') is None:
kwargs['prog'] = '%s %s' % (self._prog_prefix, name)
+ # set color
+ if kwargs.get('color') is None:
+ kwargs['color'] = self._color
+
aliases = kwargs.pop('aliases', ())
if name in self._name_parser_map:
- raise ArgumentError(self, _('conflicting subparser: %s') % name)
+ raise ValueError(f'conflicting subparser: {name}')
for alias in aliases:
if alias in self._name_parser_map:
- raise ArgumentError(
- self, _('conflicting subparser alias: %s') % alias)
+ raise ValueError(f'conflicting subparser alias: {alias}')
# create a pseudo-action to hold the choice help
if 'help' in kwargs:
help = kwargs.pop('help')
choice_action = self._ChoicesPseudoAction(name, aliases, help)
self._choices_actions.append(choice_action)
+ else:
+ choice_action = None
# create the parser and add it to the map
parser = self._parser_class(**kwargs)
+ if choice_action is not None:
+ parser._check_help(choice_action)
self._name_parser_map[name] = parser
# make parser available under aliases also
for alias in aliases:
self._name_parser_map[alias] = parser
+ if deprecated:
+ self._deprecated.add(name)
+ self._deprecated.update(aliases)
+
return parser
def _get_subactions(self):
@@ -1255,13 +1297,17 @@ def __call__(self, parser, namespace, values, option_string=None):
# select the parser
try:
- parser = self._name_parser_map[parser_name]
+ subparser = self._name_parser_map[parser_name]
except KeyError:
args = {'parser_name': parser_name,
'choices': ', '.join(self._name_parser_map)}
msg = _('unknown parser %(parser_name)r (choices: %(choices)s)') % args
raise ArgumentError(self, msg)
+ if parser_name in self._deprecated:
+ parser._warning(_("command '%(parser_name)s' is deprecated") %
+ {'parser_name': parser_name})
+
# parse all the remaining options into the namespace
# store any unrecognized options on the object, so that the top
# level parser can decide what to do with them
@@ -1269,7 +1315,7 @@ def __call__(self, parser, namespace, values, option_string=None):
# In case this subparser defines new defaults, we parse them
# in a new namespace object and then update the original
# namespace for the relevant parts.
- subnamespace, arg_strings = parser.parse_known_args(arg_strings, None)
+ subnamespace, arg_strings = subparser.parse_known_args(arg_strings, None)
for key, value in vars(subnamespace).items():
setattr(namespace, key, value)
@@ -1290,7 +1336,7 @@ def __call__(self, parser, namespace, values, option_string=None):
# ==============
class FileType(object):
- """Factory for creating file object types
+ """Deprecated factory for creating file object types
Instances of FileType are typically passed as type= arguments to the
ArgumentParser add_argument() method.
@@ -1307,6 +1353,12 @@ class FileType(object):
"""
def __init__(self, mode='r', bufsize=-1, encoding=None, errors=None):
+ import warnings
+ warnings.warn(
+ "FileType is deprecated. Simply open files after parsing arguments.",
+ category=PendingDeprecationWarning,
+ stacklevel=2
+ )
self._mode = mode
self._bufsize = bufsize
self._encoding = encoding
@@ -1410,7 +1462,7 @@ def __init__(self,
self._defaults = {}
# determines whether an "option" looks like a negative number
- self._negative_number_matcher = _re.compile(r'^-\d+$|^-\d*\.\d+$')
+ self._negative_number_matcher = _re.compile(r'-\.?\d')
# whether or not there are any optionals that look like negative
# numbers -- uses a list so it can be shared and edited
@@ -1419,6 +1471,7 @@ def __init__(self,
# ====================
# Registration methods
# ====================
+
def register(self, registry_name, value, object):
registry = self._registries.setdefault(registry_name, {})
registry[value] = object
@@ -1429,6 +1482,7 @@ def _registry_get(self, registry_name, value, default=None):
# ==================================
# Namespace default accessor methods
# ==================================
+
def set_defaults(self, **kwargs):
self._defaults.update(kwargs)
@@ -1448,6 +1502,7 @@ def get_default(self, dest):
# =======================
# Adding argument actions
# =======================
+
def add_argument(self, *args, **kwargs):
"""
add_argument(dest, ..., name=value, ...)
@@ -1460,7 +1515,8 @@ def add_argument(self, *args, **kwargs):
chars = self.prefix_chars
if not args or len(args) == 1 and args[0][0] not in chars:
if args and 'dest' in kwargs:
- raise ValueError('dest supplied twice for positional argument')
+ raise TypeError('dest supplied twice for positional argument,'
+ ' did you mean metavar?')
kwargs = self._get_positional_kwargs(*args, **kwargs)
# otherwise, we're adding an optional argument
@@ -1476,27 +1532,34 @@ def add_argument(self, *args, **kwargs):
kwargs['default'] = self.argument_default
# create the action object, and add it to the parser
+ action_name = kwargs.get('action')
action_class = self._pop_action_class(kwargs)
if not callable(action_class):
- raise ValueError('unknown action "%s"' % (action_class,))
+ raise ValueError(f'unknown action {action_class!r}')
action = action_class(**kwargs)
+ # raise an error if action for positional argument does not
+ # consume arguments
+ if not action.option_strings and action.nargs == 0:
+ raise ValueError(f'action {action_name!r} is not valid for positional arguments')
+
# raise an error if the action type is not callable
type_func = self._registry_get('type', action.type, action.type)
if not callable(type_func):
- raise ValueError('%r is not callable' % (type_func,))
+ raise TypeError(f'{type_func!r} is not callable')
if type_func is FileType:
- raise ValueError('%r is a FileType class object, instance of it'
- ' must be passed' % (type_func,))
+ raise TypeError(f'{type_func!r} is a FileType class object, '
+ f'instance of it must be passed')
# raise an error if the metavar does not match the type
- if hasattr(self, "_get_formatter"):
+ if hasattr(self, "_get_validation_formatter"):
+ formatter = self._get_validation_formatter()
try:
- self._get_formatter()._format_args(action, None)
+ formatter._format_args(action, None)
except TypeError:
raise ValueError("length of metavar tuple does not match nargs")
-
+ self._check_help(action)
return self._add_action(action)
def add_argument_group(self, *args, **kwargs):
@@ -1538,8 +1601,10 @@ def _add_container_actions(self, container):
title_group_map = {}
for group in self._action_groups:
if group.title in title_group_map:
- msg = _('cannot merge actions - two groups are named %r')
- raise ValueError(msg % (group.title))
+ # This branch could happen if a derived class added
+ # groups with duplicated titles in __init__
+ msg = f'cannot merge actions - two groups are named {group.title!r}'
+ raise ValueError(msg)
title_group_map[group.title] = group
# map each action to its group
@@ -1580,13 +1645,15 @@ def _add_container_actions(self, container):
def _get_positional_kwargs(self, dest, **kwargs):
# make sure required is not specified
if 'required' in kwargs:
- msg = _("'required' is an invalid argument for positionals")
+ msg = "'required' is an invalid argument for positionals"
raise TypeError(msg)
# mark positional arguments as required if at least one is
# always required
nargs = kwargs.get('nargs')
- if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS, 0]:
+ if nargs == 0:
+ raise ValueError('nargs for positionals must be != 0')
+ if nargs not in [OPTIONAL, ZERO_OR_MORE, REMAINDER, SUPPRESS]:
kwargs['required'] = True
# return the keyword arguments with no option strings
@@ -1599,11 +1666,9 @@ def _get_optional_kwargs(self, *args, **kwargs):
for option_string in args:
# error on strings that don't start with an appropriate prefix
if not option_string[0] in self.prefix_chars:
- args = {'option': option_string,
- 'prefix_chars': self.prefix_chars}
- msg = _('invalid option string %(option)r: '
- 'must start with a character %(prefix_chars)r')
- raise ValueError(msg % args)
+ raise ValueError(
+ f'invalid option string {option_string!r}: '
+ f'must start with a character {self.prefix_chars!r}')
# strings starting with two prefix characters are long options
option_strings.append(option_string)
@@ -1619,8 +1684,8 @@ def _get_optional_kwargs(self, *args, **kwargs):
dest_option_string = option_strings[0]
dest = dest_option_string.lstrip(self.prefix_chars)
if not dest:
- msg = _('dest= is required for options like %r')
- raise ValueError(msg % option_string)
+ msg = f'dest= is required for options like {option_string!r}'
+ raise TypeError(msg)
dest = dest.replace('-', '_')
# return the updated keyword arguments
@@ -1636,8 +1701,8 @@ def _get_handler(self):
try:
return getattr(self, handler_func_name)
except AttributeError:
- msg = _('invalid conflict_resolution value: %r')
- raise ValueError(msg % self.conflict_handler)
+ msg = f'invalid conflict_resolution value: {self.conflict_handler!r}'
+ raise ValueError(msg)
def _check_conflict(self, action):
@@ -1676,10 +1741,26 @@ def _handle_conflict_resolve(self, action, conflicting_actions):
if not action.option_strings:
action.container._remove_action(action)
+ def _check_help(self, action):
+ if action.help and hasattr(self, "_get_validation_formatter"):
+ formatter = self._get_validation_formatter()
+ try:
+ formatter._expand_help(action)
+ except (ValueError, TypeError, KeyError) as exc:
+ raise ValueError('badly formed help string') from exc
+
class _ArgumentGroup(_ActionsContainer):
def __init__(self, container, title=None, description=None, **kwargs):
+ if 'prefix_chars' in kwargs:
+ import warnings
+ depr_msg = (
+ "The use of the undocumented 'prefix_chars' parameter in "
+ "ArgumentParser.add_argument_group() is deprecated."
+ )
+ warnings.warn(depr_msg, DeprecationWarning, stacklevel=3)
+
# add any missing keyword arguments by checking the container
update = kwargs.setdefault
update('conflict_handler', container.conflict_handler)
@@ -1711,13 +1792,7 @@ def _remove_action(self, action):
self._group_actions.remove(action)
def add_argument_group(self, *args, **kwargs):
- warnings.warn(
- "Nesting argument groups is deprecated.",
- category=DeprecationWarning,
- stacklevel=2
- )
- return super().add_argument_group(*args, **kwargs)
-
+ raise ValueError('argument groups cannot be nested')
class _MutuallyExclusiveGroup(_ArgumentGroup):
@@ -1728,7 +1803,7 @@ def __init__(self, container, required=False):
def _add_action(self, action):
if action.required:
- msg = _('mutually exclusive arguments must be optional')
+ msg = 'mutually exclusive arguments must be optional'
raise ValueError(msg)
action = self._container._add_action(action)
self._group_actions.append(action)
@@ -1738,13 +1813,29 @@ def _remove_action(self, action):
self._container._remove_action(action)
self._group_actions.remove(action)
- def add_mutually_exclusive_group(self, *args, **kwargs):
- warnings.warn(
- "Nesting mutually exclusive groups is deprecated.",
- category=DeprecationWarning,
- stacklevel=2
- )
- return super().add_mutually_exclusive_group(*args, **kwargs)
+ def add_mutually_exclusive_group(self, **kwargs):
+ raise ValueError('mutually exclusive groups cannot be nested')
+
+def _prog_name(prog=None):
+ if prog is not None:
+ return prog
+ arg0 = _sys.argv[0]
+ try:
+ modspec = _sys.modules['__main__'].__spec__
+ except (KeyError, AttributeError):
+ # possibly PYTHONSTARTUP or -X presite or other weird edge case
+ # no good answer here, so fall back to the default
+ modspec = None
+ if modspec is None:
+ # simple script
+ return _os.path.basename(arg0)
+ py = _os.path.basename(_sys.executable)
+ if modspec.name != '__main__':
+ # imported module or package
+ modname = modspec.name.removesuffix('.__main__')
+ return f'{py} -m {modname}'
+ # directory or ZIP file
+ return f'{py} {arg0}'
class ArgumentParser(_AttributeHolder, _ActionsContainer):
@@ -1767,6 +1858,9 @@ class ArgumentParser(_AttributeHolder, _ActionsContainer):
- allow_abbrev -- Allow long options to be abbreviated unambiguously
- exit_on_error -- Determines whether or not ArgumentParser exits with
error info when an error occurs
+ - suggest_on_error - Enables suggestions for mistyped argument choices
+ and subparser names (default: ``False``)
+ - color - Allow color output in help messages (default: ``False``)
"""
def __init__(self,
@@ -1782,19 +1876,18 @@ def __init__(self,
conflict_handler='error',
add_help=True,
allow_abbrev=True,
- exit_on_error=True):
-
+ exit_on_error=True,
+ *,
+ suggest_on_error=False,
+ color=True,
+ ):
superinit = super(ArgumentParser, self).__init__
superinit(description=description,
prefix_chars=prefix_chars,
argument_default=argument_default,
conflict_handler=conflict_handler)
- # default setting for prog
- if prog is None:
- prog = _os.path.basename(_sys.argv[0])
-
- self.prog = prog
+ self.prog = _prog_name(prog)
self.usage = usage
self.epilog = epilog
self.formatter_class = formatter_class
@@ -1802,6 +1895,11 @@ def __init__(self,
self.add_help = add_help
self.allow_abbrev = allow_abbrev
self.exit_on_error = exit_on_error
+ self.suggest_on_error = suggest_on_error
+ self.color = color
+
+ # Cached formatter for validation (avoids repeated _set_color calls)
+ self._cached_formatter = None
add_group = self.add_argument_group
self._positionals = add_group(_('positional arguments'))
@@ -1809,9 +1907,7 @@ def __init__(self,
self._subparsers = None
# register types
- def identity(string):
- return string
- self.register('type', None, identity)
+ self.register('type', None, _identity)
# add help argument if necessary
# (using explicit default to override global argument_default)
@@ -1824,17 +1920,16 @@ def identity(string):
# add parent arguments and defaults
for parent in parents:
+ if not isinstance(parent, ArgumentParser):
+ raise TypeError('parents must be a list of ArgumentParser')
self._add_container_actions(parent)
- try:
- defaults = parent._defaults
- except AttributeError:
- pass
- else:
- self._defaults.update(defaults)
+ defaults = parent._defaults
+ self._defaults.update(defaults)
# =======================
# Pretty __repr__ methods
# =======================
+
def _get_kwargs(self):
names = [
'prog',
@@ -1849,9 +1944,10 @@ def _get_kwargs(self):
# ==================================
# Optional/Positional adding methods
# ==================================
+
def add_subparsers(self, **kwargs):
if self._subparsers is not None:
- raise ArgumentError(None, _('cannot have multiple subparser arguments'))
+ raise ValueError('cannot have multiple subparser arguments')
# add the parser class to the arguments if it's not present
kwargs.setdefault('parser_class', type(self))
@@ -1866,15 +1962,19 @@ def add_subparsers(self, **kwargs):
# prog defaults to the usage message of this parser, skipping
# optional arguments and with no "usage:" prefix
if kwargs.get('prog') is None:
- formatter = self._get_formatter()
+ # Create formatter without color to avoid storing ANSI codes in prog
+ formatter = self.formatter_class(prog=self.prog)
+ formatter._set_color(False)
positionals = self._get_positional_actions()
groups = self._mutually_exclusive_groups
- formatter.add_usage(self.usage, positionals, groups, '')
+ formatter.add_usage(None, positionals, groups, '')
kwargs['prog'] = formatter.format_help().strip()
# create the parsers action and add it to the positionals list
parsers_class = self._pop_action_class(kwargs, 'parsers')
action = parsers_class(option_strings=[], **kwargs)
+ action._color = self.color
+ self._check_help(action)
self._subparsers._add_action(action)
# return the created parsers action
@@ -1900,6 +2000,7 @@ def _get_positional_actions(self):
# =====================================
# Command line argument parsing methods
# =====================================
+
def parse_args(self, args=None, namespace=None):
args, argv = self.parse_known_args(args, namespace)
if argv:
@@ -1997,6 +2098,7 @@ def _parse_known_args(self, arg_strings, namespace, intermixed):
# converts arg strings to the appropriate and then takes the action
seen_actions = set()
seen_non_default_actions = set()
+ warned = set()
def take_action(action, argument_strings, option_string=None):
seen_actions.add(action)
@@ -2110,6 +2212,10 @@ def consume_optional(start_index):
# the Optional's string args stopped
assert action_tuples
for action, args, option_string in action_tuples:
+ if action.deprecated and option_string not in warned:
+ self._warning(_("option '%(option)s' is deprecated") %
+ {'option': option_string})
+ warned.add(option_string)
take_action(action, args, option_string)
return stop
@@ -2138,6 +2244,10 @@ def consume_positionals(start_index):
start_index + arg_count) >= 0):
args.remove('--')
start_index += arg_count
+ if args and action.deprecated and action.dest not in warned:
+ self._warning(_("argument '%(argument_name)s' is deprecated") %
+ {'argument_name': action.dest})
+ warned.add(action.dest)
take_action(action, args)
# slice off the Positionals that we just parsed and return the
@@ -2157,10 +2267,11 @@ def consume_positionals(start_index):
while start_index <= max_option_string_index:
# consume any Positionals preceding the next option
- next_option_string_index = min([
- index
- for index in option_string_indices
- if index >= start_index])
+ next_option_string_index = start_index
+ while next_option_string_index <= max_option_string_index:
+ if next_option_string_index in option_string_indices:
+ break
+ next_option_string_index += 1
if not intermixed and start_index != next_option_string_index:
positionals_end_index = consume_positionals(start_index)
@@ -2484,6 +2595,7 @@ def parse_known_intermixed_args(self, args=None, namespace=None):
# ========================
# Value conversion methods
# ========================
+
def _get_values(self, action, arg_strings):
# optional argument produces a default when not present
if not arg_strings and action.nargs == OPTIONAL:
@@ -2493,7 +2605,6 @@ def _get_values(self, action, arg_strings):
value = action.default
if isinstance(value, str) and value is not SUPPRESS:
value = self._get_value(action, value)
- self._check_value(action, value)
# when nargs='*' on a positional, if there were no command-line
# args, use the default if it is anything other than None
@@ -2501,11 +2612,8 @@ def _get_values(self, action, arg_strings):
not action.option_strings):
if action.default is not None:
value = action.default
- self._check_value(action, value)
else:
- # since arg_strings is always [] at this point
- # there is no need to use self._check_value(action, value)
- value = arg_strings
+ value = []
# single argument or optional argument produces a single value
elif len(arg_strings) == 1 and action.nargs in [None, OPTIONAL]:
@@ -2538,8 +2646,7 @@ def _get_values(self, action, arg_strings):
def _get_value(self, action, arg_string):
type_func = self._registry_get('type', action.type, action.type)
if not callable(type_func):
- msg = _('%r is not callable')
- raise ArgumentError(action, msg % type_func)
+ raise TypeError(f'{type_func!r} is not callable')
# convert the value to the appropriate type
try:
@@ -2563,18 +2670,32 @@ def _get_value(self, action, arg_string):
def _check_value(self, action, value):
# converted value must be one of the choices (if specified)
choices = action.choices
- if choices is not None:
- if isinstance(choices, str):
- choices = iter(choices)
- if value not in choices:
- args = {'value': str(value),
- 'choices': ', '.join(map(str, action.choices))}
- msg = _('invalid choice: %(value)r (choose from %(choices)s)')
- raise ArgumentError(action, msg % args)
+ if choices is None:
+ return
+
+ if isinstance(choices, str):
+ choices = iter(choices)
+
+ if value not in choices:
+ args = {'value': str(value),
+ 'choices': ', '.join(repr(str(choice)) for choice in action.choices)}
+ msg = _('invalid choice: %(value)r (choose from %(choices)s)')
+
+ if self.suggest_on_error and isinstance(value, str):
+ if all(isinstance(choice, str) for choice in action.choices):
+ import difflib
+ suggestions = difflib.get_close_matches(value, action.choices, 1)
+ if suggestions:
+ args['closest'] = suggestions[0]
+ msg = _('invalid choice: %(value)r, maybe you meant %(closest)r? '
+ '(choose from %(choices)s)')
+
+ raise ArgumentError(action, msg % args)
# =======================
# Help-formatting methods
# =======================
+
def format_usage(self):
formatter = self._get_formatter()
formatter.add_usage(self.usage, self._actions,
@@ -2605,11 +2726,21 @@ def format_help(self):
return formatter.format_help()
def _get_formatter(self):
- return self.formatter_class(prog=self.prog)
+ formatter = self.formatter_class(prog=self.prog)
+ formatter._set_color(self.color)
+ return formatter
+
+ def _get_validation_formatter(self):
+ # Return cached formatter for read-only validation operations
+ # (_expand_help and _format_args). Avoids repeated slow _set_color calls.
+ if self._cached_formatter is None:
+ self._cached_formatter = self._get_formatter()
+ return self._cached_formatter
# =====================
# Help-printing methods
# =====================
+
def print_usage(self, file=None):
if file is None:
file = _sys.stdout
@@ -2631,6 +2762,7 @@ def _print_message(self, message, file=None):
# ===============
# Exiting methods
# ===============
+
def exit(self, status=0, message=None):
if message:
self._print_message(message, _sys.stderr)
@@ -2648,3 +2780,7 @@ def error(self, message):
self.print_usage(_sys.stderr)
args = {'prog': self.prog, 'message': message}
self.exit(2, _('%(prog)s: error: %(message)s\n') % args)
+
+ def _warning(self, message):
+ args = {'prog': self.prog, 'message': message}
+ self._print_message(_('%(prog)s: warning: %(message)s\n') % args, _sys.stderr)
diff --git a/PythonLib/full/ast.py b/PythonLib/full/ast.py
index b0995fa7f..2f11683ec 100644
--- a/PythonLib/full/ast.py
+++ b/PythonLib/full/ast.py
@@ -1,44 +1,38 @@
"""
- ast
- ~~~
-
- The `ast` module helps Python applications to process trees of the Python
- abstract syntax grammar. The abstract syntax itself might change with
- each Python release; this module helps to find out programmatically what
- the current grammar looks like and allows modifications of it.
-
- An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
- a flag to the `compile()` builtin function or by using the `parse()`
- function from this module. The result will be a tree of objects whose
- classes all inherit from `ast.AST`.
-
- A modified abstract syntax tree can be compiled into a Python code object
- using the built-in `compile()` function.
-
- Additionally various helper functions are provided that make working with
- the trees simpler. The main intention of the helper functions and this
- module in general is to provide an easy to use interface for libraries
- that work tightly with the python syntax (template engines for example).
-
-
- :copyright: Copyright 2008 by Armin Ronacher.
- :license: Python License.
+The `ast` module helps Python applications to process trees of the Python
+abstract syntax grammar. The abstract syntax itself might change with
+each Python release; this module helps to find out programmatically what
+the current grammar looks like and allows modifications of it.
+
+An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as
+a flag to the `compile()` builtin function or by using the `parse()`
+function from this module. The result will be a tree of objects whose
+classes all inherit from `ast.AST`.
+
+A modified abstract syntax tree can be compiled into a Python code object
+using the built-in `compile()` function.
+
+Additionally various helper functions are provided that make working with
+the trees simpler. The main intention of the helper functions and this
+module in general is to provide an easy to use interface for libraries
+that work tightly with the python syntax (template engines for example).
+
+:copyright: Copyright 2008 by Armin Ronacher.
+:license: Python License.
"""
-import sys
-import re
from _ast import *
-from contextlib import contextmanager, nullcontext
-from enum import IntEnum, auto, _simple_enum
def parse(source, filename='', mode='exec', *,
- type_comments=False, feature_version=None):
+ type_comments=False, feature_version=None, optimize=-1):
"""
Parse the source into an AST node.
Equivalent to compile(source, filename, mode, PyCF_ONLY_AST).
Pass type_comments=True to get back type comments where the syntax allows.
"""
flags = PyCF_ONLY_AST
+ if optimize > 0:
+ flags |= PyCF_OPTIMIZED_AST
if type_comments:
flags |= PyCF_TYPE_COMMENTS
if feature_version is None:
@@ -50,7 +44,7 @@ def parse(source, filename='', mode='exec', *,
feature_version = minor
# Else it should be an int giving the minor version for 3.x.
return compile(source, filename, mode, flags,
- _feature_version=feature_version)
+ _feature_version=feature_version, optimize=optimize)
def literal_eval(node_or_string):
@@ -112,7 +106,11 @@ def _convert(node):
return _convert(node_or_string)
-def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
+def dump(
+ node, annotate_fields=True, include_attributes=False,
+ *,
+ indent=None, show_empty=False,
+):
"""
Return a formatted dump of the tree in node. This is mainly useful for
debugging purposes. If annotate_fields is true (by default),
@@ -123,6 +121,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None):
include_attributes can be set to true. If indent is a non-negative
integer or string, then the tree will be pretty-printed with that indent
level. None (the default) selects the single line representation.
+ If show_empty is False, then empty lists and fields that are None
+ will be omitted from the output for better readability.
"""
def _format(node, level=0):
if indent is not None:
@@ -135,6 +135,7 @@ def _format(node, level=0):
if isinstance(node, AST):
cls = type(node)
args = []
+ args_buffer = []
allsimple = True
keywords = annotate_fields
for name in node._fields:
@@ -146,6 +147,16 @@ def _format(node, level=0):
if value is None and getattr(cls, name, ...) is None:
keywords = True
continue
+ if not show_empty:
+ if value == []:
+ field_type = cls._field_types.get(name, object)
+ if getattr(field_type, '__origin__', ...) is list:
+ if not keywords:
+ args_buffer.append(repr(value))
+ continue
+ if not keywords:
+ args.extend(args_buffer)
+ args_buffer = []
value, simple = _format(value, level)
allsimple = allsimple and simple
if keywords:
@@ -304,12 +315,18 @@ def get_docstring(node, clean=True):
return text
-_line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))")
+_line_pattern = None
def _splitlines_no_ff(source, maxlines=None):
"""Split a string into lines ignoring form feed and other chars.
This mimics how the Python parser splits source code.
"""
+ global _line_pattern
+ if _line_pattern is None:
+ # lazily computed to speedup import time of `ast`
+ import re
+ _line_pattern = re.compile(r"(.*?(?:\r\n|\n|\r|$))")
+
lines = []
for lineno, match in enumerate(_line_pattern.finditer(source), 1):
if maxlines is not None and lineno > maxlines:
@@ -380,6 +397,88 @@ def walk(node):
yield node
+def compare(
+ a,
+ b,
+ /,
+ *,
+ compare_attributes=False,
+):
+ """Recursively compares two ASTs.
+
+ compare_attributes affects whether AST attributes are considered
+ in the comparison. If compare_attributes is False (default), then
+ attributes are ignored. Otherwise they must all be equal. This
+ option is useful to check whether the ASTs are structurally equal but
+ might differ in whitespace or similar details.
+ """
+
+ sentinel = object() # handle the possibility of a missing attribute/field
+
+ def _compare(a, b):
+ # Compare two fields on an AST object, which may themselves be
+ # AST objects, lists of AST objects, or primitive ASDL types
+ # like identifiers and constants.
+ if isinstance(a, AST):
+ return compare(
+ a,
+ b,
+ compare_attributes=compare_attributes,
+ )
+ elif isinstance(a, list):
+ # If a field is repeated, then both objects will represent
+ # the value as a list.
+ if len(a) != len(b):
+ return False
+ for a_item, b_item in zip(a, b):
+ if not _compare(a_item, b_item):
+ return False
+ else:
+ return True
+ else:
+ return type(a) is type(b) and a == b
+
+ def _compare_fields(a, b):
+ if a._fields != b._fields:
+ return False
+ for field in a._fields:
+ a_field = getattr(a, field, sentinel)
+ b_field = getattr(b, field, sentinel)
+ if a_field is sentinel and b_field is sentinel:
+ # both nodes are missing a field at runtime
+ continue
+ if a_field is sentinel or b_field is sentinel:
+ # one of the node is missing a field
+ return False
+ if not _compare(a_field, b_field):
+ return False
+ else:
+ return True
+
+ def _compare_attributes(a, b):
+ if a._attributes != b._attributes:
+ return False
+ # Attributes are always ints.
+ for attr in a._attributes:
+ a_attr = getattr(a, attr, sentinel)
+ b_attr = getattr(b, attr, sentinel)
+ if a_attr is sentinel and b_attr is sentinel:
+ # both nodes are missing an attribute at runtime
+ continue
+ if a_attr != b_attr:
+ return False
+ else:
+ return True
+
+ if type(a) is not type(b):
+ return False
+ if not _compare_fields(a, b):
+ return False
+ if compare_attributes and not _compare_attributes(a, b):
+ return False
+ return True
+
+
class NodeVisitor(object):
"""
A node visitor base class that walks the abstract syntax tree and calls a
@@ -416,27 +515,6 @@ def generic_visit(self, node):
elif isinstance(value, AST):
self.visit(value)
- def visit_Constant(self, node):
- value = node.value
- type_name = _const_node_type_names.get(type(value))
- if type_name is None:
- for cls, name in _const_node_type_names.items():
- if isinstance(value, cls):
- type_name = name
- break
- if type_name is not None:
- method = 'visit_' + type_name
- try:
- visitor = getattr(self, method)
- except AttributeError:
- pass
- else:
- import warnings
- warnings.warn(f"{method} is deprecated; add visit_Constant",
- DeprecationWarning, 2)
- return visitor(node)
- return self.generic_visit(node)
-
class NodeTransformer(NodeVisitor):
"""
@@ -496,151 +574,6 @@ def generic_visit(self, node):
setattr(node, field, new_node)
return node
-
-_DEPRECATED_VALUE_ALIAS_MESSAGE = (
- "{name} is deprecated and will be removed in Python {remove}; use value instead"
-)
-_DEPRECATED_CLASS_MESSAGE = (
- "{name} is deprecated and will be removed in Python {remove}; "
- "use ast.Constant instead"
-)
-
-
-# If the ast module is loaded more than once, only add deprecated methods once
-if not hasattr(Constant, 'n'):
- # The following code is for backward compatibility.
- # It will be removed in future.
-
- def _n_getter(self):
- """Deprecated. Use value instead."""
- import warnings
- warnings._deprecated(
- "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
- )
- return self.value
-
- def _n_setter(self, value):
- import warnings
- warnings._deprecated(
- "Attribute n", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
- )
- self.value = value
-
- def _s_getter(self):
- """Deprecated. Use value instead."""
- import warnings
- warnings._deprecated(
- "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
- )
- return self.value
-
- def _s_setter(self, value):
- import warnings
- warnings._deprecated(
- "Attribute s", message=_DEPRECATED_VALUE_ALIAS_MESSAGE, remove=(3, 14)
- )
- self.value = value
-
- Constant.n = property(_n_getter, _n_setter)
- Constant.s = property(_s_getter, _s_setter)
-
-class _ABC(type):
-
- def __init__(cls, *args):
- cls.__doc__ = """Deprecated AST node class. Use ast.Constant instead"""
-
- def __instancecheck__(cls, inst):
- if cls in _const_types:
- import warnings
- warnings._deprecated(
- f"ast.{cls.__qualname__}",
- message=_DEPRECATED_CLASS_MESSAGE,
- remove=(3, 14)
- )
- if not isinstance(inst, Constant):
- return False
- if cls in _const_types:
- try:
- value = inst.value
- except AttributeError:
- return False
- else:
- return (
- isinstance(value, _const_types[cls]) and
- not isinstance(value, _const_types_not.get(cls, ()))
- )
- return type.__instancecheck__(cls, inst)
-
-def _new(cls, *args, **kwargs):
- for key in kwargs:
- if key not in cls._fields:
- # arbitrary keyword arguments are accepted
- continue
- pos = cls._fields.index(key)
- if pos < len(args):
- raise TypeError(f"{cls.__name__} got multiple values for argument {key!r}")
- if cls in _const_types:
- import warnings
- warnings._deprecated(
- f"ast.{cls.__qualname__}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
- )
- return Constant(*args, **kwargs)
- return Constant.__new__(cls, *args, **kwargs)
-
-class Num(Constant, metaclass=_ABC):
- _fields = ('n',)
- __new__ = _new
-
-class Str(Constant, metaclass=_ABC):
- _fields = ('s',)
- __new__ = _new
-
-class Bytes(Constant, metaclass=_ABC):
- _fields = ('s',)
- __new__ = _new
-
-class NameConstant(Constant, metaclass=_ABC):
- __new__ = _new
-
-class Ellipsis(Constant, metaclass=_ABC):
- _fields = ()
-
- def __new__(cls, *args, **kwargs):
- if cls is _ast_Ellipsis:
- import warnings
- warnings._deprecated(
- "ast.Ellipsis", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
- )
- return Constant(..., *args, **kwargs)
- return Constant.__new__(cls, *args, **kwargs)
-
-# Keep another reference to Ellipsis in the global namespace
-# so it can be referenced in Ellipsis.__new__
-# (The original "Ellipsis" name is removed from the global namespace later on)
-_ast_Ellipsis = Ellipsis
-
-_const_types = {
- Num: (int, float, complex),
- Str: (str,),
- Bytes: (bytes,),
- NameConstant: (type(None), bool),
- Ellipsis: (type(...),),
-}
-_const_types_not = {
- Num: (bool,),
-}
-
-_const_node_type_names = {
- bool: 'NameConstant', # should be before int
- type(None): 'NameConstant',
- int: 'Num',
- float: 'Num',
- complex: 'Num',
- str: 'Str',
- bytes: 'Bytes',
- type(...): 'Ellipsis',
-}
-
class slice(AST):
"""Deprecated AST node class."""
@@ -681,1138 +614,22 @@ class Param(expr_context):
"""Deprecated AST node class. Unused in Python 3."""
-# Large float and imaginary literals get turned into infinities in the AST.
-# We unparse those infinities to INFSTR.
-_INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1)
-
-@_simple_enum(IntEnum)
-class _Precedence:
- """Precedence table that originated from python grammar."""
-
- NAMED_EXPR = auto() # :=
- TUPLE = auto() # ,
- YIELD = auto() # 'yield', 'yield from'
- TEST = auto() # 'if'-'else', 'lambda'
- OR = auto() # 'or'
- AND = auto() # 'and'
- NOT = auto() # 'not'
- CMP = auto() # '<', '>', '==', '>=', '<=', '!=',
- # 'in', 'not in', 'is', 'is not'
- EXPR = auto()
- BOR = EXPR # '|'
- BXOR = auto() # '^'
- BAND = auto() # '&'
- SHIFT = auto() # '<<', '>>'
- ARITH = auto() # '+', '-'
- TERM = auto() # '*', '@', '/', '%', '//'
- FACTOR = auto() # unary '+', '-', '~'
- POWER = auto() # '**'
- AWAIT = auto() # 'await'
- ATOM = auto()
-
- def next(self):
- try:
- return self.__class__(self + 1)
- except ValueError:
- return self
-
-
-_SINGLE_QUOTES = ("'", '"')
-_MULTI_QUOTES = ('"""', "'''")
-_ALL_QUOTES = (*_SINGLE_QUOTES, *_MULTI_QUOTES)
-
-class _Unparser(NodeVisitor):
- """Methods in this class recursively traverse an AST and
- output source code for the abstract syntax; original formatting
- is disregarded."""
-
- def __init__(self, *, _avoid_backslashes=False):
- self._source = []
- self._precedences = {}
- self._type_ignores = {}
- self._indent = 0
- self._avoid_backslashes = _avoid_backslashes
- self._in_try_star = False
-
- def interleave(self, inter, f, seq):
- """Call f on each item in seq, calling inter() in between."""
- seq = iter(seq)
- try:
- f(next(seq))
- except StopIteration:
- pass
- else:
- for x in seq:
- inter()
- f(x)
-
- def items_view(self, traverser, items):
- """Traverse and separate the given *items* with a comma and append it to
- the buffer. If *items* is a single item sequence, a trailing comma
- will be added."""
- if len(items) == 1:
- traverser(items[0])
- self.write(",")
- else:
- self.interleave(lambda: self.write(", "), traverser, items)
-
- def maybe_newline(self):
- """Adds a newline if it isn't the start of generated source"""
- if self._source:
- self.write("\n")
-
- def fill(self, text=""):
- """Indent a piece of text and append it, according to the current
- indentation level"""
- self.maybe_newline()
- self.write(" " * self._indent + text)
-
- def write(self, *text):
- """Add new source parts"""
- self._source.extend(text)
-
- @contextmanager
- def buffered(self, buffer = None):
- if buffer is None:
- buffer = []
-
- original_source = self._source
- self._source = buffer
- yield buffer
- self._source = original_source
-
- @contextmanager
- def block(self, *, extra = None):
- """A context manager for preparing the source for blocks. It adds
- the character':', increases the indentation on enter and decreases
- the indentation on exit. If *extra* is given, it will be directly
- appended after the colon character.
- """
- self.write(":")
- if extra:
- self.write(extra)
- self._indent += 1
- yield
- self._indent -= 1
-
- @contextmanager
- def delimit(self, start, end):
- """A context manager for preparing the source for expressions. It adds
- *start* to the buffer and enters, after exit it adds *end*."""
-
- self.write(start)
- yield
- self.write(end)
-
- def delimit_if(self, start, end, condition):
- if condition:
- return self.delimit(start, end)
- else:
- return nullcontext()
-
- def require_parens(self, precedence, node):
- """Shortcut to adding precedence related parens"""
- return self.delimit_if("(", ")", self.get_precedence(node) > precedence)
-
- def get_precedence(self, node):
- return self._precedences.get(node, _Precedence.TEST)
-
- def set_precedence(self, precedence, *nodes):
- for node in nodes:
- self._precedences[node] = precedence
-
- def get_raw_docstring(self, node):
- """If a docstring node is found in the body of the *node* parameter,
- return that docstring node, None otherwise.
-
- Logic mirrored from ``_PyAST_GetDocString``."""
- if not isinstance(
- node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)
- ) or len(node.body) < 1:
- return None
- node = node.body[0]
- if not isinstance(node, Expr):
- return None
- node = node.value
- if isinstance(node, Constant) and isinstance(node.value, str):
- return node
-
- def get_type_comment(self, node):
- comment = self._type_ignores.get(node.lineno) or node.type_comment
- if comment is not None:
- return f" # type: {comment}"
-
- def traverse(self, node):
- if isinstance(node, list):
- for item in node:
- self.traverse(item)
- else:
- super().visit(node)
-
- # Note: as visit() resets the output text, do NOT rely on
- # NodeVisitor.generic_visit to handle any nodes (as it calls back in to
- # the subclass visit() method, which resets self._source to an empty list)
- def visit(self, node):
- """Outputs a source code string that, if converted back to an ast
- (using ast.parse) will generate an AST equivalent to *node*"""
- self._source = []
- self.traverse(node)
- return "".join(self._source)
-
- def _write_docstring_and_traverse_body(self, node):
- if (docstring := self.get_raw_docstring(node)):
- self._write_docstring(docstring)
- self.traverse(node.body[1:])
- else:
- self.traverse(node.body)
-
- def visit_Module(self, node):
- self._type_ignores = {
- ignore.lineno: f"ignore{ignore.tag}"
- for ignore in node.type_ignores
- }
- self._write_docstring_and_traverse_body(node)
- self._type_ignores.clear()
-
- def visit_FunctionType(self, node):
- with self.delimit("(", ")"):
- self.interleave(
- lambda: self.write(", "), self.traverse, node.argtypes
- )
-
- self.write(" -> ")
- self.traverse(node.returns)
-
- def visit_Expr(self, node):
- self.fill()
- self.set_precedence(_Precedence.YIELD, node.value)
- self.traverse(node.value)
-
- def visit_NamedExpr(self, node):
- with self.require_parens(_Precedence.NAMED_EXPR, node):
- self.set_precedence(_Precedence.ATOM, node.target, node.value)
- self.traverse(node.target)
- self.write(" := ")
- self.traverse(node.value)
-
- def visit_Import(self, node):
- self.fill("import ")
- self.interleave(lambda: self.write(", "), self.traverse, node.names)
-
- def visit_ImportFrom(self, node):
- self.fill("from ")
- self.write("." * (node.level or 0))
- if node.module:
- self.write(node.module)
- self.write(" import ")
- self.interleave(lambda: self.write(", "), self.traverse, node.names)
-
- def visit_Assign(self, node):
- self.fill()
- for target in node.targets:
- self.set_precedence(_Precedence.TUPLE, target)
- self.traverse(target)
- self.write(" = ")
- self.traverse(node.value)
- if type_comment := self.get_type_comment(node):
- self.write(type_comment)
-
- def visit_AugAssign(self, node):
- self.fill()
- self.traverse(node.target)
- self.write(" " + self.binop[node.op.__class__.__name__] + "= ")
- self.traverse(node.value)
-
- def visit_AnnAssign(self, node):
- self.fill()
- with self.delimit_if("(", ")", not node.simple and isinstance(node.target, Name)):
- self.traverse(node.target)
- self.write(": ")
- self.traverse(node.annotation)
- if node.value:
- self.write(" = ")
- self.traverse(node.value)
-
- def visit_Return(self, node):
- self.fill("return")
- if node.value:
- self.write(" ")
- self.traverse(node.value)
-
- def visit_Pass(self, node):
- self.fill("pass")
-
- def visit_Break(self, node):
- self.fill("break")
-
- def visit_Continue(self, node):
- self.fill("continue")
-
- def visit_Delete(self, node):
- self.fill("del ")
- self.interleave(lambda: self.write(", "), self.traverse, node.targets)
-
- def visit_Assert(self, node):
- self.fill("assert ")
- self.traverse(node.test)
- if node.msg:
- self.write(", ")
- self.traverse(node.msg)
-
- def visit_Global(self, node):
- self.fill("global ")
- self.interleave(lambda: self.write(", "), self.write, node.names)
-
- def visit_Nonlocal(self, node):
- self.fill("nonlocal ")
- self.interleave(lambda: self.write(", "), self.write, node.names)
-
- def visit_Await(self, node):
- with self.require_parens(_Precedence.AWAIT, node):
- self.write("await")
- if node.value:
- self.write(" ")
- self.set_precedence(_Precedence.ATOM, node.value)
- self.traverse(node.value)
-
- def visit_Yield(self, node):
- with self.require_parens(_Precedence.YIELD, node):
- self.write("yield")
- if node.value:
- self.write(" ")
- self.set_precedence(_Precedence.ATOM, node.value)
- self.traverse(node.value)
-
- def visit_YieldFrom(self, node):
- with self.require_parens(_Precedence.YIELD, node):
- self.write("yield from ")
- if not node.value:
- raise ValueError("Node can't be used without a value attribute.")
- self.set_precedence(_Precedence.ATOM, node.value)
- self.traverse(node.value)
-
- def visit_Raise(self, node):
- self.fill("raise")
- if not node.exc:
- if node.cause:
- raise ValueError(f"Node can't use cause without an exception.")
- return
- self.write(" ")
- self.traverse(node.exc)
- if node.cause:
- self.write(" from ")
- self.traverse(node.cause)
-
- def do_visit_try(self, node):
- self.fill("try")
- with self.block():
- self.traverse(node.body)
- for ex in node.handlers:
- self.traverse(ex)
- if node.orelse:
- self.fill("else")
- with self.block():
- self.traverse(node.orelse)
- if node.finalbody:
- self.fill("finally")
- with self.block():
- self.traverse(node.finalbody)
-
- def visit_Try(self, node):
- prev_in_try_star = self._in_try_star
- try:
- self._in_try_star = False
- self.do_visit_try(node)
- finally:
- self._in_try_star = prev_in_try_star
-
- def visit_TryStar(self, node):
- prev_in_try_star = self._in_try_star
- try:
- self._in_try_star = True
- self.do_visit_try(node)
- finally:
- self._in_try_star = prev_in_try_star
-
- def visit_ExceptHandler(self, node):
- self.fill("except*" if self._in_try_star else "except")
- if node.type:
- self.write(" ")
- self.traverse(node.type)
- if node.name:
- self.write(" as ")
- self.write(node.name)
- with self.block():
- self.traverse(node.body)
-
- def visit_ClassDef(self, node):
- self.maybe_newline()
- for deco in node.decorator_list:
- self.fill("@")
- self.traverse(deco)
- self.fill("class " + node.name)
- if hasattr(node, "type_params"):
- self._type_params_helper(node.type_params)
- with self.delimit_if("(", ")", condition = node.bases or node.keywords):
- comma = False
- for e in node.bases:
- if comma:
- self.write(", ")
- else:
- comma = True
- self.traverse(e)
- for e in node.keywords:
- if comma:
- self.write(", ")
- else:
- comma = True
- self.traverse(e)
-
- with self.block():
- self._write_docstring_and_traverse_body(node)
-
- def visit_FunctionDef(self, node):
- self._function_helper(node, "def")
-
- def visit_AsyncFunctionDef(self, node):
- self._function_helper(node, "async def")
-
- def _function_helper(self, node, fill_suffix):
- self.maybe_newline()
- for deco in node.decorator_list:
- self.fill("@")
- self.traverse(deco)
- def_str = fill_suffix + " " + node.name
- self.fill(def_str)
- if hasattr(node, "type_params"):
- self._type_params_helper(node.type_params)
- with self.delimit("(", ")"):
- self.traverse(node.args)
- if node.returns:
- self.write(" -> ")
- self.traverse(node.returns)
- with self.block(extra=self.get_type_comment(node)):
- self._write_docstring_and_traverse_body(node)
-
- def _type_params_helper(self, type_params):
- if type_params is not None and len(type_params) > 0:
- with self.delimit("[", "]"):
- self.interleave(lambda: self.write(", "), self.traverse, type_params)
-
- def visit_TypeVar(self, node):
- self.write(node.name)
- if node.bound:
- self.write(": ")
- self.traverse(node.bound)
-
- def visit_TypeVarTuple(self, node):
- self.write("*" + node.name)
-
- def visit_ParamSpec(self, node):
- self.write("**" + node.name)
-
- def visit_TypeAlias(self, node):
- self.fill("type ")
- self.traverse(node.name)
- self._type_params_helper(node.type_params)
- self.write(" = ")
- self.traverse(node.value)
-
- def visit_For(self, node):
- self._for_helper("for ", node)
-
- def visit_AsyncFor(self, node):
- self._for_helper("async for ", node)
-
- def _for_helper(self, fill, node):
- self.fill(fill)
- self.set_precedence(_Precedence.TUPLE, node.target)
- self.traverse(node.target)
- self.write(" in ")
- self.traverse(node.iter)
- with self.block(extra=self.get_type_comment(node)):
- self.traverse(node.body)
- if node.orelse:
- self.fill("else")
- with self.block():
- self.traverse(node.orelse)
-
- def visit_If(self, node):
- self.fill("if ")
- self.traverse(node.test)
- with self.block():
- self.traverse(node.body)
- # collapse nested ifs into equivalent elifs.
- while node.orelse and len(node.orelse) == 1 and isinstance(node.orelse[0], If):
- node = node.orelse[0]
- self.fill("elif ")
- self.traverse(node.test)
- with self.block():
- self.traverse(node.body)
- # final else
- if node.orelse:
- self.fill("else")
- with self.block():
- self.traverse(node.orelse)
-
- def visit_While(self, node):
- self.fill("while ")
- self.traverse(node.test)
- with self.block():
- self.traverse(node.body)
- if node.orelse:
- self.fill("else")
- with self.block():
- self.traverse(node.orelse)
-
- def visit_With(self, node):
- self.fill("with ")
- self.interleave(lambda: self.write(", "), self.traverse, node.items)
- with self.block(extra=self.get_type_comment(node)):
- self.traverse(node.body)
-
- def visit_AsyncWith(self, node):
- self.fill("async with ")
- self.interleave(lambda: self.write(", "), self.traverse, node.items)
- with self.block(extra=self.get_type_comment(node)):
- self.traverse(node.body)
-
- def _str_literal_helper(
- self, string, *, quote_types=_ALL_QUOTES, escape_special_whitespace=False
- ):
- """Helper for writing string literals, minimizing escapes.
- Returns the tuple (string literal to write, possible quote types).
- """
- def escape_char(c):
- # \n and \t are non-printable, but we only escape them if
- # escape_special_whitespace is True
- if not escape_special_whitespace and c in "\n\t":
- return c
- # Always escape backslashes and other non-printable characters
- if c == "\\" or not c.isprintable():
- return c.encode("unicode_escape").decode("ascii")
- return c
-
- escaped_string = "".join(map(escape_char, string))
- possible_quotes = quote_types
- if "\n" in escaped_string:
- possible_quotes = [q for q in possible_quotes if q in _MULTI_QUOTES]
- possible_quotes = [q for q in possible_quotes if q not in escaped_string]
- if not possible_quotes:
- # If there aren't any possible_quotes, fallback to using repr
- # on the original string. Try to use a quote from quote_types,
- # e.g., so that we use triple quotes for docstrings.
- string = repr(string)
- quote = next((q for q in quote_types if string[0] in q), string[0])
- return string[1:-1], [quote]
- if escaped_string:
- # Sort so that we prefer '''"''' over """\""""
- possible_quotes.sort(key=lambda q: q[0] == escaped_string[-1])
- # If we're using triple quotes and we'd need to escape a final
- # quote, escape it
- if possible_quotes[0][0] == escaped_string[-1]:
- assert len(possible_quotes[0]) == 3
- escaped_string = escaped_string[:-1] + "\\" + escaped_string[-1]
- return escaped_string, possible_quotes
-
- def _write_str_avoiding_backslashes(self, string, *, quote_types=_ALL_QUOTES):
- """Write string literal value with a best effort attempt to avoid backslashes."""
- string, quote_types = self._str_literal_helper(string, quote_types=quote_types)
- quote_type = quote_types[0]
- self.write(f"{quote_type}{string}{quote_type}")
-
- def visit_JoinedStr(self, node):
- self.write("f")
-
- fstring_parts = []
- for value in node.values:
- with self.buffered() as buffer:
- self._write_fstring_inner(value)
- fstring_parts.append(
- ("".join(buffer), isinstance(value, Constant))
- )
-
- new_fstring_parts = []
- quote_types = list(_ALL_QUOTES)
- fallback_to_repr = False
- for value, is_constant in fstring_parts:
- if is_constant:
- value, new_quote_types = self._str_literal_helper(
- value,
- quote_types=quote_types,
- escape_special_whitespace=True,
- )
- if set(new_quote_types).isdisjoint(quote_types):
- fallback_to_repr = True
- break
- quote_types = new_quote_types
- elif "\n" in value:
- quote_types = [q for q in quote_types if q in _MULTI_QUOTES]
- assert quote_types
- new_fstring_parts.append(value)
-
- if fallback_to_repr:
- # If we weren't able to find a quote type that works for all parts
- # of the JoinedStr, fallback to using repr and triple single quotes.
- quote_types = ["'''"]
- new_fstring_parts.clear()
- for value, is_constant in fstring_parts:
- if is_constant:
- value = repr('"' + value) # force repr to use single quotes
- expected_prefix = "'\""
- assert value.startswith(expected_prefix), repr(value)
- value = value[len(expected_prefix):-1]
- new_fstring_parts.append(value)
-
- value = "".join(new_fstring_parts)
- quote_type = quote_types[0]
- self.write(f"{quote_type}{value}{quote_type}")
-
- def _write_fstring_inner(self, node, is_format_spec=False):
- if isinstance(node, JoinedStr):
- # for both the f-string itself, and format_spec
- for value in node.values:
- self._write_fstring_inner(value, is_format_spec=is_format_spec)
- elif isinstance(node, Constant) and isinstance(node.value, str):
- value = node.value.replace("{", "{{").replace("}", "}}")
-
- if is_format_spec:
- value = value.replace("\\", "\\\\")
- value = value.replace("'", "\\'")
- value = value.replace('"', '\\"')
- value = value.replace("\n", "\\n")
- self.write(value)
- elif isinstance(node, FormattedValue):
- self.visit_FormattedValue(node)
- else:
- raise ValueError(f"Unexpected node inside JoinedStr, {node!r}")
-
- def visit_FormattedValue(self, node):
- def unparse_inner(inner):
- unparser = type(self)()
- unparser.set_precedence(_Precedence.TEST.next(), inner)
- return unparser.visit(inner)
-
- with self.delimit("{", "}"):
- expr = unparse_inner(node.value)
- if expr.startswith("{"):
- # Separate pair of opening brackets as "{ {"
- self.write(" ")
- self.write(expr)
- if node.conversion != -1:
- self.write(f"!{chr(node.conversion)}")
- if node.format_spec:
- self.write(":")
- self._write_fstring_inner(node.format_spec, is_format_spec=True)
-
- def visit_Name(self, node):
- self.write(node.id)
-
- def _write_docstring(self, node):
- self.fill()
- if node.kind == "u":
- self.write("u")
- self._write_str_avoiding_backslashes(node.value, quote_types=_MULTI_QUOTES)
-
- def _write_constant(self, value):
- if isinstance(value, (float, complex)):
- # Substitute overflowing decimal literal for AST infinities,
- # and inf - inf for NaNs.
- self.write(
- repr(value)
- .replace("inf", _INFSTR)
- .replace("nan", f"({_INFSTR}-{_INFSTR})")
- )
- elif self._avoid_backslashes and isinstance(value, str):
- self._write_str_avoiding_backslashes(value)
- else:
- self.write(repr(value))
-
- def visit_Constant(self, node):
- value = node.value
- if isinstance(value, tuple):
- with self.delimit("(", ")"):
- self.items_view(self._write_constant, value)
- elif value is ...:
- self.write("...")
- else:
- if node.kind == "u":
- self.write("u")
- self._write_constant(node.value)
-
- def visit_List(self, node):
- with self.delimit("[", "]"):
- self.interleave(lambda: self.write(", "), self.traverse, node.elts)
-
- def visit_ListComp(self, node):
- with self.delimit("[", "]"):
- self.traverse(node.elt)
- for gen in node.generators:
- self.traverse(gen)
-
- def visit_GeneratorExp(self, node):
- with self.delimit("(", ")"):
- self.traverse(node.elt)
- for gen in node.generators:
- self.traverse(gen)
-
- def visit_SetComp(self, node):
- with self.delimit("{", "}"):
- self.traverse(node.elt)
- for gen in node.generators:
- self.traverse(gen)
-
- def visit_DictComp(self, node):
- with self.delimit("{", "}"):
- self.traverse(node.key)
- self.write(": ")
- self.traverse(node.value)
- for gen in node.generators:
- self.traverse(gen)
-
- def visit_comprehension(self, node):
- if node.is_async:
- self.write(" async for ")
- else:
- self.write(" for ")
- self.set_precedence(_Precedence.TUPLE, node.target)
- self.traverse(node.target)
- self.write(" in ")
- self.set_precedence(_Precedence.TEST.next(), node.iter, *node.ifs)
- self.traverse(node.iter)
- for if_clause in node.ifs:
- self.write(" if ")
- self.traverse(if_clause)
-
- def visit_IfExp(self, node):
- with self.require_parens(_Precedence.TEST, node):
- self.set_precedence(_Precedence.TEST.next(), node.body, node.test)
- self.traverse(node.body)
- self.write(" if ")
- self.traverse(node.test)
- self.write(" else ")
- self.set_precedence(_Precedence.TEST, node.orelse)
- self.traverse(node.orelse)
-
- def visit_Set(self, node):
- if node.elts:
- with self.delimit("{", "}"):
- self.interleave(lambda: self.write(", "), self.traverse, node.elts)
- else:
- # `{}` would be interpreted as a dictionary literal, and
- # `set` might be shadowed. Thus:
- self.write('{*()}')
-
- def visit_Dict(self, node):
- def write_key_value_pair(k, v):
- self.traverse(k)
- self.write(": ")
- self.traverse(v)
-
- def write_item(item):
- k, v = item
- if k is None:
- # for dictionary unpacking operator in dicts {**{'y': 2}}
- # see PEP 448 for details
- self.write("**")
- self.set_precedence(_Precedence.EXPR, v)
- self.traverse(v)
- else:
- write_key_value_pair(k, v)
-
- with self.delimit("{", "}"):
- self.interleave(
- lambda: self.write(", "), write_item, zip(node.keys, node.values)
- )
-
- def visit_Tuple(self, node):
- with self.delimit_if(
- "(",
- ")",
- len(node.elts) == 0 or self.get_precedence(node) > _Precedence.TUPLE
- ):
- self.items_view(self.traverse, node.elts)
-
- unop = {"Invert": "~", "Not": "not", "UAdd": "+", "USub": "-"}
- unop_precedence = {
- "not": _Precedence.NOT,
- "~": _Precedence.FACTOR,
- "+": _Precedence.FACTOR,
- "-": _Precedence.FACTOR,
- }
-
- def visit_UnaryOp(self, node):
- operator = self.unop[node.op.__class__.__name__]
- operator_precedence = self.unop_precedence[operator]
- with self.require_parens(operator_precedence, node):
- self.write(operator)
- # factor prefixes (+, -, ~) shouldn't be separated
- # from the value they belong, (e.g: +1 instead of + 1)
- if operator_precedence is not _Precedence.FACTOR:
- self.write(" ")
- self.set_precedence(operator_precedence, node.operand)
- self.traverse(node.operand)
-
- binop = {
- "Add": "+",
- "Sub": "-",
- "Mult": "*",
- "MatMult": "@",
- "Div": "/",
- "Mod": "%",
- "LShift": "<<",
- "RShift": ">>",
- "BitOr": "|",
- "BitXor": "^",
- "BitAnd": "&",
- "FloorDiv": "//",
- "Pow": "**",
- }
-
- binop_precedence = {
- "+": _Precedence.ARITH,
- "-": _Precedence.ARITH,
- "*": _Precedence.TERM,
- "@": _Precedence.TERM,
- "/": _Precedence.TERM,
- "%": _Precedence.TERM,
- "<<": _Precedence.SHIFT,
- ">>": _Precedence.SHIFT,
- "|": _Precedence.BOR,
- "^": _Precedence.BXOR,
- "&": _Precedence.BAND,
- "//": _Precedence.TERM,
- "**": _Precedence.POWER,
- }
-
- binop_rassoc = frozenset(("**",))
- def visit_BinOp(self, node):
- operator = self.binop[node.op.__class__.__name__]
- operator_precedence = self.binop_precedence[operator]
- with self.require_parens(operator_precedence, node):
- if operator in self.binop_rassoc:
- left_precedence = operator_precedence.next()
- right_precedence = operator_precedence
- else:
- left_precedence = operator_precedence
- right_precedence = operator_precedence.next()
-
- self.set_precedence(left_precedence, node.left)
- self.traverse(node.left)
- self.write(f" {operator} ")
- self.set_precedence(right_precedence, node.right)
- self.traverse(node.right)
-
- cmpops = {
- "Eq": "==",
- "NotEq": "!=",
- "Lt": "<",
- "LtE": "<=",
- "Gt": ">",
- "GtE": ">=",
- "Is": "is",
- "IsNot": "is not",
- "In": "in",
- "NotIn": "not in",
- }
-
- def visit_Compare(self, node):
- with self.require_parens(_Precedence.CMP, node):
- self.set_precedence(_Precedence.CMP.next(), node.left, *node.comparators)
- self.traverse(node.left)
- for o, e in zip(node.ops, node.comparators):
- self.write(" " + self.cmpops[o.__class__.__name__] + " ")
- self.traverse(e)
-
- boolops = {"And": "and", "Or": "or"}
- boolop_precedence = {"and": _Precedence.AND, "or": _Precedence.OR}
-
- def visit_BoolOp(self, node):
- operator = self.boolops[node.op.__class__.__name__]
- operator_precedence = self.boolop_precedence[operator]
-
- def increasing_level_traverse(node):
- nonlocal operator_precedence
- operator_precedence = operator_precedence.next()
- self.set_precedence(operator_precedence, node)
- self.traverse(node)
-
- with self.require_parens(operator_precedence, node):
- s = f" {operator} "
- self.interleave(lambda: self.write(s), increasing_level_traverse, node.values)
-
- def visit_Attribute(self, node):
- self.set_precedence(_Precedence.ATOM, node.value)
- self.traverse(node.value)
- # Special case: 3.__abs__() is a syntax error, so if node.value
- # is an integer literal then we need to either parenthesize
- # it or add an extra space to get 3 .__abs__().
- if isinstance(node.value, Constant) and isinstance(node.value.value, int):
- self.write(" ")
- self.write(".")
- self.write(node.attr)
-
- def visit_Call(self, node):
- self.set_precedence(_Precedence.ATOM, node.func)
- self.traverse(node.func)
- with self.delimit("(", ")"):
- comma = False
- for e in node.args:
- if comma:
- self.write(", ")
- else:
- comma = True
- self.traverse(e)
- for e in node.keywords:
- if comma:
- self.write(", ")
- else:
- comma = True
- self.traverse(e)
-
- def visit_Subscript(self, node):
- def is_non_empty_tuple(slice_value):
- return (
- isinstance(slice_value, Tuple)
- and slice_value.elts
- )
-
- self.set_precedence(_Precedence.ATOM, node.value)
- self.traverse(node.value)
- with self.delimit("[", "]"):
- if is_non_empty_tuple(node.slice):
- # parentheses can be omitted if the tuple isn't empty
- self.items_view(self.traverse, node.slice.elts)
- else:
- self.traverse(node.slice)
-
- def visit_Starred(self, node):
- self.write("*")
- self.set_precedence(_Precedence.EXPR, node.value)
- self.traverse(node.value)
-
- def visit_Ellipsis(self, node):
- self.write("...")
-
- def visit_Slice(self, node):
- if node.lower:
- self.traverse(node.lower)
- self.write(":")
- if node.upper:
- self.traverse(node.upper)
- if node.step:
- self.write(":")
- self.traverse(node.step)
-
- def visit_Match(self, node):
- self.fill("match ")
- self.traverse(node.subject)
- with self.block():
- for case in node.cases:
- self.traverse(case)
-
- def visit_arg(self, node):
- self.write(node.arg)
- if node.annotation:
- self.write(": ")
- self.traverse(node.annotation)
-
- def visit_arguments(self, node):
- first = True
- # normal arguments
- all_args = node.posonlyargs + node.args
- defaults = [None] * (len(all_args) - len(node.defaults)) + node.defaults
- for index, elements in enumerate(zip(all_args, defaults), 1):
- a, d = elements
- if first:
- first = False
- else:
- self.write(", ")
- self.traverse(a)
- if d:
- self.write("=")
- self.traverse(d)
- if index == len(node.posonlyargs):
- self.write(", /")
-
- # varargs, or bare '*' if no varargs but keyword-only arguments present
- if node.vararg or node.kwonlyargs:
- if first:
- first = False
- else:
- self.write(", ")
- self.write("*")
- if node.vararg:
- self.write(node.vararg.arg)
- if node.vararg.annotation:
- self.write(": ")
- self.traverse(node.vararg.annotation)
-
- # keyword-only arguments
- if node.kwonlyargs:
- for a, d in zip(node.kwonlyargs, node.kw_defaults):
- self.write(", ")
- self.traverse(a)
- if d:
- self.write("=")
- self.traverse(d)
-
- # kwargs
- if node.kwarg:
- if first:
- first = False
- else:
- self.write(", ")
- self.write("**" + node.kwarg.arg)
- if node.kwarg.annotation:
- self.write(": ")
- self.traverse(node.kwarg.annotation)
-
- def visit_keyword(self, node):
- if node.arg is None:
- self.write("**")
- else:
- self.write(node.arg)
- self.write("=")
- self.traverse(node.value)
-
- def visit_Lambda(self, node):
- with self.require_parens(_Precedence.TEST, node):
- self.write("lambda")
- with self.buffered() as buffer:
- self.traverse(node.args)
- if buffer:
- self.write(" ", *buffer)
- self.write(": ")
- self.set_precedence(_Precedence.TEST, node.body)
- self.traverse(node.body)
-
- def visit_alias(self, node):
- self.write(node.name)
- if node.asname:
- self.write(" as " + node.asname)
-
- def visit_withitem(self, node):
- self.traverse(node.context_expr)
- if node.optional_vars:
- self.write(" as ")
- self.traverse(node.optional_vars)
-
- def visit_match_case(self, node):
- self.fill("case ")
- self.traverse(node.pattern)
- if node.guard:
- self.write(" if ")
- self.traverse(node.guard)
- with self.block():
- self.traverse(node.body)
-
- def visit_MatchValue(self, node):
- self.traverse(node.value)
-
- def visit_MatchSingleton(self, node):
- self._write_constant(node.value)
-
- def visit_MatchSequence(self, node):
- with self.delimit("[", "]"):
- self.interleave(
- lambda: self.write(", "), self.traverse, node.patterns
- )
-
- def visit_MatchStar(self, node):
- name = node.name
- if name is None:
- name = "_"
- self.write(f"*{name}")
-
- def visit_MatchMapping(self, node):
- def write_key_pattern_pair(pair):
- k, p = pair
- self.traverse(k)
- self.write(": ")
- self.traverse(p)
-
- with self.delimit("{", "}"):
- keys = node.keys
- self.interleave(
- lambda: self.write(", "),
- write_key_pattern_pair,
- zip(keys, node.patterns, strict=True),
- )
- rest = node.rest
- if rest is not None:
- if keys:
- self.write(", ")
- self.write(f"**{rest}")
-
- def visit_MatchClass(self, node):
- self.set_precedence(_Precedence.ATOM, node.cls)
- self.traverse(node.cls)
- with self.delimit("(", ")"):
- patterns = node.patterns
- self.interleave(
- lambda: self.write(", "), self.traverse, patterns
- )
- attrs = node.kwd_attrs
- if attrs:
- def write_attr_pattern(pair):
- attr, pattern = pair
- self.write(f"{attr}=")
- self.traverse(pattern)
-
- if patterns:
- self.write(", ")
- self.interleave(
- lambda: self.write(", "),
- write_attr_pattern,
- zip(attrs, node.kwd_patterns, strict=True),
- )
-
- def visit_MatchAs(self, node):
- name = node.name
- pattern = node.pattern
- if name is None:
- self.write("_")
- elif pattern is None:
- self.write(node.name)
- else:
- with self.require_parens(_Precedence.TEST, node):
- self.set_precedence(_Precedence.BOR, node.pattern)
- self.traverse(node.pattern)
- self.write(f" as {node.name}")
-
- def visit_MatchOr(self, node):
- with self.require_parens(_Precedence.BOR, node):
- self.set_precedence(_Precedence.BOR.next(), *node.patterns)
- self.interleave(lambda: self.write(" | "), self.traverse, node.patterns)
-
def unparse(ast_obj):
- unparser = _Unparser()
+ global _Unparser
+ try:
+ unparser = _Unparser()
+ except NameError:
+ from _ast_unparse import Unparser as _Unparser
+ unparser = _Unparser()
return unparser.visit(ast_obj)
-_deprecated_globals = {
- name: globals().pop(name)
- for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis')
-}
-
-def __getattr__(name):
- if name in _deprecated_globals:
- globals()[name] = value = _deprecated_globals[name]
- import warnings
- warnings._deprecated(
- f"ast.{name}", message=_DEPRECATED_CLASS_MESSAGE, remove=(3, 14)
- )
- return value
- raise AttributeError(f"module 'ast' has no attribute '{name}'")
-
-
-def main():
+def main(args=None):
import argparse
+ import sys
- parser = argparse.ArgumentParser(prog='python -m ast')
- parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?',
- default='-',
+ parser = argparse.ArgumentParser(color=True)
+ parser.add_argument('infile', nargs='?', default='-',
help='the file to parse; defaults to stdin')
parser.add_argument('-m', '--mode', default='exec',
choices=('exec', 'single', 'eval', 'func_type'),
@@ -1824,12 +641,40 @@ def main():
'column offsets')
parser.add_argument('-i', '--indent', type=int, default=3,
help='indentation of nodes (number of spaces)')
- args = parser.parse_args()
+ parser.add_argument('--feature-version',
+ type=str, default=None, metavar='VERSION',
+ help='Python version in the format 3.x '
+ '(for example, 3.10)')
+ parser.add_argument('-O', '--optimize',
+ type=int, default=-1, metavar='LEVEL',
+ help='optimization level for parser (default -1)')
+ parser.add_argument('--show-empty', default=False, action='store_true',
+ help='show empty lists and fields in dump output')
+ args = parser.parse_args(args)
+
+ if args.infile == '-':
+ name = ''
+ source = sys.stdin.buffer.read()
+ else:
+ name = args.infile
+ with open(args.infile, 'rb') as infile:
+ source = infile.read()
+
+ # Process feature_version
+ feature_version = None
+ if args.feature_version:
+ try:
+ major, minor = map(int, args.feature_version.split('.', 1))
+ except ValueError:
+ parser.error('Invalid format for --feature-version; '
+ 'expected format 3.x (for example, 3.10)')
+
+ feature_version = (major, minor)
- with args.infile as infile:
- source = infile.read()
- tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments)
- print(dump(tree, include_attributes=args.include_attributes, indent=args.indent))
+ tree = parse(source, name, args.mode, type_comments=args.no_type_comments,
+ feature_version=feature_version, optimize=args.optimize)
+ print(dump(tree, include_attributes=args.include_attributes,
+ indent=args.indent, show_empty=args.show_empty))
if __name__ == '__main__':
main()
diff --git a/PythonLib/full/asyncio/__init__.py b/PythonLib/full/asyncio/__init__.py
index 03165a425..32a5dbae0 100644
--- a/PythonLib/full/asyncio/__init__.py
+++ b/PythonLib/full/asyncio/__init__.py
@@ -10,6 +10,7 @@
from .events import *
from .exceptions import *
from .futures import *
+from .graph import *
from .locks import *
from .protocols import *
from .runners import *
@@ -27,6 +28,7 @@
events.__all__ +
exceptions.__all__ +
futures.__all__ +
+ graph.__all__ +
locks.__all__ +
protocols.__all__ +
runners.__all__ +
@@ -45,3 +47,28 @@
else:
from .unix_events import * # pragma: no cover
__all__ += unix_events.__all__
+
+def __getattr__(name: str):
+ import warnings
+
+ match name:
+ case "AbstractEventLoopPolicy":
+ warnings._deprecated(f"asyncio.{name}", remove=(3, 16))
+ return events._AbstractEventLoopPolicy
+ case "DefaultEventLoopPolicy":
+ warnings._deprecated(f"asyncio.{name}", remove=(3, 16))
+ if sys.platform == 'win32':
+ return windows_events._DefaultEventLoopPolicy
+ return unix_events._DefaultEventLoopPolicy
+ case "WindowsSelectorEventLoopPolicy":
+ if sys.platform == 'win32':
+ warnings._deprecated(f"asyncio.{name}", remove=(3, 16))
+ return windows_events._WindowsSelectorEventLoopPolicy
+ # Else fall through to the AttributeError below.
+ case "WindowsProactorEventLoopPolicy":
+ if sys.platform == 'win32':
+ warnings._deprecated(f"asyncio.{name}", remove=(3, 16))
+ return windows_events._WindowsProactorEventLoopPolicy
+ # Else fall through to the AttributeError below.
+
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/PythonLib/full/asyncio/__main__.py b/PythonLib/full/asyncio/__main__.py
index 29e528aee..44e14771b 100644
--- a/PythonLib/full/asyncio/__main__.py
+++ b/PythonLib/full/asyncio/__main__.py
@@ -1,42 +1,53 @@
+import argparse
import ast
import asyncio
-import code
+import asyncio.tools
import concurrent.futures
import contextvars
import inspect
+import os
+import site
import sys
import threading
import types
import warnings
+from _colorize import get_theme
+from _pyrepl.console import InteractiveColoredConsole
+
from . import futures
-class AsyncIOInteractiveConsole(code.InteractiveConsole):
+class AsyncIOInteractiveConsole(InteractiveColoredConsole):
def __init__(self, locals, loop):
- super().__init__(locals)
+ super().__init__(locals, filename="")
self.compile.compiler.flags |= ast.PyCF_ALLOW_TOP_LEVEL_AWAIT
+
self.loop = loop
self.context = contextvars.copy_context()
def runcode(self, code):
+ global return_code
future = concurrent.futures.Future()
def callback():
+ global return_code
global repl_future
- global repl_future_interrupted
+ global keyboard_interrupted
repl_future = None
- repl_future_interrupted = False
+ keyboard_interrupted = False
func = types.FunctionType(code, self.locals)
try:
coro = func()
- except SystemExit:
- raise
+ except SystemExit as se:
+ return_code = se.code
+ self.loop.stop()
+ return
except KeyboardInterrupt as ex:
- repl_future_interrupted = True
+ keyboard_interrupted = True
future.set_exception(ex)
return
except BaseException as ex:
@@ -53,34 +64,72 @@ def callback():
except BaseException as exc:
future.set_exception(exc)
- loop.call_soon_threadsafe(callback, context=self.context)
+ self.loop.call_soon_threadsafe(callback, context=self.context)
try:
return future.result()
- except SystemExit:
- raise
+ except SystemExit as se:
+ return_code = se.code
+ self.loop.stop()
+ return
except BaseException:
- if repl_future_interrupted:
- self.write("\nKeyboardInterrupt\n")
+ if keyboard_interrupted:
+ if not CAN_USE_PYREPL:
+ self.write("\nKeyboardInterrupt\n")
else:
self.showtraceback()
-
+ return self.STATEMENT_FAILED
class REPLThread(threading.Thread):
def run(self):
+ global return_code
+
try:
- banner = (
- f'asyncio REPL {sys.version} on {sys.platform}\n'
- f'Use "await" directly instead of "asyncio.run()".\n'
- f'Type "help", "copyright", "credits" or "license" '
- f'for more information.\n'
- f'{getattr(sys, "ps1", ">>> ")}import asyncio'
- )
-
- console.interact(
- banner=banner,
- exitmsg='exiting asyncio REPL...')
+ if not sys.flags.quiet:
+ banner = (
+ f'asyncio REPL {sys.version} on {sys.platform}\n'
+ f'Use "await" directly instead of "asyncio.run()".\n'
+ f'Type "help", "copyright", "credits" or "license" '
+ f'for more information.\n'
+ )
+
+ console.write(banner)
+
+ if not sys.flags.isolated and (startup_path := os.getenv("PYTHONSTARTUP")):
+ sys.audit("cpython.run_startup", startup_path)
+ try:
+ import tokenize
+ with tokenize.open(startup_path) as f:
+ startup_code = compile(f.read(), startup_path, "exec")
+ exec(startup_code, console.locals)
+ except SystemExit:
+ raise
+ except BaseException:
+ console.showtraceback()
+
+ ps1 = getattr(sys, "ps1", ">>> ")
+ if CAN_USE_PYREPL:
+ theme = get_theme().syntax
+ ps1 = f"{theme.prompt}{ps1}{theme.reset}"
+ console.write(f"{ps1}import asyncio\n")
+
+ if CAN_USE_PYREPL:
+ from _pyrepl.simple_interact import (
+ run_multiline_interactive_console,
+ )
+ try:
+ run_multiline_interactive_console(console)
+ except SystemExit:
+ # expected via the `exit` and `quit` commands
+ pass
+ except BaseException:
+ # unexpected issue
+ console.showtraceback()
+ console.write("Internal error, ")
+ return_code = 1
+ else:
+ console.interact(banner="", exitmsg="")
finally:
warnings.filterwarnings(
'ignore',
@@ -89,10 +138,56 @@ def run(self):
loop.call_soon_threadsafe(loop.stop)
+ def interrupt(self) -> None:
+ if not CAN_USE_PYREPL:
+ return
+
+ from _pyrepl.simple_interact import _get_reader
+ r = _get_reader()
+ if r.threading_hook is not None:
+ r.threading_hook.add("") # type: ignore
+
if __name__ == '__main__':
+ parser = argparse.ArgumentParser(
+ prog="python3 -m asyncio",
+ description="Interactive asyncio shell and CLI tools",
+ color=True,
+ )
+ subparsers = parser.add_subparsers(help="sub-commands", dest="command")
+ ps = subparsers.add_parser(
+ "ps", help="Display a table of all pending tasks in a process"
+ )
+ ps.add_argument("pid", type=int, help="Process ID to inspect")
+ pstree = subparsers.add_parser(
+ "pstree", help="Display a tree of all pending tasks in a process"
+ )
+ pstree.add_argument("pid", type=int, help="Process ID to inspect")
+ args = parser.parse_args()
+ match args.command:
+ case "ps":
+ asyncio.tools.display_awaited_by_tasks_table(args.pid)
+ sys.exit(0)
+ case "pstree":
+ asyncio.tools.display_awaited_by_tasks_tree(args.pid)
+ sys.exit(0)
+ case None:
+ pass # continue to the interactive shell
+ case _:
+ # shouldn't happen as an invalid command-line wouldn't parse
+ # but let's keep it for the next person adding a command
+ print(f"error: unhandled command {args.command}", file=sys.stderr)
+ parser.print_usage(file=sys.stderr)
+ sys.exit(1)
+
sys.audit("cpython.run_stdin")
+ if os.getenv('PYTHON_BASIC_REPL'):
+ CAN_USE_PYREPL = False
+ else:
+ from _pyrepl.main import CAN_USE_PYREPL
+
+ return_code = 0
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
@@ -105,14 +200,31 @@ def run(self):
console = AsyncIOInteractiveConsole(repl_locals, loop)
repl_future = None
- repl_future_interrupted = False
+ keyboard_interrupted = False
try:
import readline # NoQA
except ImportError:
- pass
+ readline = None
+
+ interactive_hook = getattr(sys, "__interactivehook__", None)
+
+ if interactive_hook is not None:
+ sys.audit("cpython.run_interactivehook", interactive_hook)
+ interactive_hook()
- repl_thread = REPLThread()
+ if interactive_hook is site.register_readline:
+ # Fix the completer function to use the interactive console locals
+ try:
+ import rlcompleter
+ except:
+ pass
+ else:
+ if readline is not None:
+ completer = rlcompleter.Completer(console.locals)
+ readline.set_completer(completer.complete)
+
+ repl_thread = REPLThread(name="Interactive thread")
repl_thread.daemon = True
repl_thread.start()
@@ -120,9 +232,14 @@ def run(self):
try:
loop.run_forever()
except KeyboardInterrupt:
+ keyboard_interrupted = True
if repl_future and not repl_future.done():
repl_future.cancel()
- repl_future_interrupted = True
+ repl_thread.interrupt()
continue
else:
break
+
+ console.write('exiting asyncio REPL...\n')
+ loop.close()
+ sys.exit(return_code)
diff --git a/PythonLib/full/asyncio/base_events.py b/PythonLib/full/asyncio/base_events.py
index 3146f7f3f..b83b84181 100644
--- a/PythonLib/full/asyncio/base_events.py
+++ b/PythonLib/full/asyncio/base_events.py
@@ -278,7 +278,9 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
ssl_handshake_timeout, ssl_shutdown_timeout=None):
self._loop = loop
self._sockets = sockets
- self._active_count = 0
+ # Weak references so we don't break Transport's ability to
+ # detect abandoned transports
+ self._clients = weakref.WeakSet()
self._waiters = []
self._protocol_factory = protocol_factory
self._backlog = backlog
@@ -291,14 +293,13 @@ def __init__(self, loop, sockets, protocol_factory, ssl_context, backlog,
def __repr__(self):
return f'<{self.__class__.__name__} sockets={self.sockets!r}>'
- def _attach(self):
+ def _attach(self, transport):
assert self._sockets is not None
- self._active_count += 1
+ self._clients.add(transport)
- def _detach(self):
- assert self._active_count > 0
- self._active_count -= 1
- if self._active_count == 0 and self._sockets is None:
+ def _detach(self, transport):
+ self._clients.discard(transport)
+ if len(self._clients) == 0 and self._sockets is None:
self._wakeup()
def _wakeup(self):
@@ -347,9 +348,17 @@ def close(self):
self._serving_forever_fut.cancel()
self._serving_forever_fut = None
- if self._active_count == 0:
+ if len(self._clients) == 0:
self._wakeup()
+ def close_clients(self):
+ for transport in self._clients.copy():
+ transport.close()
+
+ def abort_clients(self):
+ for transport in self._clients.copy():
+ transport.abort()
+
async def start_serving(self):
self._start_serving()
# Skip one loop iteration so that all 'loop.add_reader'
@@ -421,6 +430,8 @@ def __init__(self):
self._clock_resolution = time.get_clock_info('monotonic').resolution
self._exception_handler = None
self.set_debug(coroutines._is_debug_mode())
+ # The preserved state of async generator hooks.
+ self._old_agen_hooks = None
# In debug mode, if the execution of a callback or a step of a task
# exceed this duration in seconds, the slow callback/task is logged.
self.slow_callback_duration = 0.1
@@ -447,26 +458,24 @@ def create_future(self):
"""Create a Future object attached to the loop."""
return futures.Future(loop=self)
- def create_task(self, coro, *, name=None, context=None):
- """Schedule a coroutine object.
+ def create_task(self, coro, **kwargs):
+ """Schedule or begin executing a coroutine object.
Return a task object.
"""
self._check_closed()
- if self._task_factory is None:
- task = tasks.Task(coro, loop=self, name=name, context=context)
- if task._source_traceback:
- del task._source_traceback[-1]
- else:
- if context is None:
- # Use legacy API if context is not needed
- task = self._task_factory(self, coro)
- else:
- task = self._task_factory(self, coro, context=context)
-
- tasks._set_task_name(task, name)
+ if self._task_factory is not None:
+ return self._task_factory(self, coro, **kwargs)
- return task
+ task = tasks.Task(coro, loop=self, **kwargs)
+ if task._source_traceback:
+ del task._source_traceback[-1]
+ try:
+ return task
+ finally:
+ # gh-128552: prevent a refcycle of
+ # task.exception().__traceback__->BaseEventLoop.create_task->task
+ del task
def set_task_factory(self, factory):
"""Set a task factory that will be used by loop.create_task().
@@ -474,9 +483,10 @@ def set_task_factory(self, factory):
If factory is None the default task factory will be set.
If factory is a callable, it should have a signature matching
- '(loop, coro)', where 'loop' will be a reference to the active
- event loop, 'coro' will be a coroutine object. The callable
- must return a Future.
+ '(loop, coro, **kwargs)', where 'loop' will be a reference to the active
+ event loop, 'coro' will be a coroutine object, and **kwargs will be
+ arbitrary keyword arguments that should be passed on to Task.
+ The callable must return a Task.
"""
if factory is not None and not callable(factory):
raise TypeError('task factory must be a callable or None')
@@ -623,29 +633,52 @@ def _check_running(self):
raise RuntimeError(
'Cannot run the event loop while another loop is running')
- def run_forever(self):
- """Run until stop() is called."""
+ def _run_forever_setup(self):
+ """Prepare the run loop to process events.
+
+ This method exists so that custom event loop subclasses (e.g., event loops
+ that integrate a GUI event loop with Python's event loop) have access to all the
+ loop setup logic.
+ """
self._check_closed()
self._check_running()
self._set_coroutine_origin_tracking(self._debug)
- old_agen_hooks = sys.get_asyncgen_hooks()
- try:
- self._thread_id = threading.get_ident()
- sys.set_asyncgen_hooks(firstiter=self._asyncgen_firstiter_hook,
- finalizer=self._asyncgen_finalizer_hook)
+ self._old_agen_hooks = sys.get_asyncgen_hooks()
+ self._thread_id = threading.get_ident()
+ sys.set_asyncgen_hooks(
+ firstiter=self._asyncgen_firstiter_hook,
+ finalizer=self._asyncgen_finalizer_hook
+ )
+
+ events._set_running_loop(self)
+
+ def _run_forever_cleanup(self):
+ """Clean up after an event loop finishes the looping over events.
+
+ This method exists so that custom event loop subclasses (e.g., event loops
+ that integrate a GUI event loop with Python's event loop) have access to all the
+ loop cleanup logic.
+ """
+ self._stopping = False
+ self._thread_id = None
+ events._set_running_loop(None)
+ self._set_coroutine_origin_tracking(False)
+ # Restore any pre-existing async generator hooks.
+ if self._old_agen_hooks is not None:
+ sys.set_asyncgen_hooks(*self._old_agen_hooks)
+ self._old_agen_hooks = None
- events._set_running_loop(self)
+ def run_forever(self):
+ """Run until stop() is called."""
+ self._run_forever_setup()
+ try:
while True:
self._run_once()
if self._stopping:
break
finally:
- self._stopping = False
- self._thread_id = None
- events._set_running_loop(None)
- self._set_coroutine_origin_tracking(False)
- sys.set_asyncgen_hooks(*old_agen_hooks)
+ self._run_forever_cleanup()
def run_until_complete(self, future):
"""Run until the Future is done.
@@ -802,7 +835,7 @@ def call_soon(self, callback, *args, context=None):
def _check_callback(self, callback, method):
if (coroutines.iscoroutine(callback) or
- coroutines.iscoroutinefunction(callback)):
+ coroutines._iscoroutinefunction(callback)):
raise TypeError(
f"coroutines cannot be used with {method}()")
if not callable(callback):
@@ -839,7 +872,10 @@ def call_soon_threadsafe(self, callback, *args, context=None):
self._check_closed()
if self._debug:
self._check_callback(callback, 'call_soon_threadsafe')
- handle = self._call_soon(callback, args, context)
+ handle = events._ThreadSafeHandle(callback, args, self, context)
+ self._ready.append(handle)
+ if handle._source_traceback:
+ del handle._source_traceback[-1]
if handle._source_traceback:
del handle._source_traceback[-1]
self._write_to_self()
@@ -980,38 +1016,43 @@ async def _connect_sock(self, exceptions, addr_info, local_addr_infos=None):
family, type_, proto, _, address = addr_info
sock = None
try:
- sock = socket.socket(family=family, type=type_, proto=proto)
- sock.setblocking(False)
- if local_addr_infos is not None:
- for lfamily, _, _, _, laddr in local_addr_infos:
- # skip local addresses of different family
- if lfamily != family:
- continue
- try:
- sock.bind(laddr)
- break
- except OSError as exc:
- msg = (
- f'error while attempting to bind on '
- f'address {laddr!r}: {str(exc).lower()}'
- )
- exc = OSError(exc.errno, msg)
- my_exceptions.append(exc)
- else: # all bind attempts failed
- if my_exceptions:
- raise my_exceptions.pop()
- else:
- raise OSError(f"no matching local address with {family=} found")
- await self.sock_connect(sock, address)
- return sock
- except OSError as exc:
- my_exceptions.append(exc)
- if sock is not None:
- sock.close()
- raise
+ try:
+ sock = socket.socket(family=family, type=type_, proto=proto)
+ sock.setblocking(False)
+ if local_addr_infos is not None:
+ for lfamily, _, _, _, laddr in local_addr_infos:
+ # skip local addresses of different family
+ if lfamily != family:
+ continue
+ try:
+ sock.bind(laddr)
+ break
+ except OSError as exc:
+ msg = (
+ f'error while attempting to bind on '
+ f'address {laddr!r}: {str(exc).lower()}'
+ )
+ exc = OSError(exc.errno, msg)
+ my_exceptions.append(exc)
+ else: # all bind attempts failed
+ if my_exceptions:
+ raise my_exceptions.pop()
+ else:
+ raise OSError(f"no matching local address with {family=} found")
+ await self.sock_connect(sock, address)
+ return sock
+ except OSError as exc:
+ my_exceptions.append(exc)
+ raise
except:
if sock is not None:
- sock.close()
+ try:
+ sock.close()
+ except OSError:
+ # An error when closing a newly created socket is
+ # not important, but it can overwrite more important
+ # non-OSError error. So ignore it.
+ pass
raise
finally:
exceptions = my_exceptions = None
@@ -1125,7 +1166,7 @@ async def create_connection(
raise ExceptionGroup("create_connection failed", exceptions)
if len(exceptions) == 1:
raise exceptions[0]
- else:
+ elif exceptions:
# If they all have the same str(), raise one.
model = str(exceptions[0])
if all(str(exc) == model for exc in exceptions):
@@ -1134,6 +1175,9 @@ async def create_connection(
# the various error messages.
raise OSError('Multiple exceptions: {}'.format(
', '.join(str(exc) for exc in exceptions)))
+ else:
+ # No exceptions were collected, raise a timeout error
+ raise TimeoutError('create_connection failed')
finally:
exceptions = None
@@ -1259,8 +1303,8 @@ async def _sendfile_fallback(self, transp, file, offset, count):
read = await self.run_in_executor(None, file.readinto, view)
if not read:
return total_sent # EOF
- await proto.drain()
transp.write(view[:read])
+ await proto.drain()
total_sent += read
finally:
if total_sent > 0 and hasattr(file, 'seek'):
@@ -1301,6 +1345,17 @@ async def start_tls(self, transport, protocol, sslcontext, *,
# have a chance to get called before "ssl_protocol.connection_made()".
transport.pause_reading()
+ # gh-142352: move buffered StreamReader data to SSLProtocol
+ if server_side:
+ from .streams import StreamReaderProtocol
+ if isinstance(protocol, StreamReaderProtocol):
+ stream_reader = getattr(protocol, '_stream_reader', None)
+ if stream_reader is not None:
+ buffer = stream_reader._buffer
+ if buffer:
+ ssl_protocol._incoming.write(buffer)
+ buffer.clear()
+
transport.set_protocol(ssl_protocol)
conmade_cb = self.call_soon(ssl_protocol.connection_made, transport)
resume_cb = self.call_soon(transport.resume_reading)
@@ -1479,6 +1534,7 @@ async def create_server(
ssl=None,
reuse_address=None,
reuse_port=None,
+ keep_alive=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
start_serving=True):
@@ -1550,8 +1606,13 @@ async def create_server(
if reuse_address:
sock.setsockopt(
socket.SOL_SOCKET, socket.SO_REUSEADDR, True)
- if reuse_port:
+ # Since Linux 6.12.9, SO_REUSEPORT is not allowed
+ # on other address families than AF_INET/AF_INET6.
+ if reuse_port and af in (socket.AF_INET, socket.AF_INET6):
_set_reuseport(sock)
+ if keep_alive:
+ sock.setsockopt(
+ socket.SOL_SOCKET, socket.SO_KEEPALIVE, True)
# Disable IPv4/IPv6 dual stack support (enabled by
# default on Linux) which makes a single socket
# listen on both address families.
@@ -1624,8 +1685,7 @@ async def connect_accepted_socket(
raise ValueError(
'ssl_shutdown_timeout is only meaningful with ssl')
- if sock is not None:
- _check_ssl_socket(sock)
+ _check_ssl_socket(sock)
transport, protocol = await self._create_connection_transport(
sock, protocol_factory, ssl, '', server_side=True,
@@ -1838,6 +1898,8 @@ def call_exception_handler(self, context):
- 'protocol' (optional): Protocol instance;
- 'transport' (optional): Transport instance;
- 'socket' (optional): Socket instance;
+ - 'source_traceback' (optional): Traceback of the source;
+ - 'handle_traceback' (optional): Traceback of the handle;
- 'asyncgen' (optional): Asynchronous generator that caused
the exception.
@@ -1948,8 +2010,11 @@ def _run_once(self):
timeout = 0
elif self._scheduled:
# Compute the desired timeout.
- when = self._scheduled[0]._when
- timeout = min(max(0, when - self.time()), MAXIMUM_SELECT_TIMEOUT)
+ timeout = self._scheduled[0]._when - self.time()
+ if timeout > MAXIMUM_SELECT_TIMEOUT:
+ timeout = MAXIMUM_SELECT_TIMEOUT
+ elif timeout < 0:
+ timeout = 0
event_list = self._selector.select(timeout)
self._process_events(event_list)
diff --git a/PythonLib/full/asyncio/base_subprocess.py b/PythonLib/full/asyncio/base_subprocess.py
index 4c9b0dd56..224b18838 100644
--- a/PythonLib/full/asyncio/base_subprocess.py
+++ b/PythonLib/full/asyncio/base_subprocess.py
@@ -1,6 +1,9 @@
import collections
import subprocess
import warnings
+import os
+import signal
+import sys
from . import protocols
from . import transports
@@ -23,6 +26,7 @@ def __init__(self, loop, protocol, args, shell,
self._pending_calls = collections.deque()
self._pipes = {}
self._finished = False
+ self._pipes_connected = False
if stdin == subprocess.PIPE:
self._pipes[0] = None
@@ -101,7 +105,12 @@ def close(self):
for proto in self._pipes.values():
if proto is None:
continue
- proto.pipe.close()
+ # See gh-114177
+ # skip closing the pipe if loop is already closed
+ # this can happen e.g. when loop is closed immediately after
+ # process is killed
+ if self._loop and not self._loop.is_closed():
+ proto.pipe.close()
if (self._proc is not None and
# has the child process finished?
@@ -115,7 +124,8 @@ def close(self):
try:
self._proc.kill()
- except ProcessLookupError:
+ except (ProcessLookupError, PermissionError):
+ # the process may have already exited or may be running setuid
pass
# Don't clear the _proc reference yet: _post_init() may still run
@@ -141,17 +151,31 @@ def _check_proc(self):
if self._proc is None:
raise ProcessLookupError()
- def send_signal(self, signal):
- self._check_proc()
- self._proc.send_signal(signal)
+ if sys.platform == 'win32':
+ def send_signal(self, signal):
+ self._check_proc()
+ self._proc.send_signal(signal)
+
+ def terminate(self):
+ self._check_proc()
+ self._proc.terminate()
+
+ def kill(self):
+ self._check_proc()
+ self._proc.kill()
+ else:
+ def send_signal(self, signal):
+ self._check_proc()
+ try:
+ os.kill(self._proc.pid, signal)
+ except ProcessLookupError:
+ pass
- def terminate(self):
- self._check_proc()
- self._proc.terminate()
+ def terminate(self):
+ self.send_signal(signal.SIGTERM)
- def kill(self):
- self._check_proc()
- self._proc.kill()
+ def kill(self):
+ self.send_signal(signal.SIGKILL)
async def _connect_pipes(self, waiter):
try:
@@ -190,6 +214,7 @@ async def _connect_pipes(self, waiter):
else:
if waiter is not None and not waiter.cancelled():
waiter.set_result(None)
+ self._pipes_connected = True
def _call(self, cb, *data):
if self._pending_calls is not None:
@@ -233,6 +258,15 @@ def _try_finish(self):
assert not self._finished
if self._returncode is None:
return
+ if not self._pipes_connected:
+ # self._pipes_connected can be False if not all pipes were connected
+ # because either the process failed to start or the self._connect_pipes task
+ # got cancelled. In this broken state we consider all pipes disconnected and
+ # to avoid hanging forever in self._wait as otherwise _exit_waiters
+ # would never be woken up, we wake them up here.
+ for waiter in self._exit_waiters:
+ if not waiter.done():
+ waiter.set_result(self._returncode)
if all(p is not None and p.disconnected
for p in self._pipes.values()):
self._finished = True
@@ -244,7 +278,7 @@ def _call_connection_lost(self, exc):
finally:
# wake up futures waiting for wait()
for waiter in self._exit_waiters:
- if not waiter.cancelled():
+ if not waiter.done():
waiter.set_result(self._returncode)
self._exit_waiters = None
self._loop = None
diff --git a/PythonLib/full/asyncio/coroutines.py b/PythonLib/full/asyncio/coroutines.py
index ab4f30eb5..a51319cb7 100644
--- a/PythonLib/full/asyncio/coroutines.py
+++ b/PythonLib/full/asyncio/coroutines.py
@@ -18,7 +18,16 @@ def _is_debug_mode():
def iscoroutinefunction(func):
+ import warnings
"""Return True if func is a decorated coroutine function."""
+ warnings._deprecated("asyncio.iscoroutinefunction",
+ f"{warnings._DEPRECATED_MSG}; "
+ "use inspect.iscoroutinefunction() instead",
+ remove=(3,16))
+ return _iscoroutinefunction(func)
+
+
+def _iscoroutinefunction(func):
return (inspect.iscoroutinefunction(func) or
getattr(func, '_is_coroutine', None) is _is_coroutine)
diff --git a/PythonLib/full/asyncio/events.py b/PythonLib/full/asyncio/events.py
index 016852880..a7fb55982 100644
--- a/PythonLib/full/asyncio/events.py
+++ b/PythonLib/full/asyncio/events.py
@@ -5,14 +5,18 @@
# SPDX-FileCopyrightText: Copyright (c) 2015-2021 MagicStack Inc. http://magic.io
__all__ = (
- 'AbstractEventLoopPolicy',
- 'AbstractEventLoop', 'AbstractServer',
- 'Handle', 'TimerHandle',
- 'get_event_loop_policy', 'set_event_loop_policy',
- 'get_event_loop', 'set_event_loop', 'new_event_loop',
- 'get_child_watcher', 'set_child_watcher',
- '_set_running_loop', 'get_running_loop',
- '_get_running_loop',
+ "AbstractEventLoop",
+ "AbstractServer",
+ "Handle",
+ "TimerHandle",
+ "get_event_loop_policy",
+ "set_event_loop_policy",
+ "get_event_loop",
+ "set_event_loop",
+ "new_event_loop",
+ "_set_running_loop",
+ "get_running_loop",
+ "_get_running_loop",
)
import contextvars
@@ -22,6 +26,7 @@
import subprocess
import sys
import threading
+import warnings
from . import format_helpers
@@ -54,7 +59,8 @@ def _repr_info(self):
info.append('cancelled')
if self._callback is not None:
info.append(format_helpers._format_callback_source(
- self._callback, self._args))
+ self._callback, self._args,
+ debug=self._loop.get_debug()))
if self._source_traceback:
frame = self._source_traceback[-1]
info.append(f'created at {frame[0]}:{frame[1]}')
@@ -90,7 +96,8 @@ def _run(self):
raise
except BaseException as exc:
cb = format_helpers._format_callback_source(
- self._callback, self._args)
+ self._callback, self._args,
+ debug=self._loop.get_debug())
msg = f'Exception in callback {cb}'
context = {
'message': msg,
@@ -102,6 +109,34 @@ def _run(self):
self._loop.call_exception_handler(context)
self = None # Needed to break cycles when an exception occurs.
+# _ThreadSafeHandle is used for callbacks scheduled with call_soon_threadsafe
+# and is thread safe unlike Handle which is not thread safe.
+class _ThreadSafeHandle(Handle):
+
+ __slots__ = ('_lock',)
+
+ def __init__(self, callback, args, loop, context=None):
+ super().__init__(callback, args, loop, context)
+ self._lock = threading.RLock()
+
+ def cancel(self):
+ with self._lock:
+ return super().cancel()
+
+ def cancelled(self):
+ with self._lock:
+ return super().cancelled()
+
+ def _run(self):
+ # The event loop checks for cancellation without holding the lock
+ # It is possible that the handle is cancelled after the check
+ # but before the callback is called so check it again after acquiring
+ # the lock and return without calling the callback if it is cancelled.
+ with self._lock:
+ if self._cancelled:
+ return
+ return super()._run()
+
class TimerHandle(Handle):
"""Object returned by timed callback registration methods."""
@@ -173,6 +208,14 @@ def close(self):
"""Stop serving. This leaves existing connections open."""
raise NotImplementedError
+ def close_clients(self):
+ """Close all active connections."""
+ raise NotImplementedError
+
+ def abort_clients(self):
+ """Close all active connections immediately."""
+ raise NotImplementedError
+
def get_loop(self):
"""Get the event loop the Server object is attached to."""
raise NotImplementedError
@@ -282,7 +325,7 @@ def create_future(self):
# Method scheduling a coroutine object: create a task.
- def create_task(self, coro, *, name=None, context=None):
+ def create_task(self, coro, **kwargs):
raise NotImplementedError
# Methods for interacting with threads.
@@ -320,6 +363,7 @@ async def create_server(
*, family=socket.AF_UNSPEC,
flags=socket.AI_PASSIVE, sock=None, backlog=100,
ssl=None, reuse_address=None, reuse_port=None,
+ keep_alive=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
start_serving=True):
@@ -358,6 +402,9 @@ async def create_server(
they all set this flag when being created. This option is not
supported on Windows.
+ keep_alive set to True keeps connections active by enabling the
+ periodic transmission of messages.
+
ssl_handshake_timeout is the time in seconds that an SSL server
will wait for completion of the SSL handshake before aborting the
connection. Default is 60s.
@@ -615,7 +662,7 @@ def set_debug(self, enabled):
raise NotImplementedError
-class AbstractEventLoopPolicy:
+class _AbstractEventLoopPolicy:
"""Abstract policy for accessing the event loop."""
def get_event_loop(self):
@@ -638,18 +685,7 @@ def new_event_loop(self):
the current context, set_event_loop must be called explicitly."""
raise NotImplementedError
- # Child processes handling (Unix only).
-
- def get_child_watcher(self):
- "Get the watcher for child processes."
- raise NotImplementedError
-
- def set_child_watcher(self, watcher):
- """Set the watcher for child processes."""
- raise NotImplementedError
-
-
-class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
+class _BaseDefaultEventLoopPolicy(_AbstractEventLoopPolicy):
"""Default policy implementation for accessing the event loop.
In this policy, each thread has its own event loop. However, we
@@ -666,7 +702,6 @@ class BaseDefaultEventLoopPolicy(AbstractEventLoopPolicy):
class _Local(threading.local):
_loop = None
- _set_called = False
def __init__(self):
self._local = self._Local()
@@ -676,28 +711,6 @@ def get_event_loop(self):
Returns an instance of EventLoop or raises an exception.
"""
- if (self._local._loop is None and
- not self._local._set_called and
- threading.current_thread() is threading.main_thread()):
- stacklevel = 2
- try:
- f = sys._getframe(1)
- except AttributeError:
- pass
- else:
- # Move up the call stack so that the warning is attached
- # to the line outside asyncio itself.
- while f:
- module = f.f_globals.get('__name__')
- if not (module == 'asyncio' or module.startswith('asyncio.')):
- break
- f = f.f_back
- stacklevel += 1
- import warnings
- warnings.warn('There is no current event loop',
- DeprecationWarning, stacklevel=stacklevel)
- self.set_event_loop(self.new_event_loop())
-
if self._local._loop is None:
raise RuntimeError('There is no current event loop in thread %r.'
% threading.current_thread().name)
@@ -706,7 +719,6 @@ def get_event_loop(self):
def set_event_loop(self, loop):
"""Set the event loop."""
- self._local._set_called = True
if loop is not None and not isinstance(loop, AbstractEventLoop):
raise TypeError(f"loop must be an instance of AbstractEventLoop or None, not '{type(loop).__name__}'")
self._local._loop = loop
@@ -776,26 +788,35 @@ def _init_event_loop_policy():
global _event_loop_policy
with _lock:
if _event_loop_policy is None: # pragma: no branch
- from . import DefaultEventLoopPolicy
- _event_loop_policy = DefaultEventLoopPolicy()
+ if sys.platform == 'win32':
+ from .windows_events import _DefaultEventLoopPolicy
+ else:
+ from .unix_events import _DefaultEventLoopPolicy
+ _event_loop_policy = _DefaultEventLoopPolicy()
-def get_event_loop_policy():
+def _get_event_loop_policy():
"""Get the current event loop policy."""
if _event_loop_policy is None:
_init_event_loop_policy()
return _event_loop_policy
+def get_event_loop_policy():
+ warnings._deprecated('asyncio.get_event_loop_policy', remove=(3, 16))
+ return _get_event_loop_policy()
-def set_event_loop_policy(policy):
+def _set_event_loop_policy(policy):
"""Set the current event loop policy.
If policy is None, the default policy is restored."""
global _event_loop_policy
- if policy is not None and not isinstance(policy, AbstractEventLoopPolicy):
+ if policy is not None and not isinstance(policy, _AbstractEventLoopPolicy):
raise TypeError(f"policy must be an instance of AbstractEventLoopPolicy or None, not '{type(policy).__name__}'")
_event_loop_policy = policy
+def set_event_loop_policy(policy):
+ warnings._deprecated('asyncio.set_event_loop_policy', remove=(3,16))
+ _set_event_loop_policy(policy)
def get_event_loop():
"""Return an asyncio event loop.
@@ -810,28 +831,17 @@ def get_event_loop():
current_loop = _get_running_loop()
if current_loop is not None:
return current_loop
- return get_event_loop_policy().get_event_loop()
+ return _get_event_loop_policy().get_event_loop()
def set_event_loop(loop):
"""Equivalent to calling get_event_loop_policy().set_event_loop(loop)."""
- get_event_loop_policy().set_event_loop(loop)
+ _get_event_loop_policy().set_event_loop(loop)
def new_event_loop():
"""Equivalent to calling get_event_loop_policy().new_event_loop()."""
- return get_event_loop_policy().new_event_loop()
-
-
-def get_child_watcher():
- """Equivalent to calling get_event_loop_policy().get_child_watcher()."""
- return get_event_loop_policy().get_child_watcher()
-
-
-def set_child_watcher(watcher):
- """Equivalent to calling
- get_event_loop_policy().set_child_watcher(watcher)."""
- return get_event_loop_policy().set_child_watcher(watcher)
+ return _get_event_loop_policy().new_event_loop()
# Alias pure-Python implementations for testing purposes.
@@ -861,7 +871,7 @@ def set_child_watcher(watcher):
def on_fork():
# Reset the loop and wakeupfd in the forked child process.
if _event_loop_policy is not None:
- _event_loop_policy._local = BaseDefaultEventLoopPolicy._Local()
+ _event_loop_policy._local = _BaseDefaultEventLoopPolicy._Local()
_set_running_loop(None)
signal.set_wakeup_fd(-1)
diff --git a/PythonLib/full/asyncio/format_helpers.py b/PythonLib/full/asyncio/format_helpers.py
index 27d11fd4f..93737b770 100644
--- a/PythonLib/full/asyncio/format_helpers.py
+++ b/PythonLib/full/asyncio/format_helpers.py
@@ -19,19 +19,26 @@ def _get_function_source(func):
return None
-def _format_callback_source(func, args):
- func_repr = _format_callback(func, args, None)
+def _format_callback_source(func, args, *, debug=False):
+ func_repr = _format_callback(func, args, None, debug=debug)
source = _get_function_source(func)
if source:
func_repr += f' at {source[0]}:{source[1]}'
return func_repr
-def _format_args_and_kwargs(args, kwargs):
+def _format_args_and_kwargs(args, kwargs, *, debug=False):
"""Format function arguments and keyword arguments.
Special case for a single parameter: ('hello',) is formatted as ('hello').
+
+ Note that this function only returns argument details when
+ debug=True is specified, as arguments may contain sensitive
+ information.
"""
+ if not debug:
+ return '()'
+
# use reprlib to limit the length of the output
items = []
if args:
@@ -41,10 +48,11 @@ def _format_args_and_kwargs(args, kwargs):
return '({})'.format(', '.join(items))
-def _format_callback(func, args, kwargs, suffix=''):
+def _format_callback(func, args, kwargs, *, debug=False, suffix=''):
if isinstance(func, functools.partial):
- suffix = _format_args_and_kwargs(args, kwargs) + suffix
- return _format_callback(func.func, func.args, func.keywords, suffix)
+ suffix = _format_args_and_kwargs(args, kwargs, debug=debug) + suffix
+ return _format_callback(func.func, func.args, func.keywords,
+ debug=debug, suffix=suffix)
if hasattr(func, '__qualname__') and func.__qualname__:
func_repr = func.__qualname__
@@ -53,7 +61,7 @@ def _format_callback(func, args, kwargs, suffix=''):
else:
func_repr = repr(func)
- func_repr += _format_args_and_kwargs(args, kwargs)
+ func_repr += _format_args_and_kwargs(args, kwargs, debug=debug)
if suffix:
func_repr += suffix
return func_repr
diff --git a/PythonLib/full/asyncio/futures.py b/PythonLib/full/asyncio/futures.py
index 0c530bbdb..e8a99556b 100644
--- a/PythonLib/full/asyncio/futures.py
+++ b/PythonLib/full/asyncio/futures.py
@@ -2,6 +2,7 @@
__all__ = (
'Future', 'wrap_future', 'isfuture',
+ 'future_add_to_awaited_by', 'future_discard_from_awaited_by',
)
import concurrent.futures
@@ -43,7 +44,6 @@ class Future:
- This class is not compatible with the wait() and as_completed()
methods in the concurrent.futures package.
- (In Python 3.4 or later we may be able to unify the implementations.)
"""
# Class variables serving as defaults for instance variables.
@@ -61,12 +61,15 @@ class Future:
# the Future protocol (i.e. is intended to be duck-type compatible).
# The value must also be not-None, to enable a subclass to declare
# that it is not compatible by setting this to None.
- # - It is set by __iter__() below so that Task._step() can tell
+ # - It is set by __iter__() below so that Task.__step() can tell
# the difference between
- # `await Future()` or`yield from Future()` (correct) vs.
+ # `await Future()` or `yield from Future()` (correct) vs.
# `yield Future()` (incorrect).
_asyncio_future_blocking = False
+ # Used by the capture_call_stack() API.
+ __asyncio_awaited_by = None
+
__log_traceback = False
def __init__(self, *, loop=None):
@@ -116,6 +119,12 @@ def _log_traceback(self, val):
raise ValueError('_log_traceback can only be set to False')
self.__log_traceback = False
+ @property
+ def _asyncio_awaited_by(self):
+ if self.__asyncio_awaited_by is None:
+ return None
+ return frozenset(self.__asyncio_awaited_by)
+
def get_loop(self):
"""Return the event loop the Future is bound to."""
loop = self._loop
@@ -138,9 +147,6 @@ def _make_cancelled_error(self):
exc = exceptions.CancelledError()
else:
exc = exceptions.CancelledError(self._cancel_message)
- exc.__context__ = self._cancelled_exc
- # Remove the reference since we don't need this anymore.
- self._cancelled_exc = None
return exc
def cancel(self, msg=None):
@@ -320,11 +326,9 @@ def _set_result_unless_cancelled(fut, result):
def _convert_future_exc(exc):
exc_class = type(exc)
if exc_class is concurrent.futures.CancelledError:
- return exceptions.CancelledError(*exc.args)
- elif exc_class is concurrent.futures.TimeoutError:
- return exceptions.TimeoutError(*exc.args)
+ return exceptions.CancelledError(*exc.args).with_traceback(exc.__traceback__)
elif exc_class is concurrent.futures.InvalidStateError:
- return exceptions.InvalidStateError(*exc.args)
+ return exceptions.InvalidStateError(*exc.args).with_traceback(exc.__traceback__)
else:
return exc
@@ -388,7 +392,7 @@ def _set_state(future, other):
def _call_check_cancel(destination):
if destination.cancelled():
- if source_loop is None or source_loop is dest_loop:
+ if source_loop is None or source_loop is events._get_running_loop():
source.cancel()
else:
source_loop.call_soon_threadsafe(source.cancel)
@@ -397,7 +401,7 @@ def _call_set_state(source):
if (destination.cancelled() and
dest_loop is not None and dest_loop.is_closed()):
return
- if dest_loop is None or dest_loop is source_loop:
+ if dest_loop is None or dest_loop is events._get_running_loop():
_set_state(destination, source)
else:
if dest_loop.is_closed():
@@ -421,6 +425,49 @@ def wrap_future(future, *, loop=None):
return new_future
+def future_add_to_awaited_by(fut, waiter, /):
+ """Record that `fut` is awaited on by `waiter`."""
+ # For the sake of keeping the implementation minimal and assuming
+ # that most of asyncio users use the built-in Futures and Tasks
+ # (or their subclasses), we only support native Future objects
+ # and their subclasses.
+ #
+ # Longer version: tracking requires storing the caller-callee
+ # dependency somewhere. One obvious choice is to store that
+ # information right in the future itself in a dedicated attribute.
+ # This means that we'd have to require all duck-type compatible
+ # futures to implement a specific attribute used by asyncio for
+ # the book keeping. Another solution would be to store that in
+ # a global dictionary. The downside here is that that would create
+ # strong references and any scenario where the "add" call isn't
+ # followed by a "discard" call would lead to a memory leak.
+ # Using WeakDict would resolve that issue, but would complicate
+ # the C code (_asynciomodule.c). The bottom line here is that
+ # it's not clear that all this work would be worth the effort.
+ #
+ # Note that there's an accelerated version of this function
+ # shadowing this implementation later in this file.
+ if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture):
+ if fut._Future__asyncio_awaited_by is None:
+ fut._Future__asyncio_awaited_by = set()
+ fut._Future__asyncio_awaited_by.add(waiter)
+
+
+def future_discard_from_awaited_by(fut, waiter, /):
+ """Record that `fut` is no longer awaited on by `waiter`."""
+ # See the comment in "future_add_to_awaited_by()" body for
+ # details on implementation.
+ #
+ # Note that there's an accelerated version of this function
+ # shadowing this implementation later in this file.
+ if isinstance(fut, _PyFuture) and isinstance(waiter, _PyFuture):
+ if fut._Future__asyncio_awaited_by is not None:
+ fut._Future__asyncio_awaited_by.discard(waiter)
+
+
+_py_future_add_to_awaited_by = future_add_to_awaited_by
+_py_future_discard_from_awaited_by = future_discard_from_awaited_by
+
try:
import _asyncio
except ImportError:
@@ -428,3 +475,7 @@ def wrap_future(future, *, loop=None):
else:
# _CFuture is needed for tests.
Future = _CFuture = _asyncio.Future
+ future_add_to_awaited_by = _asyncio.future_add_to_awaited_by
+ future_discard_from_awaited_by = _asyncio.future_discard_from_awaited_by
+ _c_future_add_to_awaited_by = future_add_to_awaited_by
+ _c_future_discard_from_awaited_by = future_discard_from_awaited_by
diff --git a/PythonLib/full/asyncio/graph.py b/PythonLib/full/asyncio/graph.py
new file mode 100644
index 000000000..b5bfeb163
--- /dev/null
+++ b/PythonLib/full/asyncio/graph.py
@@ -0,0 +1,276 @@
+"""Introspection utils for tasks call graphs."""
+
+import dataclasses
+import io
+import sys
+import types
+
+from . import events
+from . import futures
+from . import tasks
+
+__all__ = (
+ 'capture_call_graph',
+ 'format_call_graph',
+ 'print_call_graph',
+ 'FrameCallGraphEntry',
+ 'FutureCallGraph',
+)
+
+# Sadly, we can't re-use the traceback module's datastructures as those
+# are tailored for error reporting, whereas we need to represent an
+# async call graph.
+#
+# Going with pretty verbose names as we'd like to export them to the
+# top level asyncio namespace, and want to avoid future name clashes.
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class FrameCallGraphEntry:
+ frame: types.FrameType
+
+
+@dataclasses.dataclass(frozen=True, slots=True)
+class FutureCallGraph:
+ future: futures.Future
+ call_stack: tuple["FrameCallGraphEntry", ...]
+ awaited_by: tuple["FutureCallGraph", ...]
+
+
+def _build_graph_for_future(
+ future: futures.Future,
+ *,
+ limit: int | None = None,
+) -> FutureCallGraph:
+ if not isinstance(future, futures.Future):
+ raise TypeError(
+ f"{future!r} object does not appear to be compatible "
+ f"with asyncio.Future"
+ )
+
+ coro = None
+ if get_coro := getattr(future, 'get_coro', None):
+ coro = get_coro() if limit != 0 else None
+
+ st: list[FrameCallGraphEntry] = []
+ awaited_by: list[FutureCallGraph] = []
+
+ while coro is not None:
+ if hasattr(coro, 'cr_await'):
+ # A native coroutine or duck-type compatible iterator
+ st.append(FrameCallGraphEntry(coro.cr_frame))
+ coro = coro.cr_await
+ elif hasattr(coro, 'ag_await'):
+ # A native async generator or duck-type compatible iterator
+ st.append(FrameCallGraphEntry(coro.cr_frame))
+ coro = coro.ag_await
+ else:
+ break
+
+ if future._asyncio_awaited_by:
+ for parent in future._asyncio_awaited_by:
+ awaited_by.append(_build_graph_for_future(parent, limit=limit))
+
+ if limit is not None:
+ if limit > 0:
+ st = st[:limit]
+ elif limit < 0:
+ st = st[limit:]
+ st.reverse()
+ return FutureCallGraph(future, tuple(st), tuple(awaited_by))
+
+
+def capture_call_graph(
+ future: futures.Future | None = None,
+ /,
+ *,
+ depth: int = 1,
+ limit: int | None = None,
+) -> FutureCallGraph | None:
+ """Capture the async call graph for the current task or the provided Future.
+
+ The graph is represented with three data structures:
+
+ * FutureCallGraph(future, call_stack, awaited_by)
+
+ Where 'future' is an instance of asyncio.Future or asyncio.Task.
+
+ 'call_stack' is a tuple of FrameGraphEntry objects.
+
+ 'awaited_by' is a tuple of FutureCallGraph objects.
+
+ * FrameCallGraphEntry(frame)
+
+ Where 'frame' is a frame object of a regular Python function
+ in the call stack.
+
+ Receives an optional 'future' argument. If not passed,
+ the current task will be used. If there's no current task, the function
+ returns None.
+
+ If "capture_call_graph()" is introspecting *the current task*, the
+ optional keyword-only 'depth' argument can be used to skip the specified
+ number of frames from top of the stack.
+
+ If the optional keyword-only 'limit' argument is provided, each call stack
+ in the resulting graph is truncated to include at most ``abs(limit)``
+ entries. If 'limit' is positive, the entries left are the closest to
+ the invocation point. If 'limit' is negative, the topmost entries are
+ left. If 'limit' is omitted or None, all entries are present.
+ If 'limit' is 0, the call stack is not captured at all, only
+ "awaited by" information is present.
+ """
+
+ loop = events._get_running_loop()
+
+ if future is not None:
+ # Check if we're in a context of a running event loop;
+ # if yes - check if the passed future is the currently
+ # running task or not.
+ if loop is None or future is not tasks.current_task(loop=loop):
+ return _build_graph_for_future(future, limit=limit)
+ # else: future is the current task, move on.
+ else:
+ if loop is None:
+ raise RuntimeError(
+ 'capture_call_graph() is called outside of a running '
+ 'event loop and no *future* to introspect was provided')
+ future = tasks.current_task(loop=loop)
+
+ if future is None:
+ # This isn't a generic call stack introspection utility. If we
+ # can't determine the current task and none was provided, we
+ # just return.
+ return None
+
+ if not isinstance(future, futures.Future):
+ raise TypeError(
+ f"{future!r} object does not appear to be compatible "
+ f"with asyncio.Future"
+ )
+
+ call_stack: list[FrameCallGraphEntry] = []
+
+ f = sys._getframe(depth) if limit != 0 else None
+ try:
+ while f is not None:
+ is_async = f.f_generator is not None
+ call_stack.append(FrameCallGraphEntry(f))
+
+ if is_async:
+ if f.f_back is not None and f.f_back.f_generator is None:
+ # We've reached the bottom of the coroutine stack, which
+ # must be the Task that runs it.
+ break
+
+ f = f.f_back
+ finally:
+ del f
+
+ awaited_by = []
+ if future._asyncio_awaited_by:
+ for parent in future._asyncio_awaited_by:
+ awaited_by.append(_build_graph_for_future(parent, limit=limit))
+
+ if limit is not None:
+ limit *= -1
+ if limit > 0:
+ call_stack = call_stack[:limit]
+ elif limit < 0:
+ call_stack = call_stack[limit:]
+
+ return FutureCallGraph(future, tuple(call_stack), tuple(awaited_by))
+
+
+def format_call_graph(
+ future: futures.Future | None = None,
+ /,
+ *,
+ depth: int = 1,
+ limit: int | None = None,
+) -> str:
+ """Return the async call graph as a string for `future`.
+
+ If `future` is not provided, format the call graph for the current task.
+ """
+
+ def render_level(st: FutureCallGraph, buf: list[str], level: int) -> None:
+ def add_line(line: str) -> None:
+ buf.append(level * ' ' + line)
+
+ if isinstance(st.future, tasks.Task):
+ add_line(
+ f'* Task(name={st.future.get_name()!r}, id={id(st.future):#x})'
+ )
+ else:
+ add_line(
+ f'* Future(id={id(st.future):#x})'
+ )
+
+ if st.call_stack:
+ add_line(
+ f' + Call stack:'
+ )
+ for ste in st.call_stack:
+ f = ste.frame
+
+ if f.f_generator is None:
+ f = ste.frame
+ add_line(
+ f' | File {f.f_code.co_filename!r},'
+ f' line {f.f_lineno}, in'
+ f' {f.f_code.co_qualname}()'
+ )
+ else:
+ c = f.f_generator
+
+ try:
+ f = c.cr_frame
+ code = c.cr_code
+ tag = 'async'
+ except AttributeError:
+ try:
+ f = c.ag_frame
+ code = c.ag_code
+ tag = 'async generator'
+ except AttributeError:
+ f = c.gi_frame
+ code = c.gi_code
+ tag = 'generator'
+
+ add_line(
+ f' | File {f.f_code.co_filename!r},'
+ f' line {f.f_lineno}, in'
+ f' {tag} {code.co_qualname}()'
+ )
+
+ if st.awaited_by:
+ add_line(
+ f' + Awaited by:'
+ )
+ for fut in st.awaited_by:
+ render_level(fut, buf, level + 1)
+
+ graph = capture_call_graph(future, depth=depth + 1, limit=limit)
+ if graph is None:
+ return ""
+
+ buf: list[str] = []
+ try:
+ render_level(graph, buf, 0)
+ finally:
+ # 'graph' has references to frames so we should
+ # make sure it's GC'ed as soon as we don't need it.
+ del graph
+ return '\n'.join(buf)
+
+def print_call_graph(
+ future: futures.Future | None = None,
+ /,
+ *,
+ file: io.Writer[str] | None = None,
+ depth: int = 1,
+ limit: int | None = None,
+) -> None:
+ """Print the async call graph for the current task or the provided Future."""
+ print(format_call_graph(future, depth=depth, limit=limit), file=file)
diff --git a/PythonLib/full/asyncio/locks.py b/PythonLib/full/asyncio/locks.py
index ce5d8d5bf..fa3a94764 100644
--- a/PythonLib/full/asyncio/locks.py
+++ b/PythonLib/full/asyncio/locks.py
@@ -24,25 +24,23 @@ class Lock(_ContextManagerMixin, mixins._LoopBoundMixin):
"""Primitive lock objects.
A primitive lock is a synchronization primitive that is not owned
- by a particular coroutine when locked. A primitive lock is in one
+ by a particular task when locked. A primitive lock is in one
of two states, 'locked' or 'unlocked'.
It is created in the unlocked state. It has two basic methods,
acquire() and release(). When the state is unlocked, acquire()
changes the state to locked and returns immediately. When the
state is locked, acquire() blocks until a call to release() in
- another coroutine changes it to unlocked, then the acquire() call
+ another task changes it to unlocked, then the acquire() call
resets it to locked and returns. The release() method should only
be called in the locked state; it changes the state to unlocked
and returns immediately. If an attempt is made to release an
unlocked lock, a RuntimeError will be raised.
- When more than one coroutine is blocked in acquire() waiting for
- the state to turn to unlocked, only one coroutine proceeds when a
- release() call resets the state to unlocked; first coroutine which
- is blocked in acquire() is being processed.
-
- acquire() is a coroutine and should be called with 'await'.
+ When more than one task is blocked in acquire() waiting for
+ the state to turn to unlocked, only one task proceeds when a
+ release() call resets the state to unlocked; successive release()
+ calls will unblock tasks in FIFO order.
Locks also support the asynchronous context management protocol.
'async with lock' statement should be used.
@@ -95,6 +93,8 @@ async def acquire(self):
This method blocks until the lock is unlocked, then sets it to
locked and returns True.
"""
+ # Implement fair scheduling, where thread always waits
+ # its turn. Jumping the queue if all are cancelled is an optimization.
if (not self._locked and (self._waiters is None or
all(w.cancelled() for w in self._waiters))):
self._locked = True
@@ -105,19 +105,22 @@ async def acquire(self):
fut = self._get_loop().create_future()
self._waiters.append(fut)
- # Finally block should be called before the CancelledError
- # handling as we don't want CancelledError to call
- # _wake_up_first() and attempt to wake up itself.
try:
try:
await fut
finally:
self._waiters.remove(fut)
except exceptions.CancelledError:
+ # Currently the only exception designed be able to occur here.
+
+ # Ensure the lock invariant: If lock is not claimed (or about
+ # to be claimed by us) and there is a Task in waiters,
+ # ensure that the Task at the head will run.
if not self._locked:
self._wake_up_first()
raise
+ # assert self._locked is False
self._locked = True
return True
@@ -125,7 +128,7 @@ def release(self):
"""Release a lock.
When the lock is locked, reset it to unlocked, and return.
- If any other coroutines are blocked waiting for the lock to become
+ If any other tasks are blocked waiting for the lock to become
unlocked, allow exactly one of them to proceed.
When invoked on an unlocked lock, a RuntimeError is raised.
@@ -139,7 +142,7 @@ def release(self):
raise RuntimeError('Lock is not acquired.')
def _wake_up_first(self):
- """Wake up the first waiter if it isn't done."""
+ """Ensure that the first waiter will wake up."""
if not self._waiters:
return
try:
@@ -147,9 +150,7 @@ def _wake_up_first(self):
except StopIteration:
return
- # .done() necessarily means that a waiter will wake up later on and
- # either take the lock, or, if it was cancelled and lock wasn't
- # taken already, will hit this again and wake up a new waiter.
+ # .done() means that the waiter is already set to wake up.
if not fut.done():
fut.set_result(True)
@@ -179,8 +180,8 @@ def is_set(self):
return self._value
def set(self):
- """Set the internal flag to true. All coroutines waiting for it to
- become true are awakened. Coroutine that call wait() once the flag is
+ """Set the internal flag to true. All tasks waiting for it to
+ become true are awakened. Tasks that call wait() once the flag is
true will not block at all.
"""
if not self._value:
@@ -191,7 +192,7 @@ def set(self):
fut.set_result(True)
def clear(self):
- """Reset the internal flag to false. Subsequently, coroutines calling
+ """Reset the internal flag to false. Subsequently, tasks calling
wait() will block until set() is called to set the internal flag
to true again."""
self._value = False
@@ -200,7 +201,7 @@ async def wait(self):
"""Block until the internal flag is true.
If the internal flag is true on entry, return True
- immediately. Otherwise, block until another coroutine calls
+ immediately. Otherwise, block until another task calls
set() to set the flag to true, then return True.
"""
if self._value:
@@ -219,8 +220,8 @@ class Condition(_ContextManagerMixin, mixins._LoopBoundMixin):
"""Asynchronous equivalent to threading.Condition.
This class implements condition variable objects. A condition variable
- allows one or more coroutines to wait until they are notified by another
- coroutine.
+ allows one or more tasks to wait until they are notified by another
+ task.
A new Lock object is created and used as the underlying lock.
"""
@@ -247,45 +248,64 @@ def __repr__(self):
async def wait(self):
"""Wait until notified.
- If the calling coroutine has not acquired the lock when this
+ If the calling task has not acquired the lock when this
method is called, a RuntimeError is raised.
This method releases the underlying lock, and then blocks
until it is awakened by a notify() or notify_all() call for
- the same condition variable in another coroutine. Once
+ the same condition variable in another task. Once
awakened, it re-acquires the lock and returns True.
+
+ This method may return spuriously,
+ which is why the caller should always
+ re-check the state and be prepared to wait() again.
"""
if not self.locked():
raise RuntimeError('cannot wait on un-acquired lock')
+ fut = self._get_loop().create_future()
self.release()
try:
- fut = self._get_loop().create_future()
- self._waiters.append(fut)
try:
- await fut
- return True
- finally:
- self._waiters.remove(fut)
-
- finally:
- # Must reacquire lock even if wait is cancelled
- cancelled = False
- while True:
+ self._waiters.append(fut)
try:
- await self.acquire()
- break
- except exceptions.CancelledError:
- cancelled = True
+ await fut
+ return True
+ finally:
+ self._waiters.remove(fut)
- if cancelled:
- raise exceptions.CancelledError
+ finally:
+ # Must re-acquire lock even if wait is cancelled.
+ # We only catch CancelledError here, since we don't want any
+ # other (fatal) errors with the future to cause us to spin.
+ err = None
+ while True:
+ try:
+ await self.acquire()
+ break
+ except exceptions.CancelledError as e:
+ err = e
+
+ if err is not None:
+ try:
+ raise err # Re-raise most recent exception instance.
+ finally:
+ err = None # Break reference cycles.
+ except BaseException:
+ # Any error raised out of here _may_ have occurred after this Task
+ # believed to have been successfully notified.
+ # Make sure to notify another Task instead. This may result
+ # in a "spurious wakeup", which is allowed as part of the
+ # Condition Variable protocol.
+ self._notify(1)
+ raise
async def wait_for(self, predicate):
"""Wait until a predicate becomes true.
- The predicate should be a callable which result will be
- interpreted as a boolean value. The final predicate value is
+ The predicate should be a callable whose result will be
+ interpreted as a boolean value. The method will repeatedly
+ wait() until it evaluates to true. The final predicate value is
the return value.
"""
result = predicate()
@@ -295,20 +315,22 @@ async def wait_for(self, predicate):
return result
def notify(self, n=1):
- """By default, wake up one coroutine waiting on this condition, if any.
- If the calling coroutine has not acquired the lock when this method
+ """By default, wake up one task waiting on this condition, if any.
+ If the calling task has not acquired the lock when this method
is called, a RuntimeError is raised.
- This method wakes up at most n of the coroutines waiting for the
- condition variable; it is a no-op if no coroutines are waiting.
+ This method wakes up n of the tasks waiting for the condition
+ variable; if fewer than n are waiting, they are all awoken.
- Note: an awakened coroutine does not actually return from its
+ Note: an awakened task does not actually return from its
wait() call until it can reacquire the lock. Since notify() does
not release the lock, its caller should.
"""
if not self.locked():
raise RuntimeError('cannot notify on un-acquired lock')
+ self._notify(n)
+ def _notify(self, n):
idx = 0
for fut in self._waiters:
if idx >= n:
@@ -319,9 +341,9 @@ def notify(self, n=1):
fut.set_result(False)
def notify_all(self):
- """Wake up all threads waiting on this condition. This method acts
- like notify(), but wakes up all waiting threads instead of one. If the
- calling thread has not acquired the lock when this method is called,
+ """Wake up all tasks waiting on this condition. This method acts
+ like notify(), but wakes up all waiting tasks instead of one. If the
+ calling task has not acquired the lock when this method is called,
a RuntimeError is raised.
"""
self.notify(len(self._waiters))
@@ -357,6 +379,7 @@ def __repr__(self):
def locked(self):
"""Returns True if semaphore cannot be acquired immediately."""
+ # Due to state, or FIFO rules (must allow others to run first).
return self._value == 0 or (
any(not w.cancelled() for w in (self._waiters or ())))
@@ -365,11 +388,12 @@ async def acquire(self):
If the internal counter is larger than zero on entry,
decrement it by one and return True immediately. If it is
- zero on entry, block, waiting until some other coroutine has
+ zero on entry, block, waiting until some other task has
called release() to make it larger than 0, and then return
True.
"""
if not self.locked():
+ # Maintain FIFO, wait for others to start even if _value > 0.
self._value -= 1
return True
@@ -378,29 +402,34 @@ async def acquire(self):
fut = self._get_loop().create_future()
self._waiters.append(fut)
- # Finally block should be called before the CancelledError
- # handling as we don't want CancelledError to call
- # _wake_up_first() and attempt to wake up itself.
try:
try:
await fut
finally:
self._waiters.remove(fut)
except exceptions.CancelledError:
- if not fut.cancelled():
+ # Currently the only exception designed be able to occur here.
+ if fut.done() and not fut.cancelled():
+ # Our Future was successfully set to True via _wake_up_next(),
+ # but we are not about to successfully acquire(). Therefore we
+ # must undo the bookkeeping already done and attempt to wake
+ # up someone else.
self._value += 1
- self._wake_up_next()
raise
- if self._value > 0:
- self._wake_up_next()
+ finally:
+ # New waiters may have arrived but had to wait due to FIFO.
+ # Wake up as many as are allowed.
+ while self._value > 0:
+ if not self._wake_up_next():
+ break # There was no-one to wake up.
return True
def release(self):
"""Release a semaphore, incrementing the internal counter by one.
- When it was zero on entry and another coroutine is waiting for it to
- become larger than zero again, wake up that coroutine.
+ When it was zero on entry and another task is waiting for it to
+ become larger than zero again, wake up that task.
"""
self._value += 1
self._wake_up_next()
@@ -408,13 +437,15 @@ def release(self):
def _wake_up_next(self):
"""Wake up the first waiter that isn't done."""
if not self._waiters:
- return
+ return False
for fut in self._waiters:
if not fut.done():
self._value -= 1
fut.set_result(True)
- return
+ # `fut` is now `done()` and not `cancelled()`.
+ return True
+ return False
class BoundedSemaphore(Semaphore):
@@ -454,7 +485,7 @@ class Barrier(mixins._LoopBoundMixin):
def __init__(self, parties):
"""Create a barrier, initialised to 'parties' tasks."""
if parties < 1:
- raise ValueError('parties must be > 0')
+ raise ValueError('parties must be >= 1')
self._cond = Condition() # notify all tasks when state changes
diff --git a/PythonLib/full/asyncio/proactor_events.py b/PythonLib/full/asyncio/proactor_events.py
index 23ea29c67..f404273c3 100644
--- a/PythonLib/full/asyncio/proactor_events.py
+++ b/PythonLib/full/asyncio/proactor_events.py
@@ -63,7 +63,7 @@ def __init__(self, loop, sock, protocol, waiter=None,
self._called_connection_lost = False
self._eof_written = False
if self._server is not None:
- self._server._attach()
+ self._server._attach(self)
self._loop.call_soon(self._protocol.connection_made, self)
if waiter is not None:
# only wake up the waiter when connection_made() has been called
@@ -167,7 +167,7 @@ def _call_connection_lost(self, exc):
self._sock = None
server = self._server
if server is not None:
- server._detach()
+ server._detach(self)
self._server = None
self._called_connection_lost = True
@@ -460,6 +460,8 @@ def _pipe_closed(self, fut):
class _ProactorDatagramTransport(_ProactorBasePipeTransport,
transports.DatagramTransport):
max_size = 256 * 1024
+ _header_size = 8
+
def __init__(self, loop, sock, protocol, address=None,
waiter=None, extra=None):
self._address = address
@@ -487,9 +489,6 @@ def sendto(self, data, addr=None):
raise TypeError('data argument must be bytes-like object (%r)',
type(data))
- if not data:
- return
-
if self._address is not None and addr not in (None, self._address):
raise ValueError(
f'Invalid address: must be None or {self._address}')
@@ -502,7 +501,7 @@ def sendto(self, data, addr=None):
# Ensure that what we buffer is immutable.
self._buffer.append((bytes(data), addr))
- self._buffer_size += len(data)
+ self._buffer_size += len(data) + self._header_size
if self._write_fut is None:
# No current write operations are active, kick one off
@@ -529,7 +528,7 @@ def _loop_writing(self, fut=None):
return
data, addr = self._buffer.popleft()
- self._buffer_size -= len(data)
+ self._buffer_size -= len(data) + self._header_size
if self._address is not None:
self._write_fut = self._loop._proactor.send(self._sock,
data)
diff --git a/PythonLib/full/asyncio/queues.py b/PythonLib/full/asyncio/queues.py
index a9656a6df..756216fac 100644
--- a/PythonLib/full/asyncio/queues.py
+++ b/PythonLib/full/asyncio/queues.py
@@ -1,4 +1,11 @@
-__all__ = ('Queue', 'PriorityQueue', 'LifoQueue', 'QueueFull', 'QueueEmpty')
+__all__ = (
+ 'Queue',
+ 'PriorityQueue',
+ 'LifoQueue',
+ 'QueueFull',
+ 'QueueEmpty',
+ 'QueueShutDown',
+)
import collections
import heapq
@@ -18,6 +25,11 @@ class QueueFull(Exception):
pass
+class QueueShutDown(Exception):
+ """Raised when putting on to or getting from a shut-down Queue."""
+ pass
+
+
class Queue(mixins._LoopBoundMixin):
"""A queue, useful for coordinating producer and consumer coroutines.
@@ -25,7 +37,7 @@ class Queue(mixins._LoopBoundMixin):
is an integer greater than 0, then "await put()" will block when the
queue reaches maxsize, until an item is removed by get().
- Unlike the standard library Queue, you can reliably know this Queue's size
+ Unlike queue.Queue, you can reliably know this Queue's size
with qsize(), since your single-threaded asyncio application won't be
interrupted between calling qsize() and doing an operation on the Queue.
"""
@@ -41,6 +53,7 @@ def __init__(self, maxsize=0):
self._finished = locks.Event()
self._finished.set()
self._init(maxsize)
+ self._is_shutdown = False
# These three are overridable in subclasses.
@@ -81,6 +94,8 @@ def _format(self):
result += f' _putters[{len(self._putters)}]'
if self._unfinished_tasks:
result += f' tasks={self._unfinished_tasks}'
+ if self._is_shutdown:
+ result += ' shutdown'
return result
def qsize(self):
@@ -112,8 +127,12 @@ async def put(self, item):
Put an item into the queue. If the queue is full, wait until a free
slot is available before adding item.
+
+ Raises QueueShutDown if the queue has been shut down.
"""
while self.full():
+ if self._is_shutdown:
+ raise QueueShutDown
putter = self._get_loop().create_future()
self._putters.append(putter)
try:
@@ -125,7 +144,7 @@ async def put(self, item):
self._putters.remove(putter)
except ValueError:
# The putter could be removed from self._putters by a
- # previous get_nowait call.
+ # previous get_nowait call or a shutdown call.
pass
if not self.full() and not putter.cancelled():
# We were woken up by get_nowait(), but can't take
@@ -138,7 +157,11 @@ def put_nowait(self, item):
"""Put an item into the queue without blocking.
If no free slot is immediately available, raise QueueFull.
+
+ Raises QueueShutDown if the queue has been shut down.
"""
+ if self._is_shutdown:
+ raise QueueShutDown
if self.full():
raise QueueFull
self._put(item)
@@ -150,8 +173,13 @@ async def get(self):
"""Remove and return an item from the queue.
If queue is empty, wait until an item is available.
+
+ Raises QueueShutDown if the queue has been shut down and is empty, or
+ if the queue has been shut down immediately.
"""
while self.empty():
+ if self._is_shutdown and self.empty():
+ raise QueueShutDown
getter = self._get_loop().create_future()
self._getters.append(getter)
try:
@@ -163,7 +191,7 @@ async def get(self):
self._getters.remove(getter)
except ValueError:
# The getter could be removed from self._getters by a
- # previous put_nowait call.
+ # previous put_nowait call, or a shutdown call.
pass
if not self.empty() and not getter.cancelled():
# We were woken up by put_nowait(), but can't take
@@ -176,8 +204,13 @@ def get_nowait(self):
"""Remove and return an item from the queue.
Return an item if one is immediately available, else raise QueueEmpty.
+
+ Raises QueueShutDown if the queue has been shut down and is empty, or
+ if the queue has been shut down immediately.
"""
if self.empty():
+ if self._is_shutdown:
+ raise QueueShutDown
raise QueueEmpty
item = self._get()
self._wakeup_next(self._putters)
@@ -214,6 +247,36 @@ async def join(self):
if self._unfinished_tasks > 0:
await self._finished.wait()
+ def shutdown(self, immediate=False):
+ """Shut-down the queue, making queue gets and puts raise QueueShutDown.
+
+ By default, gets will only raise once the queue is empty. Set
+ 'immediate' to True to make gets raise immediately instead.
+
+ All blocked callers of put() and get() will be unblocked.
+
+ If 'immediate', the queue is drained and unfinished tasks
+ is reduced by the number of drained tasks. If unfinished tasks
+ is reduced to zero, callers of Queue.join are unblocked.
+ """
+ self._is_shutdown = True
+ if immediate:
+ while not self.empty():
+ self._get()
+ if self._unfinished_tasks > 0:
+ self._unfinished_tasks -= 1
+ if self._unfinished_tasks == 0:
+ self._finished.set()
+ # All getters need to re-check queue-empty to raise ShutDown
+ while self._getters:
+ getter = self._getters.popleft()
+ if not getter.done():
+ getter.set_result(None)
+ while self._putters:
+ putter = self._putters.popleft()
+ if not putter.done():
+ putter.set_result(None)
+
class PriorityQueue(Queue):
"""A subclass of Queue; retrieves entries in priority order (lowest first).
diff --git a/PythonLib/full/asyncio/runners.py b/PythonLib/full/asyncio/runners.py
index 1b8923659..ba37e003a 100644
--- a/PythonLib/full/asyncio/runners.py
+++ b/PythonLib/full/asyncio/runners.py
@@ -3,6 +3,7 @@
import contextvars
import enum
import functools
+import inspect
import threading
import signal
from . import coroutines
@@ -84,10 +85,7 @@ def get_loop(self):
return self._loop
def run(self, coro, *, context=None):
- """Run a coroutine inside the embedded event loop."""
- if not coroutines.iscoroutine(coro):
- raise ValueError("a coroutine was expected, got {!r}".format(coro))
-
+ """Run code in the embedded event loop."""
if events._get_running_loop() is not None:
# fail fast with short traceback
raise RuntimeError(
@@ -95,8 +93,19 @@ def run(self, coro, *, context=None):
self._lazy_init()
+ if not coroutines.iscoroutine(coro):
+ if inspect.isawaitable(coro):
+ async def _wrap_awaitable(awaitable):
+ return await awaitable
+
+ coro = _wrap_awaitable(coro)
+ else:
+ raise TypeError('An asyncio.Future, a coroutine or an '
+ 'awaitable is required')
+
if context is None:
context = self._context
+
task = self._loop.create_task(coro, context=context)
if (threading.current_thread() is threading.main_thread()
@@ -168,6 +177,7 @@ def run(main, *, debug=None, loop_factory=None):
running in the same thread.
If debug is True, the event loop will be run in debug mode.
+ If loop_factory is passed, it is used for new event loop creation.
This function always creates a new event loop and closes it at the end.
It should be used as a main entry point for asyncio programs, and should
diff --git a/PythonLib/full/asyncio/selector_events.py b/PythonLib/full/asyncio/selector_events.py
index 790711f83..ff7e16df3 100644
--- a/PythonLib/full/asyncio/selector_events.py
+++ b/PythonLib/full/asyncio/selector_events.py
@@ -173,16 +173,20 @@ def _accept_connection(
# listening socket has triggered an EVENT_READ. There may be multiple
# connections waiting for an .accept() so it is called in a loop.
# See https://bugs.python.org/issue27906 for more details.
- for _ in range(backlog):
+ for _ in range(backlog + 1):
try:
conn, addr = sock.accept()
if self._debug:
logger.debug("%r got a new connection from %r: %r",
server, addr, conn)
conn.setblocking(False)
- except (BlockingIOError, InterruptedError, ConnectionAbortedError):
- # Early exit because the socket accept buffer is empty.
- return None
+ except ConnectionAbortedError:
+ # Discard connections that were aborted before accept().
+ continue
+ except (BlockingIOError, InterruptedError):
+ # Early exit because of a signal or
+ # the socket accept buffer is empty.
+ return
except OSError as exc:
# There's nowhere to send the error, so just log it.
if exc.errno in (errno.EMFILE, errno.ENFILE,
@@ -265,22 +269,17 @@ def _ensure_fd_no_transport(self, fd):
except (AttributeError, TypeError, ValueError):
# This code matches selectors._fileobj_to_fd function.
raise ValueError(f"Invalid file object: {fd!r}") from None
- try:
- transport = self._transports[fileno]
- except KeyError:
- pass
- else:
- if not transport.is_closing():
- raise RuntimeError(
- f'File descriptor {fd!r} is used by transport '
- f'{transport!r}')
+ transport = self._transports.get(fileno)
+ if transport and not transport.is_closing():
+ raise RuntimeError(
+ f'File descriptor {fd!r} is used by transport '
+ f'{transport!r}')
def _add_reader(self, fd, callback, *args):
self._check_closed()
handle = events.Handle(callback, args, self, None)
- try:
- key = self._selector.get_key(fd)
- except KeyError:
+ key = self._selector.get_map().get(fd)
+ if key is None:
self._selector.register(fd, selectors.EVENT_READ,
(handle, None))
else:
@@ -294,30 +293,27 @@ def _add_reader(self, fd, callback, *args):
def _remove_reader(self, fd):
if self.is_closed():
return False
- try:
- key = self._selector.get_key(fd)
- except KeyError:
+ key = self._selector.get_map().get(fd)
+ if key is None:
return False
+ mask, (reader, writer) = key.events, key.data
+ mask &= ~selectors.EVENT_READ
+ if not mask:
+ self._selector.unregister(fd)
else:
- mask, (reader, writer) = key.events, key.data
- mask &= ~selectors.EVENT_READ
- if not mask:
- self._selector.unregister(fd)
- else:
- self._selector.modify(fd, mask, (None, writer))
+ self._selector.modify(fd, mask, (None, writer))
- if reader is not None:
- reader.cancel()
- return True
- else:
- return False
+ if reader is not None:
+ reader.cancel()
+ return True
+ else:
+ return False
def _add_writer(self, fd, callback, *args):
self._check_closed()
handle = events.Handle(callback, args, self, None)
- try:
- key = self._selector.get_key(fd)
- except KeyError:
+ key = self._selector.get_map().get(fd)
+ if key is None:
self._selector.register(fd, selectors.EVENT_WRITE,
(None, handle))
else:
@@ -332,24 +328,22 @@ def _remove_writer(self, fd):
"""Remove a writer callback."""
if self.is_closed():
return False
- try:
- key = self._selector.get_key(fd)
- except KeyError:
+ key = self._selector.get_map().get(fd)
+ if key is None:
return False
+ mask, (reader, writer) = key.events, key.data
+ # Remove both writer and connector.
+ mask &= ~selectors.EVENT_WRITE
+ if not mask:
+ self._selector.unregister(fd)
else:
- mask, (reader, writer) = key.events, key.data
- # Remove both writer and connector.
- mask &= ~selectors.EVENT_WRITE
- if not mask:
- self._selector.unregister(fd)
- else:
- self._selector.modify(fd, mask, (reader, None))
+ self._selector.modify(fd, mask, (reader, None))
- if writer is not None:
- writer.cancel()
- return True
- else:
- return False
+ if writer is not None:
+ writer.cancel()
+ return True
+ else:
+ return False
def add_reader(self, fd, callback, *args):
"""Add a reader callback."""
@@ -801,7 +795,7 @@ def __init__(self, loop, sock, protocol, extra=None, server=None):
self._paused = False # Set when pause_reading() called
if self._server is not None:
- self._server._attach()
+ self._server._attach(self)
loop._transports[self._sock_fd] = self
def __repr__(self):
@@ -878,6 +872,8 @@ def __del__(self, _warn=warnings.warn):
if self._sock is not None:
_warn(f"unclosed transport {self!r}", ResourceWarning, source=self)
self._sock.close()
+ if self._server is not None:
+ self._server._detach(self)
def _fatal_error(self, exc, message='Fatal error on transport'):
# Should be called from exception handler only.
@@ -916,7 +912,7 @@ def _call_connection_lost(self, exc):
self._loop = None
server = self._server
if server is not None:
- server._detach()
+ server._detach(self)
self._server = None
def get_write_buffer_size(self):
@@ -1054,8 +1050,8 @@ def _read_ready__on_eof(self):
def write(self, data):
if not isinstance(data, (bytes, bytearray, memoryview)):
- raise TypeError(f'data argument must be a bytes-like object, '
- f'not {type(data).__name__!r}')
+ raise TypeError(f'data argument must be a bytes, bytearray, or memoryview '
+ f'object, not {type(data).__name__!r}')
if self._eof:
raise RuntimeError('Cannot call write() after write_eof()')
if self._empty_waiter is not None:
@@ -1178,20 +1174,31 @@ def writelines(self, list_of_data):
raise RuntimeError('unable to writelines; sendfile is in progress')
if not list_of_data:
return
+
+ if self._conn_lost:
+ if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
+ logger.warning('socket.send() raised exception.')
+ self._conn_lost += 1
+ return
+
self._buffer.extend([memoryview(data) for data in list_of_data])
self._write_ready()
# If the entire buffer couldn't be written, register a write handler
if self._buffer:
self._loop._add_writer(self._sock_fd, self._write_ready)
+ self._maybe_pause_protocol()
def can_write_eof(self):
return True
def _call_connection_lost(self, exc):
- super()._call_connection_lost(exc)
- if self._empty_waiter is not None:
- self._empty_waiter.set_exception(
- ConnectionError("Connection is closed by peer"))
+ try:
+ super()._call_connection_lost(exc)
+ finally:
+ self._write_ready = None
+ if self._empty_waiter is not None:
+ self._empty_waiter.set_exception(
+ ConnectionError("Connection is closed by peer"))
def _make_empty_waiter(self):
if self._empty_waiter is not None:
@@ -1206,13 +1213,13 @@ def _reset_empty_waiter(self):
def close(self):
self._read_ready_cb = None
- self._write_ready = None
super().close()
class _SelectorDatagramTransport(_SelectorTransport, transports.DatagramTransport):
_buffer_factory = collections.deque
+ _header_size = 8
def __init__(self, loop, sock, protocol, address=None,
waiter=None, extra=None):
@@ -1251,8 +1258,6 @@ def sendto(self, data, addr=None):
if not isinstance(data, (bytes, bytearray, memoryview)):
raise TypeError(f'data argument must be a bytes-like object, '
f'not {type(data).__name__!r}')
- if not data:
- return
if self._address:
if addr not in (None, self._address):
@@ -1288,13 +1293,13 @@ def sendto(self, data, addr=None):
# Ensure that what we buffer is immutable.
self._buffer.append((bytes(data), addr))
- self._buffer_size += len(data)
+ self._buffer_size += len(data) + self._header_size
self._maybe_pause_protocol()
def _sendto_ready(self):
while self._buffer:
data, addr = self._buffer.popleft()
- self._buffer_size -= len(data)
+ self._buffer_size -= len(data) + self._header_size
try:
if self._extra['peername']:
self._sock.send(data)
@@ -1302,7 +1307,7 @@ def _sendto_ready(self):
self._sock.sendto(data, addr)
except (BlockingIOError, InterruptedError):
self._buffer.appendleft((data, addr)) # Try again later.
- self._buffer_size += len(data)
+ self._buffer_size += len(data) + self._header_size
break
except OSError as exc:
self._protocol.error_received(exc)
diff --git a/PythonLib/full/asyncio/sslproto.py b/PythonLib/full/asyncio/sslproto.py
index 29e72b1fd..74c5f0d5c 100644
--- a/PythonLib/full/asyncio/sslproto.py
+++ b/PythonLib/full/asyncio/sslproto.py
@@ -545,7 +545,7 @@ def _start_handshake(self):
# start handshake timeout count down
self._handshake_timeout_handle = \
self._loop.call_later(self._ssl_handshake_timeout,
- lambda: self._check_handshake_timeout())
+ self._check_handshake_timeout)
self._do_handshake()
@@ -626,7 +626,7 @@ def _start_shutdown(self):
self._set_state(SSLProtocolState.FLUSHING)
self._shutdown_timeout_handle = self._loop.call_later(
self._ssl_shutdown_timeout,
- lambda: self._check_shutdown_timeout()
+ self._check_shutdown_timeout
)
self._do_flush()
@@ -765,7 +765,7 @@ def _do_read__buffered(self):
else:
break
else:
- self._loop.call_soon(lambda: self._do_read())
+ self._loop.call_soon(self._do_read)
except SSLAgainErrors:
pass
if offset > 0:
diff --git a/PythonLib/full/asyncio/staggered.py b/PythonLib/full/asyncio/staggered.py
index 0f4df8855..2ad65d864 100644
--- a/PythonLib/full/asyncio/staggered.py
+++ b/PythonLib/full/asyncio/staggered.py
@@ -8,6 +8,7 @@
from . import exceptions as exceptions_mod
from . import locks
from . import tasks
+from . import futures
async def staggered_race(coro_fns, delay, *, loop=None):
@@ -63,11 +64,32 @@ async def staggered_race(coro_fns, delay, *, loop=None):
"""
# TODO: when we have aiter() and anext(), allow async iterables in coro_fns.
loop = loop or events.get_running_loop()
+ parent_task = tasks.current_task(loop)
enum_coro_fns = enumerate(coro_fns)
winner_result = None
winner_index = None
+ unhandled_exceptions = []
exceptions = []
- running_tasks = []
+ running_tasks = set()
+ on_completed_fut = None
+
+ def task_done(task):
+ running_tasks.discard(task)
+ futures.future_discard_from_awaited_by(task, parent_task)
+ if (
+ on_completed_fut is not None
+ and not on_completed_fut.done()
+ and not running_tasks
+ ):
+ on_completed_fut.set_result(None)
+
+ if task.cancelled():
+ return
+
+ exc = task.exception()
+ if exc is None:
+ return
+ unhandled_exceptions.append(exc)
async def run_one_coro(ok_to_start, previous_failed) -> None:
# in eager tasks this waits for the calling task to append this task
@@ -91,11 +113,12 @@ async def run_one_coro(ok_to_start, previous_failed) -> None:
this_failed = locks.Event()
next_ok_to_start = locks.Event()
next_task = loop.create_task(run_one_coro(next_ok_to_start, this_failed))
- running_tasks.append(next_task)
+ futures.future_add_to_awaited_by(next_task, parent_task)
+ running_tasks.add(next_task)
+ next_task.add_done_callback(task_done)
# next_task has been appended to running_tasks so next_task is ok to
# start.
next_ok_to_start.set()
- assert len(running_tasks) == this_index + 2
# Prepare place to put this coroutine's exceptions if not won
exceptions.append(None)
assert len(exceptions) == this_index + 1
@@ -120,31 +143,37 @@ async def run_one_coro(ok_to_start, previous_failed) -> None:
# up as done() == True, cancelled() == False, exception() ==
# asyncio.CancelledError. This behavior is specified in
# https://bugs.python.org/issue30048
- for i, t in enumerate(running_tasks):
- if i != this_index:
+ current_task = tasks.current_task(loop)
+ for t in running_tasks:
+ if t is not current_task:
t.cancel()
- ok_to_start = locks.Event()
- first_task = loop.create_task(run_one_coro(ok_to_start, None))
- running_tasks.append(first_task)
- # first_task has been appended to running_tasks so first_task is ok to start.
- ok_to_start.set()
+ propagate_cancellation_error = None
try:
- # Wait for a growing list of tasks to all finish: poor man's version of
- # curio's TaskGroup or trio's nursery
- done_count = 0
- while done_count != len(running_tasks):
- done, _ = await tasks.wait(running_tasks)
- done_count = len(done)
+ ok_to_start = locks.Event()
+ first_task = loop.create_task(run_one_coro(ok_to_start, None))
+ futures.future_add_to_awaited_by(first_task, parent_task)
+ running_tasks.add(first_task)
+ first_task.add_done_callback(task_done)
+ # first_task has been appended to running_tasks so first_task is ok to start.
+ ok_to_start.set()
+ propagate_cancellation_error = None
+ # Make sure no tasks are left running if we leave this function
+ while running_tasks:
+ on_completed_fut = loop.create_future()
+ try:
+ await on_completed_fut
+ except exceptions_mod.CancelledError as ex:
+ propagate_cancellation_error = ex
+ for task in running_tasks:
+ task.cancel(*ex.args)
+ on_completed_fut = None
+ if __debug__ and unhandled_exceptions:
# If run_one_coro raises an unhandled exception, it's probably a
# programming error, and I want to see it.
- if __debug__:
- for d in done:
- if d.done() and not d.cancelled() and d.exception():
- raise d.exception()
+ raise ExceptionGroup("staggered race failed", unhandled_exceptions)
+ if propagate_cancellation_error is not None:
+ raise propagate_cancellation_error
return winner_result, winner_index, exceptions
finally:
- del exceptions
- # Make sure no tasks are left running if we leave this function
- for t in running_tasks:
- t.cancel()
+ del exceptions, propagate_cancellation_error, unhandled_exceptions, parent_task
diff --git a/PythonLib/full/asyncio/streams.py b/PythonLib/full/asyncio/streams.py
index f310aa2f3..64aac4cc5 100644
--- a/PythonLib/full/asyncio/streams.py
+++ b/PythonLib/full/asyncio/streams.py
@@ -201,7 +201,6 @@ def __init__(self, stream_reader, client_connected_cb=None, loop=None):
# is established.
self._strong_reader = stream_reader
self._reject_connection = False
- self._stream_writer = None
self._task = None
self._transport = None
self._client_connected_cb = client_connected_cb
@@ -214,10 +213,8 @@ def _stream_reader(self):
return None
return self._stream_reader_wr()
- def _replace_writer(self, writer):
+ def _replace_transport(self, transport):
loop = self._loop
- transport = writer.transport
- self._stream_writer = writer
self._transport = transport
self._over_ssl = transport.get_extra_info('sslcontext') is not None
@@ -239,11 +236,8 @@ def connection_made(self, transport):
reader.set_transport(transport)
self._over_ssl = transport.get_extra_info('sslcontext') is not None
if self._client_connected_cb is not None:
- self._stream_writer = StreamWriter(transport, self,
- reader,
- self._loop)
- res = self._client_connected_cb(reader,
- self._stream_writer)
+ writer = StreamWriter(transport, self, reader, self._loop)
+ res = self._client_connected_cb(reader, writer)
if coroutines.iscoroutine(res):
def callback(task):
if task.cancelled():
@@ -405,9 +399,9 @@ async def start_tls(self, sslcontext, *,
ssl_handshake_timeout=ssl_handshake_timeout,
ssl_shutdown_timeout=ssl_shutdown_timeout)
self._transport = new_transport
- protocol._replace_writer(self)
+ protocol._replace_transport(new_transport)
- def __del__(self):
+ def __del__(self, warnings=warnings):
if not self._transport.is_closing():
if self._loop.is_closed():
warnings.warn("loop is closed", ResourceWarning)
@@ -596,20 +590,34 @@ async def readuntil(self, separator=b'\n'):
If the data cannot be read because of over limit, a
LimitOverrunError exception will be raised, and the data
will be left in the internal buffer, so it can be read again.
+
+ The ``separator`` may also be a tuple of separators. In this
+ case the return value will be the shortest possible that has any
+ separator as the suffix. For the purposes of LimitOverrunError,
+ the shortest possible separator is considered to be the one that
+ matched.
"""
- seplen = len(separator)
- if seplen == 0:
+ if isinstance(separator, tuple):
+ # Makes sure shortest matches wins
+ separator = sorted(separator, key=len)
+ else:
+ separator = [separator]
+ if not separator:
+ raise ValueError('Separator should contain at least one element')
+ min_seplen = len(separator[0])
+ max_seplen = len(separator[-1])
+ if min_seplen == 0:
raise ValueError('Separator should be at least one-byte string')
if self._exception is not None:
raise self._exception
# Consume whole buffer except last bytes, which length is
- # one less than seplen. Let's check corner cases with
- # separator='SEPARATOR':
+ # one less than max_seplen. Let's check corner cases with
+ # separator[-1]='SEPARATOR':
# * we have received almost complete separator (without last
# byte). i.e buffer='some textSEPARATO'. In this case we
- # can safely consume len(separator) - 1 bytes.
+ # can safely consume max_seplen - 1 bytes.
# * last byte of buffer is first byte of separator, i.e.
# buffer='abcdefghijklmnopqrS'. We may safely consume
# everything except that last byte, but this require to
@@ -622,26 +630,35 @@ async def readuntil(self, separator=b'\n'):
# messages :)
# `offset` is the number of bytes from the beginning of the buffer
- # where there is no occurrence of `separator`.
+ # where there is no occurrence of any `separator`.
offset = 0
- # Loop until we find `separator` in the buffer, exceed the buffer size,
+ # Loop until we find a `separator` in the buffer, exceed the buffer size,
# or an EOF has happened.
while True:
buflen = len(self._buffer)
- # Check if we now have enough data in the buffer for `separator` to
- # fit.
- if buflen - offset >= seplen:
- isep = self._buffer.find(separator, offset)
-
- if isep != -1:
- # `separator` is in the buffer. `isep` will be used later
- # to retrieve the data.
+ # Check if we now have enough data in the buffer for shortest
+ # separator to fit.
+ if buflen - offset >= min_seplen:
+ match_start = None
+ match_end = None
+ for sep in separator:
+ isep = self._buffer.find(sep, offset)
+
+ if isep != -1:
+ # `separator` is in the buffer. `match_start` and
+ # `match_end` will be used later to retrieve the
+ # data.
+ end = isep + len(sep)
+ if match_end is None or end < match_end:
+ match_end = end
+ match_start = isep
+ if match_end is not None:
break
# see upper comment for explanation.
- offset = buflen + 1 - seplen
+ offset = max(0, buflen + 1 - max_seplen)
if offset > self._limit:
raise exceptions.LimitOverrunError(
'Separator is not found, and chunk exceed the limit',
@@ -650,7 +667,7 @@ async def readuntil(self, separator=b'\n'):
# Complete message (with full separator) may be present in buffer
# even when EOF flag is set. This may happen when the last chunk
# adds data which makes separator be found. That's why we check for
- # EOF *ater* inspecting the buffer.
+ # EOF *after* inspecting the buffer.
if self._eof:
chunk = bytes(self._buffer)
self._buffer.clear()
@@ -659,12 +676,12 @@ async def readuntil(self, separator=b'\n'):
# _wait_for_data() will resume reading if stream was paused.
await self._wait_for_data('readuntil')
- if isep > self._limit:
+ if match_start > self._limit:
raise exceptions.LimitOverrunError(
- 'Separator is found, but chunk is longer than limit', isep)
+ 'Separator is found, but chunk is longer than limit', match_start)
- chunk = self._buffer[:isep + seplen]
- del self._buffer[:isep + seplen]
+ chunk = self._buffer[:match_end]
+ del self._buffer[:match_end]
self._maybe_resume_transport()
return bytes(chunk)
diff --git a/PythonLib/full/asyncio/taskgroups.py b/PythonLib/full/asyncio/taskgroups.py
index aada3ffa8..00e8f6d5d 100644
--- a/PythonLib/full/asyncio/taskgroups.py
+++ b/PythonLib/full/asyncio/taskgroups.py
@@ -6,6 +6,7 @@
from . import events
from . import exceptions
+from . import futures
from . import tasks
@@ -87,14 +88,10 @@ async def _aexit(self, et, exc):
self._base_error is None):
self._base_error = exc
- propagate_cancellation_error = \
- exc if et is exceptions.CancelledError else None
- if self._parent_cancel_requested:
- # If this flag is set we *must* call uncancel().
- if self._parent_task.uncancel() == 0:
- # If there are no pending cancellations left,
- # don't propagate CancelledError.
- propagate_cancellation_error = None
+ if et is not None and issubclass(et, exceptions.CancelledError):
+ propagate_cancellation_error = exc
+ else:
+ propagate_cancellation_error = None
if et is not None:
if not self._aborting:
@@ -145,10 +142,17 @@ async def _aexit(self, et, exc):
finally:
exc = None
+ if self._parent_cancel_requested:
+ # If this flag is set we *must* call uncancel().
+ if self._parent_task.uncancel() == 0:
+ # If there are no pending cancellations left,
+ # don't propagate CancelledError.
+ propagate_cancellation_error = None
+
# Propagate CancelledError if there is one, except if there
# are other errors -- those have priority.
try:
- if propagate_cancellation_error and not self._errors:
+ if propagate_cancellation_error is not None and not self._errors:
try:
raise propagate_cancellation_error
finally:
@@ -156,10 +160,16 @@ async def _aexit(self, et, exc):
finally:
propagate_cancellation_error = None
- if et is not None and et is not exceptions.CancelledError:
+ if et is not None and not issubclass(et, exceptions.CancelledError):
self._errors.append(exc)
if self._errors:
+ # If the parent task is being cancelled from the outside
+ # of the taskgroup, un-cancel and re-cancel the parent task,
+ # which will keep the cancel count stable.
+ if self._parent_task.cancelling():
+ self._parent_task.uncancel()
+ self._parent_task.cancel()
try:
raise BaseExceptionGroup(
'unhandled errors in a TaskGroup',
@@ -169,31 +179,36 @@ async def _aexit(self, et, exc):
exc = None
- def create_task(self, coro, *, name=None, context=None):
+ def create_task(self, coro, **kwargs):
"""Create a new task in this group and return it.
Similar to `asyncio.create_task`.
"""
if not self._entered:
+ coro.close()
raise RuntimeError(f"TaskGroup {self!r} has not been entered")
if self._exiting and not self._tasks:
+ coro.close()
raise RuntimeError(f"TaskGroup {self!r} is finished")
if self._aborting:
+ coro.close()
raise RuntimeError(f"TaskGroup {self!r} is shutting down")
- if context is None:
- task = self._loop.create_task(coro)
- else:
- task = self._loop.create_task(coro, context=context)
- tasks._set_task_name(task, name)
- # optimization: Immediately call the done callback if the task is
+ task = self._loop.create_task(coro, **kwargs)
+
+ futures.future_add_to_awaited_by(task, self._parent_task)
+
+ # Always schedule the done callback even if the task is
# already done (e.g. if the coro was able to complete eagerly),
- # and skip scheduling a done callback
- if task.done():
- self._on_task_done(task)
- else:
- self._tasks.add(task)
- task.add_done_callback(self._on_task_done)
- return task
+ # otherwise if the task completes with an exception then it will cancel
+ # the current task too early. gh-128550, gh-128588
+ self._tasks.add(task)
+ task.add_done_callback(self._on_task_done)
+ try:
+ return task
+ finally:
+ # gh-128552: prevent a refcycle of
+ # task.exception().__traceback__->TaskGroup.create_task->task
+ del task
# Since Python 3.8 Tasks propagate all exceptions correctly,
# except for KeyboardInterrupt and SystemExit which are
@@ -213,6 +228,8 @@ def _abort(self):
def _on_task_done(self, task):
self._tasks.discard(task)
+ futures.future_discard_from_awaited_by(task, self._parent_task)
+
if self._on_completed_fut is not None and not self._tasks:
if not self._on_completed_fut.done():
self._on_completed_fut.set_result(True)
diff --git a/PythonLib/full/asyncio/tasks.py b/PythonLib/full/asyncio/tasks.py
index 0b22e28d8..fbd5c39a7 100644
--- a/PythonLib/full/asyncio/tasks.py
+++ b/PythonLib/full/asyncio/tasks.py
@@ -15,8 +15,8 @@
import functools
import inspect
import itertools
+import math
import types
-import warnings
import weakref
from types import GenericAlias
@@ -25,6 +25,7 @@
from . import events
from . import exceptions
from . import futures
+from . import queues
from . import timeouts
# Helper to generate new task names
@@ -47,37 +48,9 @@ def all_tasks(loop=None):
# capturing the set of eager tasks first, so if an eager task "graduates"
# to a regular task in another thread, we don't risk missing it.
eager_tasks = list(_eager_tasks)
- # Looping over the WeakSet isn't safe as it can be updated from another
- # thread, therefore we cast it to list prior to filtering. The list cast
- # itself requires iteration, so we repeat it several times ignoring
- # RuntimeErrors (which are not very likely to occur).
- # See issues 34970 and 36607 for details.
- scheduled_tasks = None
- i = 0
- while True:
- try:
- scheduled_tasks = list(_scheduled_tasks)
- except RuntimeError:
- i += 1
- if i >= 1000:
- raise
- else:
- break
- return {t for t in itertools.chain(scheduled_tasks, eager_tasks)
- if futures._get_loop(t) is loop and not t.done()}
-
-def _set_task_name(task, name):
- if name is not None:
- try:
- set_name = task.set_name
- except AttributeError:
- warnings.warn("Task.set_name() was added in Python 3.8, "
- "the method support will be mandatory for third-party "
- "task implementations since 3.13.",
- DeprecationWarning, stacklevel=3)
- else:
- set_name(name)
+ return {t for t in itertools.chain(_scheduled_tasks, eager_tasks)
+ if futures._get_loop(t) is loop and not t.done()}
class Task(futures._PyFuture): # Inherit Python Task implementation
@@ -137,7 +110,7 @@ def __init__(self, coro, *, loop=None, name=None, context=None,
self.__eager_start()
else:
self._loop.call_soon(self.__step, context=self._context)
- _register_task(self)
+ _py_register_task(self)
def __del__(self):
if self._state == futures._PENDING and self._log_destroy_pending:
@@ -267,42 +240,44 @@ def uncancel(self):
"""
if self._num_cancels_requested > 0:
self._num_cancels_requested -= 1
+ if self._num_cancels_requested == 0:
+ self._must_cancel = False
return self._num_cancels_requested
def __eager_start(self):
- prev_task = _swap_current_task(self._loop, self)
+ prev_task = _py_swap_current_task(self._loop, self)
try:
- _register_eager_task(self)
+ _py_register_eager_task(self)
try:
self._context.run(self.__step_run_and_handle_result, None)
finally:
- _unregister_eager_task(self)
+ _py_unregister_eager_task(self)
finally:
try:
- curtask = _swap_current_task(self._loop, prev_task)
+ curtask = _py_swap_current_task(self._loop, prev_task)
assert curtask is self
finally:
if self.done():
self._coro = None
self = None # Needed to break cycles when an exception occurs.
else:
- _register_task(self)
+ _py_register_task(self)
def __step(self, exc=None):
if self.done():
raise exceptions.InvalidStateError(
- f'_step(): already done: {self!r}, {exc!r}')
+ f'__step(): already done: {self!r}, {exc!r}')
if self._must_cancel:
if not isinstance(exc, exceptions.CancelledError):
exc = self._make_cancelled_error()
self._must_cancel = False
self._fut_waiter = None
- _enter_task(self._loop, self)
+ _py_enter_task(self._loop, self)
try:
self.__step_run_and_handle_result(exc)
finally:
- _leave_task(self._loop, self)
+ _py_leave_task(self._loop, self)
self = None # Needed to break cycles when an exception occurs.
def __step_run_and_handle_result(self, exc):
@@ -347,6 +322,7 @@ def __step_run_and_handle_result(self, exc):
self._loop.call_soon(
self.__step, new_exc, context=self._context)
else:
+ futures.future_add_to_awaited_by(result, self)
result._asyncio_future_blocking = False
result.add_done_callback(
self.__wakeup, context=self._context)
@@ -381,6 +357,7 @@ def __step_run_and_handle_result(self, exc):
self = None # Needed to break cycles when an exception occurs.
def __wakeup(self, future):
+ futures.future_discard_from_awaited_by(future, self)
try:
future.result()
except BaseException as exc:
@@ -389,7 +366,7 @@ def __wakeup(self, future):
else:
# Don't pass the value of `future.result()` explicitly,
# as `Future.__iter__` and `Future.__await__` don't need it.
- # If we call `_step(value, None)` instead of `_step()`,
+ # If we call `__step(value, None)` instead of `__step()`,
# Python eval loop would use `.send(value)` method call,
# instead of `__next__()`, which is slower for futures
# that return non-generator iterators from their `__iter__`.
@@ -409,20 +386,13 @@ def __wakeup(self, future):
Task = _CTask = _asyncio.Task
-def create_task(coro, *, name=None, context=None):
+def create_task(coro, **kwargs):
"""Schedule the execution of a coroutine object in a spawn task.
Return a Task object.
"""
loop = events.get_running_loop()
- if context is None:
- # Use legacy API if context is not needed
- task = loop.create_task(coro)
- else:
- task = loop.create_task(coro, context=context)
-
- _set_task_name(task, name)
- return task
+ return loop.create_task(coro, **kwargs)
# wait() and as_completed() similar to those in PEP 3148.
@@ -437,8 +407,6 @@ async def wait(fs, *, timeout=None, return_when=ALL_COMPLETED):
The fs iterable must not be empty.
- Coroutines will be wrapped in Tasks.
-
Returns two sets of Future: (done, pending).
Usage:
@@ -530,6 +498,7 @@ async def _wait(fs, timeout, return_when, loop):
if timeout is not None:
timeout_handle = loop.call_later(timeout, _release_waiter, waiter)
counter = len(fs)
+ cur_task = current_task()
def _on_completion(f):
nonlocal counter
@@ -542,9 +511,11 @@ def _on_completion(f):
timeout_handle.cancel()
if not waiter.done():
waiter.set_result(None)
+ futures.future_discard_from_awaited_by(f, cur_task)
for f in fs:
f.add_done_callback(_on_completion)
+ futures.future_add_to_awaited_by(f, cur_task)
try:
await waiter
@@ -580,62 +551,125 @@ async def _cancel_and_wait(fut):
fut.remove_done_callback(cb)
-# This is *not* a @coroutine! It is just an iterator (yielding Futures).
+class _AsCompletedIterator:
+ """Iterator of awaitables representing tasks of asyncio.as_completed.
+
+ As an asynchronous iterator, iteration yields futures as they finish. As a
+ plain iterator, new coroutines are yielded that will return or raise the
+ result of the next underlying future to complete.
+ """
+ def __init__(self, aws, timeout):
+ self._done = queues.Queue()
+ self._timeout_handle = None
+
+ loop = events.get_event_loop()
+ todo = {ensure_future(aw, loop=loop) for aw in set(aws)}
+ for f in todo:
+ f.add_done_callback(self._handle_completion)
+ if todo and timeout is not None:
+ self._timeout_handle = (
+ loop.call_later(timeout, self._handle_timeout)
+ )
+ self._todo = todo
+ self._todo_left = len(todo)
+
+ def __aiter__(self):
+ return self
+
+ def __iter__(self):
+ return self
+
+ async def __anext__(self):
+ if not self._todo_left:
+ raise StopAsyncIteration
+ assert self._todo_left > 0
+ self._todo_left -= 1
+ return await self._wait_for_one()
+
+ def __next__(self):
+ if not self._todo_left:
+ raise StopIteration
+ assert self._todo_left > 0
+ self._todo_left -= 1
+ return self._wait_for_one(resolve=True)
+
+ def _handle_timeout(self):
+ for f in self._todo:
+ f.remove_done_callback(self._handle_completion)
+ self._done.put_nowait(None) # Sentinel for _wait_for_one().
+ self._todo.clear() # Can't do todo.remove(f) in the loop.
+
+ def _handle_completion(self, f):
+ if not self._todo:
+ return # _handle_timeout() was here first.
+ self._todo.remove(f)
+ self._done.put_nowait(f)
+ if not self._todo and self._timeout_handle is not None:
+ self._timeout_handle.cancel()
+
+ async def _wait_for_one(self, resolve=False):
+ # Wait for the next future to be done and return it unless resolve is
+ # set, in which case return either the result of the future or raise
+ # an exception.
+ f = await self._done.get()
+ if f is None:
+ # Dummy value from _handle_timeout().
+ raise exceptions.TimeoutError
+ return f.result() if resolve else f
+
+
def as_completed(fs, *, timeout=None):
- """Return an iterator whose values are coroutines.
+ """Create an iterator of awaitables or their results in completion order.
- When waiting for the yielded coroutines you'll get the results (or
- exceptions!) of the original Futures (or coroutines), in the order
- in which and as soon as they complete.
+ Run the supplied awaitables concurrently. The returned object can be
+ iterated to obtain the results of the awaitables as they finish.
- This differs from PEP 3148; the proper way to use this is:
+ The object returned can be iterated as an asynchronous iterator or a plain
+ iterator. When asynchronous iteration is used, the originally-supplied
+ awaitables are yielded if they are tasks or futures. This makes it easy to
+ correlate previously-scheduled tasks with their results:
- for f in as_completed(fs):
- result = await f # The 'await' may raise.
- # Use result.
+ ipv4_connect = create_task(open_connection("127.0.0.1", 80))
+ ipv6_connect = create_task(open_connection("::1", 80))
+ tasks = [ipv4_connect, ipv6_connect]
- If a timeout is specified, the 'await' will raise
- TimeoutError when the timeout occurs before all Futures are done.
+ async for earliest_connect in as_completed(tasks):
+ # earliest_connect is done. The result can be obtained by
+ # awaiting it or calling earliest_connect.result()
+ reader, writer = await earliest_connect
- Note: The futures 'f' are not necessarily members of fs.
- """
- if futures.isfuture(fs) or coroutines.iscoroutine(fs):
- raise TypeError(f"expect an iterable of futures, not {type(fs).__name__}")
+ if earliest_connect is ipv6_connect:
+ print("IPv6 connection established.")
+ else:
+ print("IPv4 connection established.")
- from .queues import Queue # Import here to avoid circular import problem.
- done = Queue()
+ During asynchronous iteration, implicitly-created tasks will be yielded for
+ supplied awaitables that aren't tasks or futures.
- loop = events.get_event_loop()
- todo = {ensure_future(f, loop=loop) for f in set(fs)}
- timeout_handle = None
+ When used as a plain iterator, each iteration yields a new coroutine that
+ returns the result or raises the exception of the next completed awaitable.
+ This pattern is compatible with Python versions older than 3.13:
- def _on_timeout():
- for f in todo:
- f.remove_done_callback(_on_completion)
- done.put_nowait(None) # Queue a dummy value for _wait_for_one().
- todo.clear() # Can't do todo.remove(f) in the loop.
+ ipv4_connect = create_task(open_connection("127.0.0.1", 80))
+ ipv6_connect = create_task(open_connection("::1", 80))
+ tasks = [ipv4_connect, ipv6_connect]
- def _on_completion(f):
- if not todo:
- return # _on_timeout() was here first.
- todo.remove(f)
- done.put_nowait(f)
- if not todo and timeout_handle is not None:
- timeout_handle.cancel()
+ for next_connect in as_completed(tasks):
+ # next_connect is not one of the original task objects. It must be
+ # awaited to obtain the result value or raise the exception of the
+ # awaitable that finishes next.
+ reader, writer = await next_connect
- async def _wait_for_one():
- f = await done.get()
- if f is None:
- # Dummy value from _on_timeout().
- raise exceptions.TimeoutError
- return f.result() # May raise f.exception().
+ A TimeoutError is raised if the timeout occurs before all awaitables are
+ done. This is raised by the async for loop during asynchronous iteration or
+ by the coroutines yielded during plain iteration.
+ """
+ if inspect.isawaitable(fs):
+ raise TypeError(
+ f"expects an iterable of awaitables, not {type(fs).__name__}"
+ )
- for f in todo:
- f.add_done_callback(_on_completion)
- if todo and timeout is not None:
- timeout_handle = loop.call_later(timeout, _on_timeout)
- for _ in range(len(todo)):
- yield _wait_for_one()
+ return _AsCompletedIterator(fs, timeout)
@types.coroutine
@@ -656,6 +690,9 @@ async def sleep(delay, result=None):
await __sleep0()
return result
+ if math.isnan(delay):
+ raise ValueError("Invalid delay: NaN (not a number)")
+
loop = events.get_running_loop()
future = loop.create_future()
h = loop.call_later(delay,
@@ -764,10 +801,19 @@ def gather(*coros_or_futures, return_exceptions=False):
outer.set_result([])
return outer
- def _done_callback(fut):
+ loop = events._get_running_loop()
+ if loop is not None:
+ cur_task = current_task(loop)
+ else:
+ cur_task = None
+
+ def _done_callback(fut, cur_task=cur_task):
nonlocal nfinished
nfinished += 1
+ if cur_task is not None:
+ futures.future_discard_from_awaited_by(fut, cur_task)
+
if outer is None or outer.done():
if not fut.cancelled():
# Mark exception retrieved.
@@ -824,7 +870,6 @@ def _done_callback(fut):
nfuts = 0
nfinished = 0
done_futs = []
- loop = None
outer = None # bpo-46672
for arg in coros_or_futures:
if arg not in arg_to_fut:
@@ -837,12 +882,13 @@ def _done_callback(fut):
# can't control it, disable the "destroy pending task"
# warning.
fut._log_destroy_pending = False
-
nfuts += 1
arg_to_fut[arg] = fut
if fut.done():
done_futs.append(fut)
else:
+ if cur_task is not None:
+ futures.future_add_to_awaited_by(fut, cur_task)
fut.add_done_callback(_done_callback)
else:
@@ -862,6 +908,25 @@ def _done_callback(fut):
return outer
+def _log_on_exception(fut):
+ if fut.cancelled():
+ return
+
+ exc = fut.exception()
+ if exc is None:
+ return
+
+ context = {
+ 'message':
+ f'{exc.__class__.__name__} exception in shielded future',
+ 'exception': exc,
+ 'future': fut,
+ }
+ if fut._source_traceback:
+ context['source_traceback'] = fut._source_traceback
+ fut._loop.call_exception_handler(context)
+
+
def shield(arg):
"""Wait for a future, shielding it from cancellation.
@@ -902,11 +967,16 @@ def shield(arg):
loop = futures._get_loop(inner)
outer = loop.create_future()
+ if loop is not None and (cur_task := current_task(loop)) is not None:
+ futures.future_add_to_awaited_by(inner, cur_task)
+ else:
+ cur_task = None
+
+ def _clear_awaited_by_callback(inner):
+ futures.future_discard_from_awaited_by(inner, cur_task)
+
def _inner_done_callback(inner):
if outer.cancelled():
- if not inner.cancelled():
- # Mark inner's result as retrieved.
- inner.exception()
return
if inner.cancelled():
@@ -918,10 +988,16 @@ def _inner_done_callback(inner):
else:
outer.set_result(inner.result())
-
def _outer_done_callback(outer):
if not inner.done():
inner.remove_done_callback(_inner_done_callback)
+ # Keep only one callback to log on cancel
+ inner.remove_done_callback(_log_on_exception)
+ inner.add_done_callback(_log_on_exception)
+
+ if cur_task is not None:
+ inner.add_done_callback(_clear_awaited_by_callback)
+
inner.add_done_callback(_inner_done_callback)
outer.add_done_callback(_outer_done_callback)
@@ -970,9 +1046,9 @@ def create_eager_task_factory(custom_task_constructor):
used. E.g. `loop.set_task_factory(asyncio.eager_task_factory)`.
"""
- def factory(loop, coro, *, name=None, context=None):
+ def factory(loop, coro, *, eager_start=True, **kwargs):
return custom_task_constructor(
- coro, loop=loop, name=name, context=context, eager_start=True)
+ coro, loop=loop, eager_start=eager_start, **kwargs)
return factory
@@ -1044,14 +1120,13 @@ def _unregister_eager_task(task):
_py_enter_task = _enter_task
_py_leave_task = _leave_task
_py_swap_current_task = _swap_current_task
-
+_py_all_tasks = all_tasks
try:
from _asyncio import (_register_task, _register_eager_task,
_unregister_task, _unregister_eager_task,
_enter_task, _leave_task, _swap_current_task,
- _scheduled_tasks, _eager_tasks, _current_tasks,
- current_task)
+ current_task, all_tasks)
except ImportError:
pass
else:
@@ -1063,3 +1138,4 @@ def _unregister_eager_task(task):
_c_enter_task = _enter_task
_c_leave_task = _leave_task
_c_swap_current_task = _swap_current_task
+ _c_all_tasks = all_tasks
diff --git a/PythonLib/full/asyncio/timeouts.py b/PythonLib/full/asyncio/timeouts.py
index 30042abb3..09342dc7c 100644
--- a/PythonLib/full/asyncio/timeouts.py
+++ b/PythonLib/full/asyncio/timeouts.py
@@ -1,7 +1,6 @@
import enum
from types import TracebackType
-from typing import final, Optional, Type
from . import events
from . import exceptions
@@ -23,14 +22,13 @@ class _State(enum.Enum):
EXITED = "finished"
-@final
class Timeout:
"""Asynchronous context manager for cancelling overdue coroutines.
Use `timeout()` or `timeout_at()` rather than instantiating this class directly.
"""
- def __init__(self, when: Optional[float]) -> None:
+ def __init__(self, when: float | None) -> None:
"""Schedule a timeout that will trigger at a given loop time.
- If `when` is `None`, the timeout will never trigger.
@@ -39,15 +37,15 @@ def __init__(self, when: Optional[float]) -> None:
"""
self._state = _State.CREATED
- self._timeout_handler: Optional[events.TimerHandle] = None
- self._task: Optional[tasks.Task] = None
+ self._timeout_handler: events.TimerHandle | None = None
+ self._task: tasks.Task | None = None
self._when = when
- def when(self) -> Optional[float]:
+ def when(self) -> float | None:
"""Return the current deadline."""
return self._when
- def reschedule(self, when: Optional[float]) -> None:
+ def reschedule(self, when: float | None) -> None:
"""Reschedule the timeout."""
if self._state is not _State.ENTERED:
if self._state is _State.CREATED:
@@ -96,10 +94,10 @@ async def __aenter__(self) -> "Timeout":
async def __aexit__(
self,
- exc_type: Optional[Type[BaseException]],
- exc_val: Optional[BaseException],
- exc_tb: Optional[TracebackType],
- ) -> Optional[bool]:
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> bool | None:
assert self._state in (_State.ENTERED, _State.EXPIRING)
if self._timeout_handler is not None:
@@ -109,10 +107,16 @@ async def __aexit__(
if self._state is _State.EXPIRING:
self._state = _State.EXPIRED
- if self._task.uncancel() <= self._cancelling and exc_type is exceptions.CancelledError:
+ if self._task.uncancel() <= self._cancelling and exc_type is not None:
# Since there are no new cancel requests, we're
# handling this.
- raise TimeoutError from exc_val
+ if issubclass(exc_type, exceptions.CancelledError):
+ raise TimeoutError from exc_val
+ elif exc_val is not None:
+ self._insert_timeout_error(exc_val)
+ if isinstance(exc_val, ExceptionGroup):
+ for exc in exc_val.exceptions:
+ self._insert_timeout_error(exc)
elif self._state is _State.ENTERED:
self._state = _State.EXITED
@@ -125,8 +129,18 @@ def _on_timeout(self) -> None:
# drop the reference early
self._timeout_handler = None
+ @staticmethod
+ def _insert_timeout_error(exc_val: BaseException) -> None:
+ while exc_val.__context__ is not None:
+ if isinstance(exc_val.__context__, exceptions.CancelledError):
+ te = TimeoutError()
+ te.__context__ = te.__cause__ = exc_val.__context__
+ exc_val.__context__ = te
+ break
+ exc_val = exc_val.__context__
-def timeout(delay: Optional[float]) -> Timeout:
+
+def timeout(delay: float | None) -> Timeout:
"""Timeout async context manager.
Useful in cases when you want to apply timeout logic around block
@@ -146,7 +160,7 @@ def timeout(delay: Optional[float]) -> Timeout:
return Timeout(loop.time() + delay if delay is not None else None)
-def timeout_at(when: Optional[float]) -> Timeout:
+def timeout_at(when: float | None) -> Timeout:
"""Schedule the timeout at absolute time.
Like timeout() but argument gives absolute time in the same clock system
diff --git a/PythonLib/full/asyncio/tools.py b/PythonLib/full/asyncio/tools.py
new file mode 100644
index 000000000..f39e11fdd
--- /dev/null
+++ b/PythonLib/full/asyncio/tools.py
@@ -0,0 +1,276 @@
+"""Tools to analyze tasks running in asyncio programs."""
+
+from collections import defaultdict, namedtuple
+from itertools import count
+from enum import Enum
+import sys
+from _remote_debugging import RemoteUnwinder, FrameInfo
+
+class NodeType(Enum):
+ COROUTINE = 1
+ TASK = 2
+
+
+class CycleFoundException(Exception):
+ """Raised when there is a cycle when drawing the call tree."""
+ def __init__(
+ self,
+ cycles: list[list[int]],
+ id2name: dict[int, str],
+ ) -> None:
+ super().__init__(cycles, id2name)
+ self.cycles = cycles
+ self.id2name = id2name
+
+
+
+# ─── indexing helpers ───────────────────────────────────────────
+def _format_stack_entry(elem: str|FrameInfo) -> str:
+ if not isinstance(elem, str):
+ if elem.lineno == 0 and elem.filename == "":
+ return f"{elem.funcname}"
+ else:
+ return f"{elem.funcname} {elem.filename}:{elem.lineno}"
+ return elem
+
+
+def _index(result):
+ id2name, awaits, task_stacks = {}, [], {}
+ for awaited_info in result:
+ for task_info in awaited_info.awaited_by:
+ task_id = task_info.task_id
+ task_name = task_info.task_name
+ id2name[task_id] = task_name
+
+ # Store the internal coroutine stack for this task
+ if task_info.coroutine_stack:
+ for coro_info in task_info.coroutine_stack:
+ call_stack = coro_info.call_stack
+ internal_stack = [_format_stack_entry(frame) for frame in call_stack]
+ task_stacks[task_id] = internal_stack
+
+ # Add the awaited_by relationships (external dependencies)
+ if task_info.awaited_by:
+ for coro_info in task_info.awaited_by:
+ call_stack = coro_info.call_stack
+ parent_task_id = coro_info.task_name
+ stack = [_format_stack_entry(frame) for frame in call_stack]
+ awaits.append((parent_task_id, stack, task_id))
+ return id2name, awaits, task_stacks
+
+
+def _build_tree(id2name, awaits, task_stacks):
+ id2label = {(NodeType.TASK, tid): name for tid, name in id2name.items()}
+ children = defaultdict(list)
+ cor_nodes = defaultdict(dict) # Maps parent -> {frame_name: node_key}
+ next_cor_id = count(1)
+
+ def get_or_create_cor_node(parent, frame):
+ """Get existing coroutine node or create new one under parent"""
+ if frame in cor_nodes[parent]:
+ return cor_nodes[parent][frame]
+
+ node_key = (NodeType.COROUTINE, f"c{next(next_cor_id)}")
+ id2label[node_key] = frame
+ children[parent].append(node_key)
+ cor_nodes[parent][frame] = node_key
+ return node_key
+
+ # Build task dependency tree with coroutine frames
+ for parent_id, stack, child_id in awaits:
+ cur = (NodeType.TASK, parent_id)
+ for frame in reversed(stack):
+ cur = get_or_create_cor_node(cur, frame)
+
+ child_key = (NodeType.TASK, child_id)
+ if child_key not in children[cur]:
+ children[cur].append(child_key)
+
+ # Add coroutine stacks for leaf tasks
+ awaiting_tasks = {parent_id for parent_id, _, _ in awaits}
+ for task_id in id2name:
+ if task_id not in awaiting_tasks and task_id in task_stacks:
+ cur = (NodeType.TASK, task_id)
+ for frame in reversed(task_stacks[task_id]):
+ cur = get_or_create_cor_node(cur, frame)
+
+ return id2label, children
+
+
+def _roots(id2label, children):
+ all_children = {c for kids in children.values() for c in kids}
+ return [n for n in id2label if n not in all_children]
+
+# ─── detect cycles in the task-to-task graph ───────────────────────
+def _task_graph(awaits):
+ """Return {parent_task_id: {child_task_id, …}, …}."""
+ g = defaultdict(set)
+ for parent_id, _stack, child_id in awaits:
+ g[parent_id].add(child_id)
+ return g
+
+
+def _find_cycles(graph):
+ """
+ Depth-first search for back-edges.
+
+ Returns a list of cycles (each cycle is a list of task-ids) or an
+ empty list if the graph is acyclic.
+ """
+ WHITE, GREY, BLACK = 0, 1, 2
+ color = defaultdict(lambda: WHITE)
+ path, cycles = [], []
+
+ def dfs(v):
+ color[v] = GREY
+ path.append(v)
+ for w in graph.get(v, ()):
+ if color[w] == WHITE:
+ dfs(w)
+ elif color[w] == GREY: # back-edge → cycle!
+ i = path.index(w)
+ cycles.append(path[i:] + [w]) # make a copy
+ color[v] = BLACK
+ path.pop()
+
+ for v in list(graph):
+ if color[v] == WHITE:
+ dfs(v)
+ return cycles
+
+
+# ─── PRINT TREE FUNCTION ───────────────────────────────────────
+def get_all_awaited_by(pid):
+ unwinder = RemoteUnwinder(pid)
+ return unwinder.get_all_awaited_by()
+
+
+def build_async_tree(result, task_emoji="(T)", cor_emoji=""):
+ """
+ Build a list of strings for pretty-print an async call tree.
+
+ The call tree is produced by `get_all_async_stacks()`, prefixing tasks
+ with `task_emoji` and coroutine frames with `cor_emoji`.
+ """
+ id2name, awaits, task_stacks = _index(result)
+ g = _task_graph(awaits)
+ cycles = _find_cycles(g)
+ if cycles:
+ raise CycleFoundException(cycles, id2name)
+ labels, children = _build_tree(id2name, awaits, task_stacks)
+
+ def pretty(node):
+ flag = task_emoji if node[0] == NodeType.TASK else cor_emoji
+ return f"{flag} {labels[node]}"
+
+ def render(node, prefix="", last=True, buf=None):
+ if buf is None:
+ buf = []
+ buf.append(f"{prefix}{'└── ' if last else '├── '}{pretty(node)}")
+ new_pref = prefix + (" " if last else "│ ")
+ kids = children.get(node, [])
+ for i, kid in enumerate(kids):
+ render(kid, new_pref, i == len(kids) - 1, buf)
+ return buf
+
+ return [render(root) for root in _roots(labels, children)]
+
+
+def build_task_table(result):
+ id2name, _, _ = _index(result)
+ table = []
+
+ for awaited_info in result:
+ thread_id = awaited_info.thread_id
+ for task_info in awaited_info.awaited_by:
+ # Get task info
+ task_id = task_info.task_id
+ task_name = task_info.task_name
+
+ # Build coroutine stack string
+ frames = [frame for coro in task_info.coroutine_stack
+ for frame in coro.call_stack]
+ coro_stack = " -> ".join(_format_stack_entry(x).split(" ")[0]
+ for x in frames)
+
+ # Handle tasks with no awaiters
+ if not task_info.awaited_by:
+ table.append([thread_id, hex(task_id), task_name, coro_stack,
+ "", "", "0x0"])
+ continue
+
+ # Handle tasks with awaiters
+ for coro_info in task_info.awaited_by:
+ parent_id = coro_info.task_name
+ awaiter_frames = [_format_stack_entry(x).split(" ")[0]
+ for x in coro_info.call_stack]
+ awaiter_chain = " -> ".join(awaiter_frames)
+ awaiter_name = id2name.get(parent_id, "Unknown")
+ parent_id_str = (hex(parent_id) if isinstance(parent_id, int)
+ else str(parent_id))
+
+ table.append([thread_id, hex(task_id), task_name, coro_stack,
+ awaiter_chain, awaiter_name, parent_id_str])
+
+ return table
+
+def _print_cycle_exception(exception: CycleFoundException):
+ print("ERROR: await-graph contains cycles - cannot print a tree!", file=sys.stderr)
+ print("", file=sys.stderr)
+ for c in exception.cycles:
+ inames = " → ".join(exception.id2name.get(tid, hex(tid)) for tid in c)
+ print(f"cycle: {inames}", file=sys.stderr)
+
+
+def exit_with_permission_help_text():
+ """
+ Prints a message pointing to platform-specific permission help text and exits the program.
+ This function is called when a PermissionError is encountered while trying
+ to attach to a process.
+ """
+ print(
+ "Error: The specified process cannot be attached to due to insufficient permissions.\n"
+ "See the Python documentation for details on required privileges and troubleshooting:\n"
+ "https://docs.python.org/3.14/howto/remote_debugging.html#permission-requirements\n"
+ )
+ sys.exit(1)
+
+
+def _get_awaited_by_tasks(pid: int) -> list:
+ try:
+ return get_all_awaited_by(pid)
+ except RuntimeError as e:
+ while e.__context__ is not None:
+ e = e.__context__
+ print(f"Error retrieving tasks: {e}")
+ sys.exit(1)
+ except PermissionError as e:
+ exit_with_permission_help_text()
+
+
+def display_awaited_by_tasks_table(pid: int) -> None:
+ """Build and print a table of all pending tasks under `pid`."""
+
+ tasks = _get_awaited_by_tasks(pid)
+ table = build_task_table(tasks)
+ # Print the table in a simple tabular format
+ print(
+ f"{'tid':<10} {'task id':<20} {'task name':<20} {'coroutine stack':<50} {'awaiter chain':<50} {'awaiter name':<15} {'awaiter id':<15}"
+ )
+ print("-" * 180)
+ for row in table:
+ print(f"{row[0]:<10} {row[1]:<20} {row[2]:<20} {row[3]:<50} {row[4]:<50} {row[5]:<15} {row[6]:<15}")
+
+
+def display_awaited_by_tasks_tree(pid: int) -> None:
+ """Build and print a tree of all pending tasks under `pid`."""
+
+ tasks = _get_awaited_by_tasks(pid)
+ try:
+ result = build_async_tree(tasks)
+ except CycleFoundException as e:
+ _print_cycle_exception(e)
+ sys.exit(1)
+
+ for tree in result:
+ print("\n".join(tree))
diff --git a/PythonLib/full/asyncio/transports.py b/PythonLib/full/asyncio/transports.py
index 30fd41d49..34c7ad44f 100644
--- a/PythonLib/full/asyncio/transports.py
+++ b/PythonLib/full/asyncio/transports.py
@@ -181,6 +181,8 @@ def sendto(self, data, addr=None):
to be sent out asynchronously.
addr is target socket address.
If addr is None use target address pointed on transport creation.
+ If data is an empty bytes object a zero-length datagram will be
+ sent.
"""
raise NotImplementedError
diff --git a/PythonLib/full/asyncio/unix_events.py b/PythonLib/full/asyncio/unix_events.py
index f2e920ada..1c1458127 100644
--- a/PythonLib/full/asyncio/unix_events.py
+++ b/PythonLib/full/asyncio/unix_events.py
@@ -28,10 +28,7 @@
__all__ = (
'SelectorEventLoop',
- 'AbstractChildWatcher', 'SafeChildWatcher',
- 'FastChildWatcher', 'PidfdChildWatcher',
- 'MultiLoopChildWatcher', 'ThreadedChildWatcher',
- 'DefaultEventLoopPolicy',
+ 'EventLoop',
)
@@ -63,6 +60,11 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
def __init__(self, selector=None):
super().__init__(selector)
self._signal_handlers = {}
+ self._unix_server_sockets = {}
+ if can_use_pidfd():
+ self._watcher = _PidfdChildWatcher()
+ else:
+ self._watcher = _ThreadedChildWatcher()
def close(self):
super().close()
@@ -92,7 +94,7 @@ def add_signal_handler(self, sig, callback, *args):
Raise RuntimeError if there is a problem setting up the handler.
"""
if (coroutines.iscoroutine(callback) or
- coroutines.iscoroutinefunction(callback)):
+ coroutines._iscoroutinefunction(callback)):
raise TypeError("coroutines cannot be used "
"with add_signal_handler()")
self._check_signal(sig)
@@ -195,33 +197,22 @@ def _make_write_pipe_transport(self, pipe, protocol, waiter=None,
async def _make_subprocess_transport(self, protocol, args, shell,
stdin, stdout, stderr, bufsize,
extra=None, **kwargs):
- with warnings.catch_warnings():
- warnings.simplefilter('ignore', DeprecationWarning)
- watcher = events.get_child_watcher()
-
- with watcher:
- if not watcher.is_active():
- # Check early.
- # Raising exception before process creation
- # prevents subprocess execution if the watcher
- # is not ready to handle it.
- raise RuntimeError("asyncio.get_child_watcher() is not activated, "
- "subprocess support is not installed.")
- waiter = self.create_future()
- transp = _UnixSubprocessTransport(self, protocol, args, shell,
- stdin, stdout, stderr, bufsize,
- waiter=waiter, extra=extra,
- **kwargs)
- watcher.add_child_handler(transp.get_pid(),
- self._child_watcher_callback, transp)
- try:
- await waiter
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException:
- transp.close()
- await transp._wait()
- raise
+ watcher = self._watcher
+ waiter = self.create_future()
+ transp = _UnixSubprocessTransport(self, protocol, args, shell,
+ stdin, stdout, stderr, bufsize,
+ waiter=waiter, extra=extra,
+ **kwargs)
+ watcher.add_child_handler(transp.get_pid(),
+ self._child_watcher_callback, transp)
+ try:
+ await waiter
+ except (SystemExit, KeyboardInterrupt):
+ raise
+ except BaseException:
+ transp.close()
+ await transp._wait()
+ raise
return transp
@@ -283,7 +274,7 @@ async def create_unix_server(
sock=None, backlog=100, ssl=None,
ssl_handshake_timeout=None,
ssl_shutdown_timeout=None,
- start_serving=True):
+ start_serving=True, cleanup_socket=True):
if isinstance(ssl, bool):
raise TypeError('ssl argument must be an SSLContext or None')
@@ -339,6 +330,15 @@ async def create_unix_server(
raise ValueError(
f'A UNIX Domain Stream Socket was expected, got {sock!r}')
+ if cleanup_socket:
+ path = sock.getsockname()
+ # Check for abstract socket. `str` and `bytes` paths are supported.
+ if path[0] not in (0, '\x00'):
+ try:
+ self._unix_server_sockets[sock] = os.stat(path).st_ino
+ except FileNotFoundError:
+ pass
+
sock.setblocking(False)
server = base_events.Server(self, [sock], protocol_factory,
ssl, backlog, ssl_handshake_timeout,
@@ -393,6 +393,9 @@ def _sock_sendfile_native_impl(self, fut, registered_fd, sock, fileno,
fut.set_result(total_sent)
return
+ # On 32-bit architectures truncate to 1GiB to avoid OverflowError
+ blocksize = min(blocksize, sys.maxsize//2 + 1)
+
try:
sent = os.sendfile(fd, fileno, offset, blocksize)
except (BlockingIOError, InterruptedError):
@@ -456,6 +459,27 @@ def cb(fut):
self.remove_writer(fd)
fut.add_done_callback(cb)
+ def _stop_serving(self, sock):
+ # Is this a unix socket that needs cleanup?
+ if sock in self._unix_server_sockets:
+ path = sock.getsockname()
+ else:
+ path = None
+
+ super()._stop_serving(sock)
+
+ if path is not None:
+ prev_ino = self._unix_server_sockets[sock]
+ del self._unix_server_sockets[sock]
+ try:
+ if os.stat(path).st_ino == prev_ino:
+ os.unlink(path)
+ except FileNotFoundError:
+ pass
+ except OSError as err:
+ logger.error('Unable to clean up listening UNIX socket '
+ '%r: %r', path, err)
+
class _UnixReadPipeTransport(transports.ReadTransport):
@@ -830,93 +854,7 @@ def _start(self, args, shell, stdin, stdout, stderr, bufsize, **kwargs):
stdin_w.close()
-class AbstractChildWatcher:
- """Abstract base class for monitoring child processes.
-
- Objects derived from this class monitor a collection of subprocesses and
- report their termination or interruption by a signal.
-
- New callbacks are registered with .add_child_handler(). Starting a new
- process must be done within a 'with' block to allow the watcher to suspend
- its activity until the new process if fully registered (this is needed to
- prevent a race condition in some implementations).
-
- Example:
- with watcher:
- proc = subprocess.Popen("sleep 1")
- watcher.add_child_handler(proc.pid, callback)
-
- Notes:
- Implementations of this class must be thread-safe.
-
- Since child watcher objects may catch the SIGCHLD signal and call
- waitpid(-1), there should be only one active object per process.
- """
-
- def __init_subclass__(cls) -> None:
- if cls.__module__ != __name__:
- warnings._deprecated("AbstractChildWatcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.",
- remove=(3, 14))
-
- def add_child_handler(self, pid, callback, *args):
- """Register a new child handler.
-
- Arrange for callback(pid, returncode, *args) to be called when
- process 'pid' terminates. Specifying another callback for the same
- process replaces the previous handler.
-
- Note: callback() must be thread-safe.
- """
- raise NotImplementedError()
-
- def remove_child_handler(self, pid):
- """Removes the handler for process 'pid'.
-
- The function returns True if the handler was successfully removed,
- False if there was nothing to remove."""
-
- raise NotImplementedError()
-
- def attach_loop(self, loop):
- """Attach the watcher to an event loop.
-
- If the watcher was previously attached to an event loop, then it is
- first detached before attaching to the new loop.
-
- Note: loop may be None.
- """
- raise NotImplementedError()
-
- def close(self):
- """Close the watcher.
-
- This must be called to make sure that any underlying resource is freed.
- """
- raise NotImplementedError()
-
- def is_active(self):
- """Return ``True`` if the watcher is active and is used by the event loop.
-
- Return True if the watcher is installed and ready to handle process exit
- notifications.
-
- """
- raise NotImplementedError()
-
- def __enter__(self):
- """Enter the watcher's context and allow starting new processes
-
- This function must return self"""
- raise NotImplementedError()
-
- def __exit__(self, a, b, c):
- """Exit the watcher's context"""
- raise NotImplementedError()
-
-
-class PidfdChildWatcher(AbstractChildWatcher):
+class _PidfdChildWatcher:
"""Child watcher implementation using Linux's pid file descriptors.
This child watcher polls process file descriptors (pidfds) to await child
@@ -928,21 +866,6 @@ class PidfdChildWatcher(AbstractChildWatcher):
recent (5.3+) kernels.
"""
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_value, exc_traceback):
- pass
-
- def is_active(self):
- return True
-
- def close(self):
- pass
-
- def attach_loop(self, loop):
- pass
-
def add_child_handler(self, pid, callback, *args):
loop = events.get_running_loop()
pidfd = os.pidfd_open(pid)
@@ -967,386 +890,7 @@ def _do_wait(self, pid, pidfd, callback, args):
os.close(pidfd)
callback(pid, returncode, *args)
- def remove_child_handler(self, pid):
- # asyncio never calls remove_child_handler() !!!
- # The method is no-op but is implemented because
- # abstract base classes require it.
- return True
-
-
-class BaseChildWatcher(AbstractChildWatcher):
-
- def __init__(self):
- self._loop = None
- self._callbacks = {}
-
- def close(self):
- self.attach_loop(None)
-
- def is_active(self):
- return self._loop is not None and self._loop.is_running()
-
- def _do_waitpid(self, expected_pid):
- raise NotImplementedError()
-
- def _do_waitpid_all(self):
- raise NotImplementedError()
-
- def attach_loop(self, loop):
- assert loop is None or isinstance(loop, events.AbstractEventLoop)
-
- if self._loop is not None and loop is None and self._callbacks:
- warnings.warn(
- 'A loop is being detached '
- 'from a child watcher with pending handlers',
- RuntimeWarning)
-
- if self._loop is not None:
- self._loop.remove_signal_handler(signal.SIGCHLD)
-
- self._loop = loop
- if loop is not None:
- loop.add_signal_handler(signal.SIGCHLD, self._sig_chld)
-
- # Prevent a race condition in case a child terminated
- # during the switch.
- self._do_waitpid_all()
-
- def _sig_chld(self):
- try:
- self._do_waitpid_all()
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException as exc:
- # self._loop should always be available here
- # as '_sig_chld' is added as a signal handler
- # in 'attach_loop'
- self._loop.call_exception_handler({
- 'message': 'Unknown exception in SIGCHLD handler',
- 'exception': exc,
- })
-
-
-class SafeChildWatcher(BaseChildWatcher):
- """'Safe' child watcher implementation.
-
- This implementation avoids disrupting other code spawning processes by
- polling explicitly each process in the SIGCHLD handler instead of calling
- os.waitpid(-1).
-
- This is a safe solution but it has a significant overhead when handling a
- big number of children (O(n) each time SIGCHLD is raised)
- """
-
- def __init__(self):
- super().__init__()
- warnings._deprecated("SafeChildWatcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.",
- remove=(3, 14))
-
- def close(self):
- self._callbacks.clear()
- super().close()
-
- def __enter__(self):
- return self
-
- def __exit__(self, a, b, c):
- pass
-
- def add_child_handler(self, pid, callback, *args):
- self._callbacks[pid] = (callback, args)
-
- # Prevent a race condition in case the child is already terminated.
- self._do_waitpid(pid)
-
- def remove_child_handler(self, pid):
- try:
- del self._callbacks[pid]
- return True
- except KeyError:
- return False
-
- def _do_waitpid_all(self):
-
- for pid in list(self._callbacks):
- self._do_waitpid(pid)
-
- def _do_waitpid(self, expected_pid):
- assert expected_pid > 0
-
- try:
- pid, status = os.waitpid(expected_pid, os.WNOHANG)
- except ChildProcessError:
- # The child process is already reaped
- # (may happen if waitpid() is called elsewhere).
- pid = expected_pid
- returncode = 255
- logger.warning(
- "Unknown child process pid %d, will report returncode 255",
- pid)
- else:
- if pid == 0:
- # The child process is still alive.
- return
-
- returncode = waitstatus_to_exitcode(status)
- if self._loop.get_debug():
- logger.debug('process %s exited with returncode %s',
- expected_pid, returncode)
-
- try:
- callback, args = self._callbacks.pop(pid)
- except KeyError: # pragma: no cover
- # May happen if .remove_child_handler() is called
- # after os.waitpid() returns.
- if self._loop.get_debug():
- logger.warning("Child watcher got an unexpected pid: %r",
- pid, exc_info=True)
- else:
- callback(pid, returncode, *args)
-
-
-class FastChildWatcher(BaseChildWatcher):
- """'Fast' child watcher implementation.
-
- This implementation reaps every terminated processes by calling
- os.waitpid(-1) directly, possibly breaking other code spawning processes
- and waiting for their termination.
-
- There is no noticeable overhead when handling a big number of children
- (O(1) each time a child terminates).
- """
- def __init__(self):
- super().__init__()
- self._lock = threading.Lock()
- self._zombies = {}
- self._forks = 0
- warnings._deprecated("FastChildWatcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.",
- remove=(3, 14))
-
- def close(self):
- self._callbacks.clear()
- self._zombies.clear()
- super().close()
-
- def __enter__(self):
- with self._lock:
- self._forks += 1
-
- return self
-
- def __exit__(self, a, b, c):
- with self._lock:
- self._forks -= 1
-
- if self._forks or not self._zombies:
- return
-
- collateral_victims = str(self._zombies)
- self._zombies.clear()
-
- logger.warning(
- "Caught subprocesses termination from unknown pids: %s",
- collateral_victims)
-
- def add_child_handler(self, pid, callback, *args):
- assert self._forks, "Must use the context manager"
-
- with self._lock:
- try:
- returncode = self._zombies.pop(pid)
- except KeyError:
- # The child is running.
- self._callbacks[pid] = callback, args
- return
-
- # The child is dead already. We can fire the callback.
- callback(pid, returncode, *args)
-
- def remove_child_handler(self, pid):
- try:
- del self._callbacks[pid]
- return True
- except KeyError:
- return False
-
- def _do_waitpid_all(self):
- # Because of signal coalescing, we must keep calling waitpid() as
- # long as we're able to reap a child.
- while True:
- try:
- pid, status = os.waitpid(-1, os.WNOHANG)
- except ChildProcessError:
- # No more child processes exist.
- return
- else:
- if pid == 0:
- # A child process is still alive.
- return
-
- returncode = waitstatus_to_exitcode(status)
-
- with self._lock:
- try:
- callback, args = self._callbacks.pop(pid)
- except KeyError:
- # unknown child
- if self._forks:
- # It may not be registered yet.
- self._zombies[pid] = returncode
- if self._loop.get_debug():
- logger.debug('unknown process %s exited '
- 'with returncode %s',
- pid, returncode)
- continue
- callback = None
- else:
- if self._loop.get_debug():
- logger.debug('process %s exited with returncode %s',
- pid, returncode)
-
- if callback is None:
- logger.warning(
- "Caught subprocess termination from unknown pid: "
- "%d -> %d", pid, returncode)
- else:
- callback(pid, returncode, *args)
-
-
-class MultiLoopChildWatcher(AbstractChildWatcher):
- """A watcher that doesn't require running loop in the main thread.
-
- This implementation registers a SIGCHLD signal handler on
- instantiation (which may conflict with other code that
- install own handler for this signal).
-
- The solution is safe but it has a significant overhead when
- handling a big number of processes (*O(n)* each time a
- SIGCHLD is received).
- """
-
- # Implementation note:
- # The class keeps compatibility with AbstractChildWatcher ABC
- # To achieve this it has empty attach_loop() method
- # and doesn't accept explicit loop argument
- # for add_child_handler()/remove_child_handler()
- # but retrieves the current loop by get_running_loop()
-
- def __init__(self):
- self._callbacks = {}
- self._saved_sighandler = None
- warnings._deprecated("MultiLoopChildWatcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.",
- remove=(3, 14))
-
- def is_active(self):
- return self._saved_sighandler is not None
-
- def close(self):
- self._callbacks.clear()
- if self._saved_sighandler is None:
- return
-
- handler = signal.getsignal(signal.SIGCHLD)
- if handler != self._sig_chld:
- logger.warning("SIGCHLD handler was changed by outside code")
- else:
- signal.signal(signal.SIGCHLD, self._saved_sighandler)
- self._saved_sighandler = None
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
- def add_child_handler(self, pid, callback, *args):
- loop = events.get_running_loop()
- self._callbacks[pid] = (loop, callback, args)
-
- # Prevent a race condition in case the child is already terminated.
- self._do_waitpid(pid)
-
- def remove_child_handler(self, pid):
- try:
- del self._callbacks[pid]
- return True
- except KeyError:
- return False
-
- def attach_loop(self, loop):
- # Don't save the loop but initialize itself if called first time
- # The reason to do it here is that attach_loop() is called from
- # unix policy only for the main thread.
- # Main thread is required for subscription on SIGCHLD signal
- if self._saved_sighandler is not None:
- return
-
- self._saved_sighandler = signal.signal(signal.SIGCHLD, self._sig_chld)
- if self._saved_sighandler is None:
- logger.warning("Previous SIGCHLD handler was set by non-Python code, "
- "restore to default handler on watcher close.")
- self._saved_sighandler = signal.SIG_DFL
-
- # Set SA_RESTART to limit EINTR occurrences.
- signal.siginterrupt(signal.SIGCHLD, False)
-
- def _do_waitpid_all(self):
- for pid in list(self._callbacks):
- self._do_waitpid(pid)
-
- def _do_waitpid(self, expected_pid):
- assert expected_pid > 0
-
- try:
- pid, status = os.waitpid(expected_pid, os.WNOHANG)
- except ChildProcessError:
- # The child process is already reaped
- # (may happen if waitpid() is called elsewhere).
- pid = expected_pid
- returncode = 255
- logger.warning(
- "Unknown child process pid %d, will report returncode 255",
- pid)
- debug_log = False
- else:
- if pid == 0:
- # The child process is still alive.
- return
-
- returncode = waitstatus_to_exitcode(status)
- debug_log = True
- try:
- loop, callback, args = self._callbacks.pop(pid)
- except KeyError: # pragma: no cover
- # May happen if .remove_child_handler() is called
- # after os.waitpid() returns.
- logger.warning("Child watcher got an unexpected pid: %r",
- pid, exc_info=True)
- else:
- if loop.is_closed():
- logger.warning("Loop %r that handles pid %r is closed", loop, pid)
- else:
- if debug_log and loop.get_debug():
- logger.debug('process %s exited with returncode %s',
- expected_pid, returncode)
- loop.call_soon_threadsafe(callback, pid, returncode, *args)
-
- def _sig_chld(self, signum, frame):
- try:
- self._do_waitpid_all()
- except (SystemExit, KeyboardInterrupt):
- raise
- except BaseException:
- logger.warning('Unknown exception in SIGCHLD handler', exc_info=True)
-
-
-class ThreadedChildWatcher(AbstractChildWatcher):
+class _ThreadedChildWatcher:
"""Threaded child watcher implementation.
The watcher uses a thread per process
@@ -1363,18 +907,6 @@ def __init__(self):
self._pid_counter = itertools.count(0)
self._threads = {}
- def is_active(self):
- return True
-
- def close(self):
- pass
-
- def __enter__(self):
- return self
-
- def __exit__(self, exc_type, exc_val, exc_tb):
- pass
-
def __del__(self, _warn=warnings.warn):
threads = [thread for thread in list(self._threads.values())
if thread.is_alive()]
@@ -1392,15 +924,6 @@ def add_child_handler(self, pid, callback, *args):
self._threads[pid] = thread
thread.start()
- def remove_child_handler(self, pid):
- # asyncio never calls remove_child_handler() !!!
- # The method is no-op but is implemented because
- # abstract base classes require it.
- return True
-
- def attach_loop(self, loop):
- pass
-
def _do_waitpid(self, loop, expected_pid, callback, args):
assert expected_pid > 0
@@ -1439,62 +962,11 @@ def can_use_pidfd():
return True
-class _UnixDefaultEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
- """UNIX event loop policy with a watcher for child processes."""
+class _UnixDefaultEventLoopPolicy(events._BaseDefaultEventLoopPolicy):
+ """UNIX event loop policy"""
_loop_factory = _UnixSelectorEventLoop
- def __init__(self):
- super().__init__()
- self._watcher = None
-
- def _init_watcher(self):
- with events._lock:
- if self._watcher is None: # pragma: no branch
- if can_use_pidfd():
- self._watcher = PidfdChildWatcher()
- else:
- self._watcher = ThreadedChildWatcher()
-
- def set_event_loop(self, loop):
- """Set the event loop.
-
- As a side effect, if a child watcher was set before, then calling
- .set_event_loop() from the main thread will call .attach_loop(loop) on
- the child watcher.
- """
-
- super().set_event_loop(loop)
-
- if (self._watcher is not None and
- threading.current_thread() is threading.main_thread()):
- self._watcher.attach_loop(loop)
-
- def get_child_watcher(self):
- """Get the watcher for child processes.
-
- If not yet set, a ThreadedChildWatcher object is automatically created.
- """
- if self._watcher is None:
- self._init_watcher()
-
- warnings._deprecated("get_child_watcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.", remove=(3, 14))
- return self._watcher
-
- def set_child_watcher(self, watcher):
- """Set the watcher for child processes."""
-
- assert watcher is None or isinstance(watcher, AbstractChildWatcher)
-
- if self._watcher is not None:
- self._watcher.close()
-
- self._watcher = watcher
- warnings._deprecated("set_child_watcher",
- "{name!r} is deprecated as of Python 3.12 and will be "
- "removed in Python {remove}.", remove=(3, 14))
-
SelectorEventLoop = _UnixSelectorEventLoop
-DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy
+_DefaultEventLoopPolicy = _UnixDefaultEventLoopPolicy
+EventLoop = SelectorEventLoop
diff --git a/PythonLib/full/asyncio/windows_events.py b/PythonLib/full/asyncio/windows_events.py
index cb613451a..5f75b17d8 100644
--- a/PythonLib/full/asyncio/windows_events.py
+++ b/PythonLib/full/asyncio/windows_events.py
@@ -29,8 +29,8 @@
__all__ = (
'SelectorEventLoop', 'ProactorEventLoop', 'IocpProactor',
- 'DefaultEventLoopPolicy', 'WindowsSelectorEventLoopPolicy',
- 'WindowsProactorEventLoopPolicy',
+ '_DefaultEventLoopPolicy', '_WindowsSelectorEventLoopPolicy',
+ '_WindowsProactorEventLoopPolicy', 'EventLoop',
)
@@ -315,24 +315,25 @@ def __init__(self, proactor=None):
proactor = IocpProactor()
super().__init__(proactor)
- def run_forever(self):
- try:
- assert self._self_reading_future is None
- self.call_soon(self._loop_self_reading)
- super().run_forever()
- finally:
- if self._self_reading_future is not None:
- ov = self._self_reading_future._ov
- self._self_reading_future.cancel()
- # self_reading_future always uses IOCP, so even though it's
- # been cancelled, we need to make sure that the IOCP message
- # is received so that the kernel is not holding on to the
- # memory, possibly causing memory corruption later. Only
- # unregister it if IO is complete in all respects. Otherwise
- # we need another _poll() later to complete the IO.
- if ov is not None and not ov.pending:
- self._proactor._unregister(ov)
- self._self_reading_future = None
+ def _run_forever_setup(self):
+ assert self._self_reading_future is None
+ self.call_soon(self._loop_self_reading)
+ super()._run_forever_setup()
+
+ def _run_forever_cleanup(self):
+ super()._run_forever_cleanup()
+ if self._self_reading_future is not None:
+ ov = self._self_reading_future._ov
+ self._self_reading_future.cancel()
+ # self_reading_future always uses IOCP, so even though it's
+ # been cancelled, we need to make sure that the IOCP message
+ # is received so that the kernel is not holding on to the
+ # memory, possibly causing memory corruption later. Only
+ # unregister it if IO is complete in all respects. Otherwise
+ # we need another _poll() later to complete the IO.
+ if ov is not None and not ov.pending:
+ self._proactor._unregister(ov)
+ self._self_reading_future = None
async def create_pipe_connection(self, protocol_factory, address):
f = self._proactor.connect_pipe(address)
@@ -890,12 +891,13 @@ def callback(f):
SelectorEventLoop = _WindowsSelectorEventLoop
-class WindowsSelectorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+class _WindowsSelectorEventLoopPolicy(events._BaseDefaultEventLoopPolicy):
_loop_factory = SelectorEventLoop
-class WindowsProactorEventLoopPolicy(events.BaseDefaultEventLoopPolicy):
+class _WindowsProactorEventLoopPolicy(events._BaseDefaultEventLoopPolicy):
_loop_factory = ProactorEventLoop
-DefaultEventLoopPolicy = WindowsProactorEventLoopPolicy
+_DefaultEventLoopPolicy = _WindowsProactorEventLoopPolicy
+EventLoop = ProactorEventLoop
diff --git a/PythonLib/full/asyncio/windows_utils.py b/PythonLib/full/asyncio/windows_utils.py
index ef277fac3..d6393f0b1 100644
--- a/PythonLib/full/asyncio/windows_utils.py
+++ b/PythonLib/full/asyncio/windows_utils.py
@@ -10,7 +10,6 @@
import msvcrt
import os
import subprocess
-import tempfile
import warnings
@@ -24,6 +23,7 @@
PIPE = subprocess.PIPE
STDOUT = subprocess.STDOUT
_mmap_counter = itertools.count()
+_MAX_PIPE_ATTEMPTS = 20
# Replacement for os.pipe() using handles instead of fds
@@ -31,10 +31,6 @@
def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
"""Like os.pipe() but with overlapped support and using handles not fds."""
- address = tempfile.mktemp(
- prefix=r'\\.\pipe\python-pipe-{:d}-{:d}-'.format(
- os.getpid(), next(_mmap_counter)))
-
if duplex:
openmode = _winapi.PIPE_ACCESS_DUPLEX
access = _winapi.GENERIC_READ | _winapi.GENERIC_WRITE
@@ -56,9 +52,20 @@ def pipe(*, duplex=False, overlapped=(True, True), bufsize=BUFSIZE):
h1 = h2 = None
try:
- h1 = _winapi.CreateNamedPipe(
- address, openmode, _winapi.PIPE_WAIT,
- 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
+ for attempts in itertools.count():
+ address = r'\\.\pipe\python-pipe-{:d}-{:d}-{}'.format(
+ os.getpid(), next(_mmap_counter), os.urandom(8).hex())
+ try:
+ h1 = _winapi.CreateNamedPipe(
+ address, openmode, _winapi.PIPE_WAIT,
+ 1, obsize, ibsize, _winapi.NMPWAIT_WAIT_FOREVER, _winapi.NULL)
+ break
+ except OSError as e:
+ if attempts >= _MAX_PIPE_ATTEMPTS:
+ raise
+ if e.winerror not in (_winapi.ERROR_PIPE_BUSY,
+ _winapi.ERROR_ACCESS_DENIED):
+ raise
h2 = _winapi.CreateFile(
address, access, 0, _winapi.NULL, _winapi.OPEN_EXISTING,
@@ -104,8 +111,9 @@ def fileno(self):
def close(self, *, CloseHandle=_winapi.CloseHandle):
if self._handle is not None:
- CloseHandle(self._handle)
+ handle = self._handle
self._handle = None
+ CloseHandle(handle)
def __del__(self, _warn=warnings.warn):
if self._handle is not None:
diff --git a/PythonLib/full/base64.py b/PythonLib/full/base64.py
index 846767a3d..f95132a42 100644
--- a/PythonLib/full/base64.py
+++ b/PythonLib/full/base64.py
@@ -1,12 +1,9 @@
-#! /usr/bin/env python3
-
"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings"""
# Modified 04-Oct-1995 by Jack Jansen to use binascii module
# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support
# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere
-import re
import struct
import binascii
@@ -18,7 +15,7 @@
'b64encode', 'b64decode', 'b32encode', 'b32decode',
'b32hexencode', 'b32hexdecode', 'b16encode', 'b16decode',
# Base85 and Ascii85 encodings
- 'b85encode', 'b85decode', 'a85encode', 'a85decode',
+ 'b85encode', 'b85decode', 'a85encode', 'a85decode', 'z85encode', 'z85decode',
# Standard Base64 encoding
'standard_b64encode', 'standard_b64decode',
# Some common Base64 alternatives. As referenced by RFC 3458, see thread
@@ -164,7 +161,6 @@ def urlsafe_b64decode(s):
_b32rev = {}
def _b32encode(alphabet, s):
- global _b32tab2
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32tab2:
@@ -200,7 +196,6 @@ def _b32encode(alphabet, s):
return bytes(encoded)
def _b32decode(alphabet, s, casefold=False, map01=None):
- global _b32rev
# Delay the initialization of the table to not waste memory
# if the function is never called
if alphabet not in _b32rev:
@@ -288,7 +283,7 @@ def b16decode(s, casefold=False):
s = _bytes_from_decode_data(s)
if casefold:
s = s.upper()
- if re.search(b'[^0-9A-F]', s):
+ if s.translate(None, delete=b'0123456789ABCDEF'):
raise binascii.Error('Non-base16 digit found')
return binascii.unhexlify(s)
@@ -467,9 +462,12 @@ def b85decode(b):
# Delay the initialization of tables to not waste memory
# if the function is never called
if _b85dec is None:
- _b85dec = [None] * 256
+ # we don't assign to _b85dec directly to avoid issues when
+ # multiple threads call this function simultaneously
+ b85dec_tmp = [None] * 256
for i, c in enumerate(_b85alphabet):
- _b85dec[c] = i
+ b85dec_tmp[c] = i
+ _b85dec = b85dec_tmp
b = _bytes_from_decode_data(b)
padding = (-len(b)) % 5
@@ -499,6 +497,33 @@ def b85decode(b):
result = result[:-padding]
return result
+_z85alphabet = (b'0123456789abcdefghijklmnopqrstuvwxyz'
+ b'ABCDEFGHIJKLMNOPQRSTUVWXYZ.-:+=^!/*?&<>()[]{}@%$#')
+# Translating b85 valid but z85 invalid chars to b'\x00' is required
+# to prevent them from being decoded as b85 valid chars.
+_z85_b85_decode_diff = b';_`|~'
+_z85_decode_translation = bytes.maketrans(
+ _z85alphabet + _z85_b85_decode_diff,
+ _b85alphabet + b'\x00' * len(_z85_b85_decode_diff)
+)
+_z85_encode_translation = bytes.maketrans(_b85alphabet, _z85alphabet)
+
+def z85encode(s):
+ """Encode bytes-like object b in z85 format and return a bytes object."""
+ return b85encode(s).translate(_z85_encode_translation)
+
+def z85decode(s):
+ """Decode the z85-encoded bytes-like object or ASCII string b
+
+ The result is returned as a bytes object.
+ """
+ s = _bytes_from_decode_data(s)
+ s = s.translate(_z85_decode_translation)
+ try:
+ return b85decode(s)
+ except ValueError as e:
+ raise ValueError(e.args[0].replace('base85', 'z85')) from None
+
# Legacy interface. This code could be cleaned up since I don't believe
# binascii has any line length limitations. It just doesn't seem worth it
# though. The files should be opened in binary mode.
@@ -579,7 +604,14 @@ def main():
with open(args[0], 'rb') as f:
func(f, sys.stdout.buffer)
else:
- func(sys.stdin.buffer, sys.stdout.buffer)
+ if sys.stdin.isatty():
+ # gh-138775: read terminal input data all at once to detect EOF
+ import io
+ data = sys.stdin.buffer.read()
+ buffer = io.BytesIO(data)
+ else:
+ buffer = sys.stdin.buffer
+ func(buffer, sys.stdout.buffer)
if __name__ == '__main__':
diff --git a/PythonLib/full/bdb.py b/PythonLib/full/bdb.py
index 196e6b178..79da4bab9 100644
--- a/PythonLib/full/bdb.py
+++ b/PythonLib/full/bdb.py
@@ -2,7 +2,10 @@
import fnmatch
import sys
+import threading
import os
+import weakref
+from contextlib import contextmanager
from inspect import CO_GENERATOR, CO_COROUTINE, CO_ASYNC_GENERATOR
__all__ = ["BdbQuit", "Bdb", "Breakpoint"]
@@ -14,6 +17,166 @@ class BdbQuit(Exception):
"""Exception to give up completely."""
+E = sys.monitoring.events
+
+class _MonitoringTracer:
+ EVENT_CALLBACK_MAP = {
+ E.PY_START: 'call',
+ E.PY_RESUME: 'call',
+ E.PY_THROW: 'call',
+ E.LINE: 'line',
+ E.JUMP: 'jump',
+ E.PY_RETURN: 'return',
+ E.PY_YIELD: 'return',
+ E.PY_UNWIND: 'unwind',
+ E.RAISE: 'exception',
+ E.STOP_ITERATION: 'exception',
+ E.INSTRUCTION: 'opcode',
+ }
+
+ GLOBAL_EVENTS = E.PY_START | E.PY_RESUME | E.PY_THROW | E.PY_UNWIND | E.RAISE
+ LOCAL_EVENTS = E.LINE | E.JUMP | E.PY_RETURN | E.PY_YIELD | E.STOP_ITERATION
+
+ def __init__(self):
+ self._tool_id = sys.monitoring.DEBUGGER_ID
+ self._name = 'bdbtracer'
+ self._tracefunc = None
+ self._disable_current_event = False
+ self._tracing_thread = None
+ self._enabled = False
+
+ def start_trace(self, tracefunc):
+ self._tracefunc = tracefunc
+ self._tracing_thread = threading.current_thread()
+ curr_tool = sys.monitoring.get_tool(self._tool_id)
+ if curr_tool is None:
+ sys.monitoring.use_tool_id(self._tool_id, self._name)
+ elif curr_tool == self._name:
+ sys.monitoring.clear_tool_id(self._tool_id)
+ else:
+ raise ValueError('Another debugger is using the monitoring tool')
+ E = sys.monitoring.events
+ all_events = 0
+ for event, cb_name in self.EVENT_CALLBACK_MAP.items():
+ callback = self.callback_wrapper(getattr(self, f'{cb_name}_callback'), event)
+ sys.monitoring.register_callback(self._tool_id, event, callback)
+ if event != E.INSTRUCTION:
+ all_events |= event
+ self.update_local_events()
+ sys.monitoring.set_events(self._tool_id, self.GLOBAL_EVENTS)
+ self._enabled = True
+
+ def stop_trace(self):
+ self._enabled = False
+ self._tracing_thread = None
+ curr_tool = sys.monitoring.get_tool(self._tool_id)
+ if curr_tool != self._name:
+ return
+ sys.monitoring.clear_tool_id(self._tool_id)
+ sys.monitoring.free_tool_id(self._tool_id)
+
+ def disable_current_event(self):
+ self._disable_current_event = True
+
+ def restart_events(self):
+ if sys.monitoring.get_tool(self._tool_id) == self._name:
+ sys.monitoring.restart_events()
+
+ def callback_wrapper(self, func, event):
+ import functools
+
+ @functools.wraps(func)
+ def wrapper(*args):
+ if self._tracing_thread != threading.current_thread():
+ return
+ try:
+ frame = sys._getframe().f_back
+ ret = func(frame, *args)
+ if self._enabled and frame.f_trace:
+ self.update_local_events()
+ if (
+ self._disable_current_event
+ and event not in (E.PY_THROW, E.PY_UNWIND, E.RAISE)
+ ):
+ return sys.monitoring.DISABLE
+ else:
+ return ret
+ except BaseException:
+ self.stop_trace()
+ sys._getframe().f_back.f_trace = None
+ raise
+ finally:
+ self._disable_current_event = False
+
+ return wrapper
+
+ def call_callback(self, frame, code, *args):
+ local_tracefunc = self._tracefunc(frame, 'call', None)
+ if local_tracefunc is not None:
+ frame.f_trace = local_tracefunc
+ if self._enabled:
+ sys.monitoring.set_local_events(self._tool_id, code, self.LOCAL_EVENTS)
+
+ def return_callback(self, frame, code, offset, retval):
+ if frame.f_trace:
+ frame.f_trace(frame, 'return', retval)
+
+ def unwind_callback(self, frame, code, *args):
+ if frame.f_trace:
+ frame.f_trace(frame, 'return', None)
+
+ def line_callback(self, frame, code, *args):
+ if frame.f_trace and frame.f_trace_lines:
+ frame.f_trace(frame, 'line', None)
+
+ def jump_callback(self, frame, code, inst_offset, dest_offset):
+ if dest_offset > inst_offset:
+ return sys.monitoring.DISABLE
+ inst_lineno = self._get_lineno(code, inst_offset)
+ dest_lineno = self._get_lineno(code, dest_offset)
+ if inst_lineno != dest_lineno:
+ return sys.monitoring.DISABLE
+ if frame.f_trace and frame.f_trace_lines:
+ frame.f_trace(frame, 'line', None)
+
+ def exception_callback(self, frame, code, offset, exc):
+ if frame.f_trace:
+ if exc.__traceback__ and hasattr(exc.__traceback__, 'tb_frame'):
+ tb = exc.__traceback__
+ while tb:
+ if tb.tb_frame.f_locals.get('self') is self:
+ return
+ tb = tb.tb_next
+ frame.f_trace(frame, 'exception', (type(exc), exc, exc.__traceback__))
+
+ def opcode_callback(self, frame, code, offset):
+ if frame.f_trace and frame.f_trace_opcodes:
+ frame.f_trace(frame, 'opcode', None)
+
+ def update_local_events(self, frame=None):
+ if sys.monitoring.get_tool(self._tool_id) != self._name:
+ return
+ if frame is None:
+ frame = sys._getframe().f_back
+ while frame is not None:
+ if frame.f_trace is not None:
+ if frame.f_trace_opcodes:
+ events = self.LOCAL_EVENTS | E.INSTRUCTION
+ else:
+ events = self.LOCAL_EVENTS
+ sys.monitoring.set_local_events(self._tool_id, frame.f_code, events)
+ frame = frame.f_back
+
+ def _get_lineno(self, code, offset):
+ import dis
+ last_lineno = None
+ for start, lineno in dis.findlinestarts(code):
+ if offset < start:
+ return last_lineno
+ last_lineno = lineno
+ return last_lineno
+
+
class Bdb:
"""Generic Python debugger base class.
@@ -28,11 +191,24 @@ class Bdb:
is determined by the __name__ in the frame globals.
"""
- def __init__(self, skip=None):
+ def __init__(self, skip=None, backend='settrace'):
self.skip = set(skip) if skip else None
self.breaks = {}
self.fncache = {}
+ self.frame_trace_lines_opcodes = {}
self.frame_returning = None
+ self.trace_opcodes = False
+ self.enterframe = None
+ self.cmdframe = None
+ self.cmdlineno = None
+ self.code_linenos = weakref.WeakKeyDictionary()
+ self.backend = backend
+ if backend == 'monitoring':
+ self.monitoring_tracer = _MonitoringTracer()
+ elif backend == 'settrace':
+ self.monitoring_tracer = None
+ else:
+ raise ValueError(f"Invalid backend '{backend}'")
self._load_breaks()
@@ -53,6 +229,18 @@ def canonic(self, filename):
self.fncache[filename] = canonic
return canonic
+ def start_trace(self):
+ if self.monitoring_tracer:
+ self.monitoring_tracer.start_trace(self.trace_dispatch)
+ else:
+ sys.settrace(self.trace_dispatch)
+
+ def stop_trace(self):
+ if self.monitoring_tracer:
+ self.monitoring_tracer.stop_trace()
+ else:
+ sys.settrace(None)
+
def reset(self):
"""Set values of attributes as ready to start debugging."""
import linecache
@@ -60,6 +248,12 @@ def reset(self):
self.botframe = None
self._set_stopinfo(None, None)
+ @contextmanager
+ def set_enterframe(self, frame):
+ self.enterframe = frame
+ yield
+ self.enterframe = None
+
def trace_dispatch(self, frame, event, arg):
"""Dispatch a trace function for debugged frames based on the event.
@@ -84,24 +278,28 @@ def trace_dispatch(self, frame, event, arg):
The arg parameter depends on the previous event.
"""
- if self.quitting:
- return # None
- if event == 'line':
- return self.dispatch_line(frame)
- if event == 'call':
- return self.dispatch_call(frame, arg)
- if event == 'return':
- return self.dispatch_return(frame, arg)
- if event == 'exception':
- return self.dispatch_exception(frame, arg)
- if event == 'c_call':
- return self.trace_dispatch
- if event == 'c_exception':
- return self.trace_dispatch
- if event == 'c_return':
+
+ with self.set_enterframe(frame):
+ if self.quitting:
+ return # None
+ if event == 'line':
+ return self.dispatch_line(frame)
+ if event == 'call':
+ return self.dispatch_call(frame, arg)
+ if event == 'return':
+ return self.dispatch_return(frame, arg)
+ if event == 'exception':
+ return self.dispatch_exception(frame, arg)
+ if event == 'c_call':
+ return self.trace_dispatch
+ if event == 'c_exception':
+ return self.trace_dispatch
+ if event == 'c_return':
+ return self.trace_dispatch
+ if event == 'opcode':
+ return self.dispatch_opcode(frame, arg)
+ print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))
return self.trace_dispatch
- print('bdb.Bdb.dispatch: unknown debugging event:', repr(event))
- return self.trace_dispatch
def dispatch_line(self, frame):
"""Invoke user function and return trace function for line event.
@@ -110,9 +308,17 @@ def dispatch_line(self, frame):
self.user_line(). Raise BdbQuit if self.quitting is set.
Return self.trace_dispatch to continue tracing in this scope.
"""
- if self.stop_here(frame) or self.break_here(frame):
+ # GH-136057
+ # For line events, we don't want to stop at the same line where
+ # the latest next/step command was issued.
+ if (self.stop_here(frame) or self.break_here(frame)) and not (
+ self.cmdframe == frame and self.cmdlineno == frame.f_lineno
+ ):
self.user_line(frame)
+ self.restart_events()
if self.quitting: raise BdbQuit
+ elif not self.get_break(frame.f_code.co_filename, frame.f_lineno):
+ self.disable_current_event()
return self.trace_dispatch
def dispatch_call(self, frame, arg):
@@ -128,12 +334,18 @@ def dispatch_call(self, frame, arg):
self.botframe = frame.f_back # (CT) Note that this may also be None!
return self.trace_dispatch
if not (self.stop_here(frame) or self.break_anywhere(frame)):
- # No need to trace this function
+ # We already know there's no breakpoint in this function
+ # If it's a next/until/return command, we don't need any CALL event
+ # and we don't need to set the f_trace on any new frame.
+ # If it's a step command, it must either hit stop_here, or skip the
+ # whole module. Either way, we don't need the CALL event here.
+ self.disable_current_event()
return # None
# Ignore call events in generator except when stepping.
if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS:
return self.trace_dispatch
self.user_call(frame, arg)
+ self.restart_events()
if self.quitting: raise BdbQuit
return self.trace_dispatch
@@ -147,10 +359,14 @@ def dispatch_return(self, frame, arg):
if self.stop_here(frame) or frame == self.returnframe:
# Ignore return events in generator except when stepping.
if self.stopframe and frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS:
+ # It's possible to trigger a StopIteration exception in
+ # the caller so we must set the trace function in the caller
+ self._set_caller_tracefunc(frame)
return self.trace_dispatch
try:
self.frame_returning = frame
self.user_return(frame, arg)
+ self.restart_events()
finally:
self.frame_returning = None
if self.quitting: raise BdbQuit
@@ -178,6 +394,7 @@ def dispatch_exception(self, frame, arg):
if not (frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS
and arg[0] is StopIteration and arg[2] is None):
self.user_exception(frame, arg)
+ self.restart_events()
if self.quitting: raise BdbQuit
# Stop at the StopIteration or GeneratorExit exception when the user
# has set stopframe in a generator by issuing a return command, or a
@@ -187,10 +404,26 @@ def dispatch_exception(self, frame, arg):
and self.stopframe.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS
and arg[0] in (StopIteration, GeneratorExit)):
self.user_exception(frame, arg)
+ self.restart_events()
if self.quitting: raise BdbQuit
return self.trace_dispatch
+ def dispatch_opcode(self, frame, arg):
+ """Invoke user function and return trace function for opcode event.
+ If the debugger stops on the current opcode, invoke
+ self.user_opcode(). Raise BdbQuit if self.quitting is set.
+ Return self.trace_dispatch to continue tracing in this scope.
+
+ Opcode event will always trigger the user callback. For now the only
+ opcode event is from an inline set_trace() and we want to stop there
+ unconditionally.
+ """
+ self.user_opcode(frame)
+ self.restart_events()
+ if self.quitting: raise BdbQuit
+ return self.trace_dispatch
+
# Normally derived classes don't override the following
# methods, but they may if they want to redefine the
# definition of stopping and breakpoints.
@@ -254,9 +487,25 @@ def do_clear(self, arg):
raise NotImplementedError("subclass of bdb must implement do_clear()")
def break_anywhere(self, frame):
- """Return True if there is any breakpoint for frame's filename.
+ """Return True if there is any breakpoint in that frame
+ """
+ filename = self.canonic(frame.f_code.co_filename)
+ if filename not in self.breaks:
+ return False
+ for lineno in self.breaks[filename]:
+ if self._lineno_in_frame(lineno, frame):
+ return True
+ return False
+
+ def _lineno_in_frame(self, lineno, frame):
+ """Return True if the line number is in the frame's code object.
"""
- return self.canonic(frame.f_code.co_filename) in self.breaks
+ code = frame.f_code
+ if lineno < code.co_firstlineno:
+ return False
+ if code not in self.code_linenos:
+ self.code_linenos[code] = set(lineno for _, _, lineno in code.co_lines())
+ return lineno in self.code_linenos[code]
# Derived classes should override the user_* methods
# to gain control.
@@ -277,7 +526,24 @@ def user_exception(self, frame, exc_info):
"""Called when we stop on an exception."""
pass
- def _set_stopinfo(self, stopframe, returnframe, stoplineno=0):
+ def user_opcode(self, frame):
+ """Called when we are about to execute an opcode."""
+ pass
+
+ def _set_trace_opcodes(self, trace_opcodes):
+ if trace_opcodes != self.trace_opcodes:
+ self.trace_opcodes = trace_opcodes
+ frame = self.enterframe
+ while frame is not None:
+ frame.f_trace_opcodes = trace_opcodes
+ if frame is self.botframe:
+ break
+ frame = frame.f_back
+ if self.monitoring_tracer:
+ self.monitoring_tracer.update_local_events()
+
+ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0, opcode=False,
+ cmdframe=None, cmdlineno=None):
"""Set the attributes for stopping.
If stoplineno is greater than or equal to 0, then stop at line
@@ -290,6 +556,11 @@ def _set_stopinfo(self, stopframe, returnframe, stoplineno=0):
# stoplineno >= 0 means: stop at line >= the stoplineno
# stoplineno -1 means: don't stop at all
self.stoplineno = stoplineno
+ # cmdframe/cmdlineno is the frame/line number when the user issued
+ # step/next commands.
+ self.cmdframe = cmdframe
+ self.cmdlineno = cmdlineno
+ self._set_trace_opcodes(opcode)
def _set_caller_tracefunc(self, current_frame):
# Issue #13183: pdb skips frames after hitting a breakpoint and running
@@ -314,16 +585,22 @@ def set_until(self, frame, lineno=None):
def set_step(self):
"""Stop after one line of code."""
- self._set_stopinfo(None, None)
+ # set_step() could be called from signal handler so enterframe might be None
+ self._set_stopinfo(None, None, cmdframe=self.enterframe,
+ cmdlineno=getattr(self.enterframe, 'f_lineno', None))
+
+ def set_stepinstr(self):
+ """Stop before the next instruction."""
+ self._set_stopinfo(None, None, opcode=True)
def set_next(self, frame):
"""Stop on the next line in or below the given frame."""
- self._set_stopinfo(frame, None)
+ self._set_stopinfo(frame, None, cmdframe=frame, cmdlineno=frame.f_lineno)
def set_return(self, frame):
"""Stop when returning from the given frame."""
if frame.f_code.co_flags & GENERATOR_AND_COROUTINE_FLAGS:
- self._set_stopinfo(frame, None, -1)
+ self._set_stopinfo(frame, frame, -1)
else:
self._set_stopinfo(frame.f_back, frame)
@@ -332,15 +609,21 @@ def set_trace(self, frame=None):
If frame is not specified, debugging starts from caller's frame.
"""
+ self.stop_trace()
if frame is None:
frame = sys._getframe().f_back
self.reset()
- while frame:
- frame.f_trace = self.trace_dispatch
- self.botframe = frame
- frame = frame.f_back
- self.set_step()
- sys.settrace(self.trace_dispatch)
+ with self.set_enterframe(frame):
+ while frame:
+ frame.f_trace = self.trace_dispatch
+ self.botframe = frame
+ self.frame_trace_lines_opcodes[frame] = (frame.f_trace_lines, frame.f_trace_opcodes)
+ # We need f_trace_lines == True for the debugger to work
+ frame.f_trace_lines = True
+ frame = frame.f_back
+ self.set_stepinstr()
+ self.enterframe = None
+ self.start_trace()
def set_continue(self):
"""Stop only at breakpoints or when finished.
@@ -351,11 +634,16 @@ def set_continue(self):
self._set_stopinfo(self.botframe, None, -1)
if not self.breaks:
# no breakpoints; run without debugger overhead
- sys.settrace(None)
+ self.stop_trace()
frame = sys._getframe().f_back
while frame and frame is not self.botframe:
del frame.f_trace
frame = frame.f_back
+ for frame, (trace_lines, trace_opcodes) in self.frame_trace_lines_opcodes.items():
+ frame.f_trace_lines, frame.f_trace_opcodes = trace_lines, trace_opcodes
+ if self.backend == 'monitoring':
+ self.monitoring_tracer.update_local_events()
+ self.frame_trace_lines_opcodes = {}
def set_quit(self):
"""Set quitting attribute to True.
@@ -365,7 +653,7 @@ def set_quit(self):
self.stopframe = self.botframe
self.returnframe = None
self.quitting = True
- sys.settrace(None)
+ self.stop_trace()
# Derived classes and clients can call the following methods
# to manipulate breakpoints. These methods return an
@@ -394,6 +682,14 @@ def set_break(self, filename, lineno, temporary=False, cond=None,
return 'Line %s:%d does not exist' % (filename, lineno)
self._add_to_breaks(filename, lineno)
bp = Breakpoint(filename, lineno, temporary, cond, funcname)
+ # After we set a new breakpoint, we need to search through all frames
+ # and set f_trace to trace_dispatch if there could be a breakpoint in
+ # that frame.
+ frame = self.enterframe
+ while frame:
+ if self.break_anywhere(frame):
+ frame.f_trace = self.trace_dispatch
+ frame = frame.f_back
return None
def _load_breaks(self):
@@ -585,6 +881,16 @@ def format_stack_entry(self, frame_lineno, lprefix=': '):
s += f'{lprefix}Warning: lineno is None'
return s
+ def disable_current_event(self):
+ """Disable the current event."""
+ if self.backend == 'monitoring':
+ self.monitoring_tracer.disable_current_event()
+
+ def restart_events(self):
+ """Restart all events."""
+ if self.backend == 'monitoring':
+ self.monitoring_tracer.restart_events()
+
# The following methods can be called by clients to use
# a debugger to debug a statement or an expression.
# Both can be given as a string, or a code object.
@@ -602,14 +908,14 @@ def run(self, cmd, globals=None, locals=None):
self.reset()
if isinstance(cmd, str):
cmd = compile(cmd, "", "exec")
- sys.settrace(self.trace_dispatch)
+ self.start_trace()
try:
exec(cmd, globals, locals)
except BdbQuit:
pass
finally:
self.quitting = True
- sys.settrace(None)
+ self.stop_trace()
def runeval(self, expr, globals=None, locals=None):
"""Debug an expression executed via the eval() function.
@@ -622,14 +928,14 @@ def runeval(self, expr, globals=None, locals=None):
if locals is None:
locals = globals
self.reset()
- sys.settrace(self.trace_dispatch)
+ self.start_trace()
try:
return eval(expr, globals, locals)
except BdbQuit:
pass
finally:
self.quitting = True
- sys.settrace(None)
+ self.stop_trace()
def runctx(self, cmd, globals, locals):
"""For backwards-compatibility. Defers to run()."""
@@ -644,7 +950,7 @@ def runcall(self, func, /, *args, **kwds):
Return the result of the function call.
"""
self.reset()
- sys.settrace(self.trace_dispatch)
+ self.start_trace()
res = None
try:
res = func(*args, **kwds)
@@ -652,7 +958,7 @@ def runcall(self, func, /, *args, **kwds):
pass
finally:
self.quitting = True
- sys.settrace(None)
+ self.stop_trace()
return res
diff --git a/PythonLib/full/bz2.py b/PythonLib/full/bz2.py
index fabe4f73c..eb58f4da5 100644
--- a/PythonLib/full/bz2.py
+++ b/PythonLib/full/bz2.py
@@ -10,20 +10,20 @@
__author__ = "Nadeem Vawda "
from builtins import open as _builtin_open
+from compression._common import _streams
import io
import os
-import _compression
from _bz2 import BZ2Compressor, BZ2Decompressor
-_MODE_CLOSED = 0
+# Value 0 no longer used
_MODE_READ = 1
# Value 2 no longer used
_MODE_WRITE = 3
-class BZ2File(_compression.BaseStream):
+class BZ2File(_streams.BaseStream):
"""A file object providing transparent bzip2 (de)compression.
@@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9):
"""
self._fp = None
self._closefp = False
- self._mode = _MODE_CLOSED
+ self._mode = None
if not (1 <= compresslevel <= 9):
raise ValueError("compresslevel must be between 1 and 9")
@@ -88,7 +88,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9):
raise TypeError("filename must be a str, bytes, file or PathLike object")
if self._mode == _MODE_READ:
- raw = _compression.DecompressReader(self._fp,
+ raw = _streams.DecompressReader(self._fp,
BZ2Decompressor, trailing_error=OSError)
self._buffer = io.BufferedReader(raw)
else:
@@ -100,7 +100,7 @@ def close(self):
May be called more than once without error. Once the file is
closed, any other operation on it will raise a ValueError.
"""
- if self._mode == _MODE_CLOSED:
+ if self.closed:
return
try:
if self._mode == _MODE_READ:
@@ -115,13 +115,21 @@ def close(self):
finally:
self._fp = None
self._closefp = False
- self._mode = _MODE_CLOSED
self._buffer = None
@property
def closed(self):
"""True if this file is closed."""
- return self._mode == _MODE_CLOSED
+ return self._fp is None
+
+ @property
+ def name(self):
+ self._check_not_closed()
+ return self._fp.name
+
+ @property
+ def mode(self):
+ return 'wb' if self._mode == _MODE_WRITE else 'rb'
def fileno(self):
"""Return the file descriptor for the underlying file."""
@@ -240,7 +248,7 @@ def writelines(self, seq):
Line separators are not added between the written byte strings.
"""
- return _compression.BaseStream.writelines(self, seq)
+ return _streams.BaseStream.writelines(self, seq)
def seek(self, offset, whence=io.SEEK_SET):
"""Change the file position.
diff --git a/PythonLib/full/cProfile.py b/PythonLib/full/cProfile.py
index 135a12c39..770d26f79 100644
--- a/PythonLib/full/cProfile.py
+++ b/PythonLib/full/cProfile.py
@@ -1,5 +1,3 @@
-#! /usr/bin/env python3
-
"""Python interface for the 'lsprof' profiler.
Compatible with the 'profile' module.
"""
@@ -8,6 +6,7 @@
import _lsprof
import importlib.machinery
+import importlib.util
import io
import profile as _pyprofile
@@ -41,7 +40,9 @@ class Profile(_lsprof.Profiler):
def print_stats(self, sort=-1):
import pstats
- pstats.Stats(self).strip_dirs().sort_stats(sort).print_stats()
+ if not isinstance(sort, tuple):
+ sort = (sort,)
+ pstats.Stats(self).strip_dirs().sort_stats(*sort).print_stats()
def dump_stats(self, file):
import marshal
@@ -173,13 +174,22 @@ def main():
code = compile(fp.read(), progname, 'exec')
spec = importlib.machinery.ModuleSpec(name='__main__', loader=None,
origin=progname)
- globs = {
+ module = importlib.util.module_from_spec(spec)
+ # Set __main__ so that importing __main__ in the profiled code will
+ # return the same namespace that the code is executing under.
+ sys.modules['__main__'] = module
+ # Ensure that we're using the same __dict__ instance as the module
+ # for the global variables so that updates to globals are reflected
+ # in the module's namespace.
+ globs = module.__dict__
+ globs.update({
'__spec__': spec,
'__file__': spec.origin,
'__name__': spec.name,
'__package__': None,
'__cached__': None,
- }
+ })
+
try:
runctx(code, globs, None, options.outfile, options.sort)
except BrokenPipeError as exc:
diff --git a/PythonLib/full/calendar.py b/PythonLib/full/calendar.py
index 350964843..18f76d52f 100644
--- a/PythonLib/full/calendar.py
+++ b/PythonLib/full/calendar.py
@@ -10,7 +10,6 @@
from enum import IntEnum, global_enum
import locale as _locale
from itertools import repeat
-import warnings
__all__ = ["IllegalMonthError", "IllegalWeekdayError", "setfirstweekday",
"firstweekday", "isleap", "leapdays", "weekday", "monthrange",
@@ -46,6 +45,7 @@ def __str__(self):
def __getattr__(name):
if name in ('January', 'February'):
+ import warnings
warnings.warn(f"The '{name}' attribute is deprecated, use '{name.upper()}' instead",
DeprecationWarning, stacklevel=2)
if name == 'January':
@@ -428,6 +428,7 @@ def formatyear(self, theyear, w=2, l=1, c=6, m=3):
headers = (header for k in months)
a(formatstring(headers, colwidth, c).rstrip())
a('\n'*l)
+
# max number of weeks for this row
height = max(len(cal) for cal in row)
for j in range(height):
@@ -593,8 +594,6 @@ def __enter__(self):
_locale.setlocale(_locale.LC_TIME, self.locale)
def __exit__(self, *args):
- if self.oldlocale is None:
- return
_locale.setlocale(_locale.LC_TIME, self.oldlocale)
@@ -648,6 +647,117 @@ def formatmonthname(self, theyear, themonth, withyear=True):
with different_locale(self.locale):
return super().formatmonthname(theyear, themonth, withyear)
+
+class _CLIDemoCalendar(TextCalendar):
+ def __init__(self, highlight_day=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.highlight_day = highlight_day
+
+ def formatweek(self, theweek, width, *, highlight_day=None):
+ """
+ Returns a single week in a string (no newline).
+ """
+ if highlight_day:
+ from _colorize import get_colors
+
+ ansi = get_colors()
+ highlight = f"{ansi.BLACK}{ansi.BACKGROUND_YELLOW}"
+ reset = ansi.RESET
+ else:
+ highlight = reset = ""
+
+ return ' '.join(
+ (
+ f"{highlight}{self.formatday(d, wd, width)}{reset}"
+ if d == highlight_day
+ else self.formatday(d, wd, width)
+ )
+ for (d, wd) in theweek
+ )
+
+ def formatmonth(self, theyear, themonth, w=0, l=0):
+ """
+ Return a month's calendar string (multi-line).
+ """
+ if (
+ self.highlight_day
+ and self.highlight_day.year == theyear
+ and self.highlight_day.month == themonth
+ ):
+ highlight_day = self.highlight_day.day
+ else:
+ highlight_day = None
+ w = max(2, w)
+ l = max(1, l)
+ s = self.formatmonthname(theyear, themonth, 7 * (w + 1) - 1)
+ s = s.rstrip()
+ s += '\n' * l
+ s += self.formatweekheader(w).rstrip()
+ s += '\n' * l
+ for week in self.monthdays2calendar(theyear, themonth):
+ s += self.formatweek(week, w, highlight_day=highlight_day).rstrip()
+ s += '\n' * l
+ return s
+
+ def formatyear(self, theyear, w=2, l=1, c=6, m=3):
+ """
+ Returns a year's calendar as a multi-line string.
+ """
+ w = max(2, w)
+ l = max(1, l)
+ c = max(2, c)
+ colwidth = (w + 1) * 7 - 1
+ v = []
+ a = v.append
+ a(repr(theyear).center(colwidth*m+c*(m-1)).rstrip())
+ a('\n'*l)
+ header = self.formatweekheader(w)
+ for (i, row) in enumerate(self.yeardays2calendar(theyear, m)):
+ # months in this row
+ months = range(m*i+1, min(m*(i+1)+1, 13))
+ a('\n'*l)
+ names = (self.formatmonthname(theyear, k, colwidth, False)
+ for k in months)
+ a(formatstring(names, colwidth, c).rstrip())
+ a('\n'*l)
+ headers = (header for k in months)
+ a(formatstring(headers, colwidth, c).rstrip())
+ a('\n'*l)
+
+ if (
+ self.highlight_day
+ and self.highlight_day.year == theyear
+ and self.highlight_day.month in months
+ ):
+ month_pos = months.index(self.highlight_day.month)
+ else:
+ month_pos = None
+
+ # max number of weeks for this row
+ height = max(len(cal) for cal in row)
+ for j in range(height):
+ weeks = []
+ for k, cal in enumerate(row):
+ if j >= len(cal):
+ weeks.append('')
+ else:
+ day = (
+ self.highlight_day.day if k == month_pos else None
+ )
+ weeks.append(
+ self.formatweek(cal[j], w, highlight_day=day)
+ )
+ a(formatstring(weeks, colwidth, c).rstrip())
+ a('\n' * l)
+ return ''.join(v)
+
+
+class _CLIDemoLocaleCalendar(LocaleTextCalendar, _CLIDemoCalendar):
+ def __init__(self, highlight_day=None, *args, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.highlight_day = highlight_day
+
+
# Support for old module level interface
c = TextCalendar()
@@ -698,9 +808,9 @@ def timegm(tuple):
return seconds
-def main(args):
+def main(args=None):
import argparse
- parser = argparse.ArgumentParser()
+ parser = argparse.ArgumentParser(color=True)
textgroup = parser.add_argument_group('text only arguments')
htmlgroup = parser.add_argument_group('html only arguments')
textgroup.add_argument(
@@ -744,6 +854,11 @@ def main(args):
choices=("text", "html"),
help="output type (text or html)"
)
+ parser.add_argument(
+ "-f", "--first-weekday",
+ type=int, default=0,
+ help="weekday (0 is Monday, 6 is Sunday) to start each week (default 0)"
+ )
parser.add_argument(
"year",
nargs='?', type=int,
@@ -755,44 +870,47 @@ def main(args):
help="month number (1-12, text only)"
)
- options = parser.parse_args(args[1:])
+ options = parser.parse_args(args)
if options.locale and not options.encoding:
parser.error("if --locale is specified --encoding is required")
sys.exit(1)
locale = options.locale, options.encoding
+ today = datetime.date.today()
if options.type == "html":
+ if options.month:
+ parser.error("incorrect number of arguments")
+ sys.exit(1)
if options.locale:
cal = LocaleHTMLCalendar(locale=locale)
else:
cal = HTMLCalendar()
+ cal.setfirstweekday(options.first_weekday)
encoding = options.encoding
if encoding is None:
encoding = sys.getdefaultencoding()
optdict = dict(encoding=encoding, css=options.css)
write = sys.stdout.buffer.write
if options.year is None:
- write(cal.formatyearpage(datetime.date.today().year, **optdict))
- elif options.month is None:
- write(cal.formatyearpage(options.year, **optdict))
+ write(cal.formatyearpage(today.year, **optdict))
else:
- parser.error("incorrect number of arguments")
- sys.exit(1)
+ write(cal.formatyearpage(options.year, **optdict))
else:
if options.locale:
- cal = LocaleTextCalendar(locale=locale)
+ cal = _CLIDemoLocaleCalendar(highlight_day=today, locale=locale)
else:
- cal = TextCalendar()
+ cal = _CLIDemoCalendar(highlight_day=today)
+ cal.setfirstweekday(options.first_weekday)
optdict = dict(w=options.width, l=options.lines)
if options.month is None:
optdict["c"] = options.spacing
optdict["m"] = options.months
- if options.month is not None:
+ else:
_validate_month(options.month)
if options.year is None:
- result = cal.formatyear(datetime.date.today().year, **optdict)
+ result = cal.formatyear(today.year, **optdict)
elif options.month is None:
result = cal.formatyear(options.year, **optdict)
else:
@@ -805,4 +923,4 @@ def main(args):
if __name__ == "__main__":
- main(sys.argv)
+ main()
diff --git a/PythonLib/full/cgi.py b/PythonLib/full/cgi.py
deleted file mode 100644
index 8787567be..000000000
--- a/PythonLib/full/cgi.py
+++ /dev/null
@@ -1,1012 +0,0 @@
-#! /usr/local/bin/python
-
-# NOTE: the above "/usr/local/bin/python" is NOT a mistake. It is
-# intentionally NOT "/usr/bin/env python". On many systems
-# (e.g. Solaris), /usr/local/bin is not in $PATH as passed to CGI
-# scripts, and /usr/local/bin is the default directory where Python is
-# installed, so /usr/bin/env would be unable to find python. Granted,
-# binary installations by Linux vendors often install Python in
-# /usr/bin. So let those vendors patch cgi.py to match their choice
-# of installation.
-
-"""Support module for CGI (Common Gateway Interface) scripts.
-
-This module defines a number of utilities for use by CGI scripts
-written in Python.
-
-The global variable maxlen can be set to an integer indicating the maximum size
-of a POST request. POST requests larger than this size will result in a
-ValueError being raised during parsing. The default value of this variable is 0,
-meaning the request size is unlimited.
-"""
-
-# History
-# -------
-#
-# Michael McLay started this module. Steve Majewski changed the
-# interface to SvFormContentDict and FormContentDict. The multipart
-# parsing was inspired by code submitted by Andreas Paepcke. Guido van
-# Rossum rewrote, reformatted and documented the module and is currently
-# responsible for its maintenance.
-#
-
-__version__ = "2.6"
-
-
-# Imports
-# =======
-
-from io import StringIO, BytesIO, TextIOWrapper
-from collections.abc import Mapping
-import sys
-import os
-import urllib.parse
-from email.parser import FeedParser
-from email.message import Message
-import html
-import locale
-import tempfile
-import warnings
-
-__all__ = ["MiniFieldStorage", "FieldStorage", "parse", "parse_multipart",
- "parse_header", "test", "print_exception", "print_environ",
- "print_form", "print_directory", "print_arguments",
- "print_environ_usage"]
-
-
-warnings._deprecated(__name__, remove=(3,13))
-
-# Logging support
-# ===============
-
-logfile = "" # Filename to log to, if not empty
-logfp = None # File object to log to, if not None
-
-def initlog(*allargs):
- """Write a log message, if there is a log file.
-
- Even though this function is called initlog(), you should always
- use log(); log is a variable that is set either to initlog
- (initially), to dolog (once the log file has been opened), or to
- nolog (when logging is disabled).
-
- The first argument is a format string; the remaining arguments (if
- any) are arguments to the % operator, so e.g.
- log("%s: %s", "a", "b")
- will write "a: b" to the log file, followed by a newline.
-
- If the global logfp is not None, it should be a file object to
- which log data is written.
-
- If the global logfp is None, the global logfile may be a string
- giving a filename to open, in append mode. This file should be
- world writable!!! If the file can't be opened, logging is
- silently disabled (since there is no safe place where we could
- send an error message).
-
- """
- global log, logfile, logfp
- warnings.warn("cgi.log() is deprecated as of 3.10. Use logging instead",
- DeprecationWarning, stacklevel=2)
- if logfile and not logfp:
- try:
- logfp = open(logfile, "a", encoding="locale")
- except OSError:
- pass
- if not logfp:
- log = nolog
- else:
- log = dolog
- log(*allargs)
-
-def dolog(fmt, *args):
- """Write a log message to the log file. See initlog() for docs."""
- logfp.write(fmt%args + "\n")
-
-def nolog(*allargs):
- """Dummy function, assigned to log when logging is disabled."""
- pass
-
-def closelog():
- """Close the log file."""
- global log, logfile, logfp
- logfile = ''
- if logfp:
- logfp.close()
- logfp = None
- log = initlog
-
-log = initlog # The current logging function
-
-
-# Parsing functions
-# =================
-
-# Maximum input we will accept when REQUEST_METHOD is POST
-# 0 ==> unlimited input
-maxlen = 0
-
-def parse(fp=None, environ=os.environ, keep_blank_values=0,
- strict_parsing=0, separator='&'):
- """Parse a query in the environment or from a file (default stdin)
-
- Arguments, all optional:
-
- fp : file pointer; default: sys.stdin.buffer
-
- environ : environment dictionary; default: os.environ
-
- keep_blank_values: flag indicating whether blank values in
- percent-encoded forms should be treated as blank strings.
- A true value indicates that blanks should be retained as
- blank strings. The default false value indicates that
- blank values are to be ignored and treated as if they were
- not included.
-
- strict_parsing: flag indicating what to do with parsing errors.
- If false (the default), errors are silently ignored.
- If true, errors raise a ValueError exception.
-
- separator: str. The symbol to use for separating the query arguments.
- Defaults to &.
- """
- if fp is None:
- fp = sys.stdin
-
- # field keys and values (except for files) are returned as strings
- # an encoding is required to decode the bytes read from self.fp
- if hasattr(fp,'encoding'):
- encoding = fp.encoding
- else:
- encoding = 'latin-1'
-
- # fp.read() must return bytes
- if isinstance(fp, TextIOWrapper):
- fp = fp.buffer
-
- if not 'REQUEST_METHOD' in environ:
- environ['REQUEST_METHOD'] = 'GET' # For testing stand-alone
- if environ['REQUEST_METHOD'] == 'POST':
- ctype, pdict = parse_header(environ['CONTENT_TYPE'])
- if ctype == 'multipart/form-data':
- return parse_multipart(fp, pdict, separator=separator)
- elif ctype == 'application/x-www-form-urlencoded':
- clength = int(environ['CONTENT_LENGTH'])
- if maxlen and clength > maxlen:
- raise ValueError('Maximum content length exceeded')
- qs = fp.read(clength).decode(encoding)
- else:
- qs = '' # Unknown content-type
- if 'QUERY_STRING' in environ:
- if qs: qs = qs + '&'
- qs = qs + environ['QUERY_STRING']
- elif sys.argv[1:]:
- if qs: qs = qs + '&'
- qs = qs + sys.argv[1]
- environ['QUERY_STRING'] = qs # XXX Shouldn't, really
- elif 'QUERY_STRING' in environ:
- qs = environ['QUERY_STRING']
- else:
- if sys.argv[1:]:
- qs = sys.argv[1]
- else:
- qs = ""
- environ['QUERY_STRING'] = qs # XXX Shouldn't, really
- return urllib.parse.parse_qs(qs, keep_blank_values, strict_parsing,
- encoding=encoding, separator=separator)
-
-
-def parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator='&'):
- """Parse multipart input.
-
- Arguments:
- fp : input file
- pdict: dictionary containing other parameters of content-type header
- encoding, errors: request encoding and error handler, passed to
- FieldStorage
-
- Returns a dictionary just like parse_qs(): keys are the field names, each
- value is a list of values for that field. For non-file fields, the value
- is a list of strings.
- """
- # RFC 2046, Section 5.1 : The "multipart" boundary delimiters are always
- # represented as 7bit US-ASCII.
- boundary = pdict['boundary'].decode('ascii')
- ctype = "multipart/form-data; boundary={}".format(boundary)
- headers = Message()
- headers.set_type(ctype)
- try:
- headers['Content-Length'] = pdict['CONTENT-LENGTH']
- except KeyError:
- pass
- fs = FieldStorage(fp, headers=headers, encoding=encoding, errors=errors,
- environ={'REQUEST_METHOD': 'POST'}, separator=separator)
- return {k: fs.getlist(k) for k in fs}
-
-def _parseparam(s):
- while s[:1] == ';':
- s = s[1:]
- end = s.find(';')
- while end > 0 and (s.count('"', 0, end) - s.count('\\"', 0, end)) % 2:
- end = s.find(';', end + 1)
- if end < 0:
- end = len(s)
- f = s[:end]
- yield f.strip()
- s = s[end:]
-
-def parse_header(line):
- """Parse a Content-type like header.
-
- Return the main content-type and a dictionary of options.
-
- """
- parts = _parseparam(';' + line)
- key = parts.__next__()
- pdict = {}
- for p in parts:
- i = p.find('=')
- if i >= 0:
- name = p[:i].strip().lower()
- value = p[i+1:].strip()
- if len(value) >= 2 and value[0] == value[-1] == '"':
- value = value[1:-1]
- value = value.replace('\\\\', '\\').replace('\\"', '"')
- pdict[name] = value
- return key, pdict
-
-
-# Classes for field storage
-# =========================
-
-class MiniFieldStorage:
-
- """Like FieldStorage, for use when no file uploads are possible."""
-
- # Dummy attributes
- filename = None
- list = None
- type = None
- file = None
- type_options = {}
- disposition = None
- disposition_options = {}
- headers = {}
-
- def __init__(self, name, value):
- """Constructor from field name and value."""
- self.name = name
- self.value = value
- # self.file = StringIO(value)
-
- def __repr__(self):
- """Return printable representation."""
- return "MiniFieldStorage(%r, %r)" % (self.name, self.value)
-
-
-class FieldStorage:
-
- """Store a sequence of fields, reading multipart/form-data.
-
- This class provides naming, typing, files stored on disk, and
- more. At the top level, it is accessible like a dictionary, whose
- keys are the field names. (Note: None can occur as a field name.)
- The items are either a Python list (if there's multiple values) or
- another FieldStorage or MiniFieldStorage object. If it's a single
- object, it has the following attributes:
-
- name: the field name, if specified; otherwise None
-
- filename: the filename, if specified; otherwise None; this is the
- client side filename, *not* the file name on which it is
- stored (that's a temporary file you don't deal with)
-
- value: the value as a *string*; for file uploads, this
- transparently reads the file every time you request the value
- and returns *bytes*
-
- file: the file(-like) object from which you can read the data *as
- bytes* ; None if the data is stored a simple string
-
- type: the content-type, or None if not specified
-
- type_options: dictionary of options specified on the content-type
- line
-
- disposition: content-disposition, or None if not specified
-
- disposition_options: dictionary of corresponding options
-
- headers: a dictionary(-like) object (sometimes email.message.Message or a
- subclass thereof) containing *all* headers
-
- The class is subclassable, mostly for the purpose of overriding
- the make_file() method, which is called internally to come up with
- a file open for reading and writing. This makes it possible to
- override the default choice of storing all files in a temporary
- directory and unlinking them as soon as they have been opened.
-
- """
- def __init__(self, fp=None, headers=None, outerboundary=b'',
- environ=os.environ, keep_blank_values=0, strict_parsing=0,
- limit=None, encoding='utf-8', errors='replace',
- max_num_fields=None, separator='&'):
- """Constructor. Read multipart/* until last part.
-
- Arguments, all optional:
-
- fp : file pointer; default: sys.stdin.buffer
- (not used when the request method is GET)
- Can be :
- 1. a TextIOWrapper object
- 2. an object whose read() and readline() methods return bytes
-
- headers : header dictionary-like object; default:
- taken from environ as per CGI spec
-
- outerboundary : terminating multipart boundary
- (for internal use only)
-
- environ : environment dictionary; default: os.environ
-
- keep_blank_values: flag indicating whether blank values in
- percent-encoded forms should be treated as blank strings.
- A true value indicates that blanks should be retained as
- blank strings. The default false value indicates that
- blank values are to be ignored and treated as if they were
- not included.
-
- strict_parsing: flag indicating what to do with parsing errors.
- If false (the default), errors are silently ignored.
- If true, errors raise a ValueError exception.
-
- limit : used internally to read parts of multipart/form-data forms,
- to exit from the reading loop when reached. It is the difference
- between the form content-length and the number of bytes already
- read
-
- encoding, errors : the encoding and error handler used to decode the
- binary stream to strings. Must be the same as the charset defined
- for the page sending the form (content-type : meta http-equiv or
- header)
-
- max_num_fields: int. If set, then __init__ throws a ValueError
- if there are more than n fields read by parse_qsl().
-
- """
- method = 'GET'
- self.keep_blank_values = keep_blank_values
- self.strict_parsing = strict_parsing
- self.max_num_fields = max_num_fields
- self.separator = separator
- if 'REQUEST_METHOD' in environ:
- method = environ['REQUEST_METHOD'].upper()
- self.qs_on_post = None
- if method == 'GET' or method == 'HEAD':
- if 'QUERY_STRING' in environ:
- qs = environ['QUERY_STRING']
- elif sys.argv[1:]:
- qs = sys.argv[1]
- else:
- qs = ""
- qs = qs.encode(locale.getpreferredencoding(), 'surrogateescape')
- fp = BytesIO(qs)
- if headers is None:
- headers = {'content-type':
- "application/x-www-form-urlencoded"}
- if headers is None:
- headers = {}
- if method == 'POST':
- # Set default content-type for POST to what's traditional
- headers['content-type'] = "application/x-www-form-urlencoded"
- if 'CONTENT_TYPE' in environ:
- headers['content-type'] = environ['CONTENT_TYPE']
- if 'QUERY_STRING' in environ:
- self.qs_on_post = environ['QUERY_STRING']
- if 'CONTENT_LENGTH' in environ:
- headers['content-length'] = environ['CONTENT_LENGTH']
- else:
- if not (isinstance(headers, (Mapping, Message))):
- raise TypeError("headers must be mapping or an instance of "
- "email.message.Message")
- self.headers = headers
- if fp is None:
- self.fp = sys.stdin.buffer
- # self.fp.read() must return bytes
- elif isinstance(fp, TextIOWrapper):
- self.fp = fp.buffer
- else:
- if not (hasattr(fp, 'read') and hasattr(fp, 'readline')):
- raise TypeError("fp must be file pointer")
- self.fp = fp
-
- self.encoding = encoding
- self.errors = errors
-
- if not isinstance(outerboundary, bytes):
- raise TypeError('outerboundary must be bytes, not %s'
- % type(outerboundary).__name__)
- self.outerboundary = outerboundary
-
- self.bytes_read = 0
- self.limit = limit
-
- # Process content-disposition header
- cdisp, pdict = "", {}
- if 'content-disposition' in self.headers:
- cdisp, pdict = parse_header(self.headers['content-disposition'])
- self.disposition = cdisp
- self.disposition_options = pdict
- self.name = None
- if 'name' in pdict:
- self.name = pdict['name']
- self.filename = None
- if 'filename' in pdict:
- self.filename = pdict['filename']
- self._binary_file = self.filename is not None
-
- # Process content-type header
- #
- # Honor any existing content-type header. But if there is no
- # content-type header, use some sensible defaults. Assume
- # outerboundary is "" at the outer level, but something non-false
- # inside a multi-part. The default for an inner part is text/plain,
- # but for an outer part it should be urlencoded. This should catch
- # bogus clients which erroneously forget to include a content-type
- # header.
- #
- # See below for what we do if there does exist a content-type header,
- # but it happens to be something we don't understand.
- if 'content-type' in self.headers:
- ctype, pdict = parse_header(self.headers['content-type'])
- elif self.outerboundary or method != 'POST':
- ctype, pdict = "text/plain", {}
- else:
- ctype, pdict = 'application/x-www-form-urlencoded', {}
- self.type = ctype
- self.type_options = pdict
- if 'boundary' in pdict:
- self.innerboundary = pdict['boundary'].encode(self.encoding,
- self.errors)
- else:
- self.innerboundary = b""
-
- clen = -1
- if 'content-length' in self.headers:
- try:
- clen = int(self.headers['content-length'])
- except ValueError:
- pass
- if maxlen and clen > maxlen:
- raise ValueError('Maximum content length exceeded')
- self.length = clen
- if self.limit is None and clen >= 0:
- self.limit = clen
-
- self.list = self.file = None
- self.done = 0
- if ctype == 'application/x-www-form-urlencoded':
- self.read_urlencoded()
- elif ctype[:10] == 'multipart/':
- self.read_multi(environ, keep_blank_values, strict_parsing)
- else:
- self.read_single()
-
- def __del__(self):
- try:
- self.file.close()
- except AttributeError:
- pass
-
- def __enter__(self):
- return self
-
- def __exit__(self, *args):
- self.file.close()
-
- def __repr__(self):
- """Return a printable representation."""
- return "FieldStorage(%r, %r, %r)" % (
- self.name, self.filename, self.value)
-
- def __iter__(self):
- return iter(self.keys())
-
- def __getattr__(self, name):
- if name != 'value':
- raise AttributeError(name)
- if self.file:
- self.file.seek(0)
- value = self.file.read()
- self.file.seek(0)
- elif self.list is not None:
- value = self.list
- else:
- value = None
- return value
-
- def __getitem__(self, key):
- """Dictionary style indexing."""
- if self.list is None:
- raise TypeError("not indexable")
- found = []
- for item in self.list:
- if item.name == key: found.append(item)
- if not found:
- raise KeyError(key)
- if len(found) == 1:
- return found[0]
- else:
- return found
-
- def getvalue(self, key, default=None):
- """Dictionary style get() method, including 'value' lookup."""
- if key in self:
- value = self[key]
- if isinstance(value, list):
- return [x.value for x in value]
- else:
- return value.value
- else:
- return default
-
- def getfirst(self, key, default=None):
- """ Return the first value received."""
- if key in self:
- value = self[key]
- if isinstance(value, list):
- return value[0].value
- else:
- return value.value
- else:
- return default
-
- def getlist(self, key):
- """ Return list of received values."""
- if key in self:
- value = self[key]
- if isinstance(value, list):
- return [x.value for x in value]
- else:
- return [value.value]
- else:
- return []
-
- def keys(self):
- """Dictionary style keys() method."""
- if self.list is None:
- raise TypeError("not indexable")
- return list(set(item.name for item in self.list))
-
- def __contains__(self, key):
- """Dictionary style __contains__ method."""
- if self.list is None:
- raise TypeError("not indexable")
- return any(item.name == key for item in self.list)
-
- def __len__(self):
- """Dictionary style len(x) support."""
- return len(self.keys())
-
- def __bool__(self):
- if self.list is None:
- raise TypeError("Cannot be converted to bool.")
- return bool(self.list)
-
- def read_urlencoded(self):
- """Internal: read data in query string format."""
- qs = self.fp.read(self.length)
- if not isinstance(qs, bytes):
- raise ValueError("%s should return bytes, got %s" \
- % (self.fp, type(qs).__name__))
- qs = qs.decode(self.encoding, self.errors)
- if self.qs_on_post:
- qs += '&' + self.qs_on_post
- query = urllib.parse.parse_qsl(
- qs, self.keep_blank_values, self.strict_parsing,
- encoding=self.encoding, errors=self.errors,
- max_num_fields=self.max_num_fields, separator=self.separator)
- self.list = [MiniFieldStorage(key, value) for key, value in query]
- self.skip_lines()
-
- FieldStorageClass = None
-
- def read_multi(self, environ, keep_blank_values, strict_parsing):
- """Internal: read a part that is itself multipart."""
- ib = self.innerboundary
- if not valid_boundary(ib):
- raise ValueError('Invalid boundary in multipart form: %r' % (ib,))
- self.list = []
- if self.qs_on_post:
- query = urllib.parse.parse_qsl(
- self.qs_on_post, self.keep_blank_values, self.strict_parsing,
- encoding=self.encoding, errors=self.errors,
- max_num_fields=self.max_num_fields, separator=self.separator)
- self.list.extend(MiniFieldStorage(key, value) for key, value in query)
-
- klass = self.FieldStorageClass or self.__class__
- first_line = self.fp.readline() # bytes
- if not isinstance(first_line, bytes):
- raise ValueError("%s should return bytes, got %s" \
- % (self.fp, type(first_line).__name__))
- self.bytes_read += len(first_line)
-
- # Ensure that we consume the file until we've hit our inner boundary
- while (first_line.strip() != (b"--" + self.innerboundary) and
- first_line):
- first_line = self.fp.readline()
- self.bytes_read += len(first_line)
-
- # Propagate max_num_fields into the sub class appropriately
- max_num_fields = self.max_num_fields
- if max_num_fields is not None:
- max_num_fields -= len(self.list)
-
- while True:
- parser = FeedParser()
- hdr_text = b""
- while True:
- data = self.fp.readline()
- hdr_text += data
- if not data.strip():
- break
- if not hdr_text:
- break
- # parser takes strings, not bytes
- self.bytes_read += len(hdr_text)
- parser.feed(hdr_text.decode(self.encoding, self.errors))
- headers = parser.close()
-
- # Some clients add Content-Length for part headers, ignore them
- if 'content-length' in headers:
- del headers['content-length']
-
- limit = None if self.limit is None \
- else self.limit - self.bytes_read
- part = klass(self.fp, headers, ib, environ, keep_blank_values,
- strict_parsing, limit,
- self.encoding, self.errors, max_num_fields, self.separator)
-
- if max_num_fields is not None:
- max_num_fields -= 1
- if part.list:
- max_num_fields -= len(part.list)
- if max_num_fields < 0:
- raise ValueError('Max number of fields exceeded')
-
- self.bytes_read += part.bytes_read
- self.list.append(part)
- if part.done or self.bytes_read >= self.length > 0:
- break
- self.skip_lines()
-
- def read_single(self):
- """Internal: read an atomic part."""
- if self.length >= 0:
- self.read_binary()
- self.skip_lines()
- else:
- self.read_lines()
- self.file.seek(0)
-
- bufsize = 8*1024 # I/O buffering size for copy to file
-
- def read_binary(self):
- """Internal: read binary data."""
- self.file = self.make_file()
- todo = self.length
- if todo >= 0:
- while todo > 0:
- data = self.fp.read(min(todo, self.bufsize)) # bytes
- if not isinstance(data, bytes):
- raise ValueError("%s should return bytes, got %s"
- % (self.fp, type(data).__name__))
- self.bytes_read += len(data)
- if not data:
- self.done = -1
- break
- self.file.write(data)
- todo = todo - len(data)
-
- def read_lines(self):
- """Internal: read lines until EOF or outerboundary."""
- if self._binary_file:
- self.file = self.__file = BytesIO() # store data as bytes for files
- else:
- self.file = self.__file = StringIO() # as strings for other fields
- if self.outerboundary:
- self.read_lines_to_outerboundary()
- else:
- self.read_lines_to_eof()
-
- def __write(self, line):
- """line is always bytes, not string"""
- if self.__file is not None:
- if self.__file.tell() + len(line) > 1000:
- self.file = self.make_file()
- data = self.__file.getvalue()
- self.file.write(data)
- self.__file = None
- if self._binary_file:
- # keep bytes
- self.file.write(line)
- else:
- # decode to string
- self.file.write(line.decode(self.encoding, self.errors))
-
- def read_lines_to_eof(self):
- """Internal: read lines until EOF."""
- while 1:
- line = self.fp.readline(1<<16) # bytes
- self.bytes_read += len(line)
- if not line:
- self.done = -1
- break
- self.__write(line)
-
- def read_lines_to_outerboundary(self):
- """Internal: read lines until outerboundary.
- Data is read as bytes: boundaries and line ends must be converted
- to bytes for comparisons.
- """
- next_boundary = b"--" + self.outerboundary
- last_boundary = next_boundary + b"--"
- delim = b""
- last_line_lfend = True
- _read = 0
- while 1:
-
- if self.limit is not None and 0 <= self.limit <= _read:
- break
- line = self.fp.readline(1<<16) # bytes
- self.bytes_read += len(line)
- _read += len(line)
- if not line:
- self.done = -1
- break
- if delim == b"\r":
- line = delim + line
- delim = b""
- if line.startswith(b"--") and last_line_lfend:
- strippedline = line.rstrip()
- if strippedline == next_boundary:
- break
- if strippedline == last_boundary:
- self.done = 1
- break
- odelim = delim
- if line.endswith(b"\r\n"):
- delim = b"\r\n"
- line = line[:-2]
- last_line_lfend = True
- elif line.endswith(b"\n"):
- delim = b"\n"
- line = line[:-1]
- last_line_lfend = True
- elif line.endswith(b"\r"):
- # We may interrupt \r\n sequences if they span the 2**16
- # byte boundary
- delim = b"\r"
- line = line[:-1]
- last_line_lfend = False
- else:
- delim = b""
- last_line_lfend = False
- self.__write(odelim + line)
-
- def skip_lines(self):
- """Internal: skip lines until outer boundary if defined."""
- if not self.outerboundary or self.done:
- return
- next_boundary = b"--" + self.outerboundary
- last_boundary = next_boundary + b"--"
- last_line_lfend = True
- while True:
- line = self.fp.readline(1<<16)
- self.bytes_read += len(line)
- if not line:
- self.done = -1
- break
- if line.endswith(b"--") and last_line_lfend:
- strippedline = line.strip()
- if strippedline == next_boundary:
- break
- if strippedline == last_boundary:
- self.done = 1
- break
- last_line_lfend = line.endswith(b'\n')
-
- def make_file(self):
- """Overridable: return a readable & writable file.
-
- The file will be used as follows:
- - data is written to it
- - seek(0)
- - data is read from it
-
- The file is opened in binary mode for files, in text mode
- for other fields
-
- This version opens a temporary file for reading and writing,
- and immediately deletes (unlinks) it. The trick (on Unix!) is
- that the file can still be used, but it can't be opened by
- another process, and it will automatically be deleted when it
- is closed or when the current process terminates.
-
- If you want a more permanent file, you derive a class which
- overrides this method. If you want a visible temporary file
- that is nevertheless automatically deleted when the script
- terminates, try defining a __del__ method in a derived class
- which unlinks the temporary files you have created.
-
- """
- if self._binary_file:
- return tempfile.TemporaryFile("wb+")
- else:
- return tempfile.TemporaryFile("w+",
- encoding=self.encoding, newline = '\n')
-
-
-# Test/debug code
-# ===============
-
-def test(environ=os.environ):
- """Robust test CGI script, usable as main program.
-
- Write minimal HTTP headers and dump all information provided to
- the script in HTML form.
-
- """
- print("Content-type: text/html")
- print()
- sys.stderr = sys.stdout
- try:
- form = FieldStorage() # Replace with other classes to test those
- print_directory()
- print_arguments()
- print_form(form)
- print_environ(environ)
- print_environ_usage()
- def f():
- exec("testing print_exception() -- italics?")
- def g(f=f):
- f()
- print("
")
- print()
- print(sys.argv)
- print()
-
-def print_environ_usage():
- """Dump a list of environment variables used by CGI as HTML."""
- print("""
-
These environment variables could have been set:
-
-
AUTH_TYPE
-
CONTENT_LENGTH
-
CONTENT_TYPE
-
DATE_GMT
-
DATE_LOCAL
-
DOCUMENT_NAME
-
DOCUMENT_ROOT
-
DOCUMENT_URI
-
GATEWAY_INTERFACE
-
LAST_MODIFIED
-
PATH
-
PATH_INFO
-
PATH_TRANSLATED
-
QUERY_STRING
-
REMOTE_ADDR
-
REMOTE_HOST
-
REMOTE_IDENT
-
REMOTE_USER
-
REQUEST_METHOD
-
SCRIPT_NAME
-
SERVER_NAME
-
SERVER_PORT
-
SERVER_PROTOCOL
-
SERVER_ROOT
-
SERVER_SOFTWARE
-
-In addition, HTTP headers sent by the server may be passed in the
-environment as well. Here are some common variable names:
-
-
HTTP_ACCEPT
-
HTTP_CONNECTION
-
HTTP_HOST
-
HTTP_PRAGMA
-
HTTP_REFERER
-
HTTP_USER_AGENT
-
-""")
-
-
-# Utilities
-# =========
-
-def valid_boundary(s):
- import re
- if isinstance(s, bytes):
- _vb_pattern = b"^[ -~]{0,200}[!-~]$"
- else:
- _vb_pattern = "^[ -~]{0,200}[!-~]$"
- return re.match(_vb_pattern, s)
-
-# Invoke mainline
-# ===============
-
-# Call test() when this file is run as a script (not imported as a module)
-if __name__ == '__main__':
- test()
diff --git a/PythonLib/full/cgitb.py b/PythonLib/full/cgitb.py
deleted file mode 100644
index f6b97f25c..000000000
--- a/PythonLib/full/cgitb.py
+++ /dev/null
@@ -1,332 +0,0 @@
-"""More comprehensive traceback formatting for Python scripts.
-
-To enable this module, do:
-
- import cgitb; cgitb.enable()
-
-at the top of your script. The optional arguments to enable() are:
-
- display - if true, tracebacks are displayed in the web browser
- logdir - if set, tracebacks are written to files in this directory
- context - number of lines of source code to show for each stack frame
- format - 'text' or 'html' controls the output format
-
-By default, tracebacks are displayed but not saved, the context is 5 lines
-and the output format is 'html' (for backwards compatibility with the
-original use of this module)
-
-Alternatively, if you have caught an exception and want cgitb to display it
-for you, call cgitb.handler(). The optional argument to handler() is a
-3-item tuple (etype, evalue, etb) just like the value of sys.exc_info().
-The default handler displays output as HTML.
-
-"""
-import inspect
-import keyword
-import linecache
-import os
-import pydoc
-import sys
-import tempfile
-import time
-import tokenize
-import traceback
-import warnings
-from html import escape as html_escape
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-
-def reset():
- """Return a string that resets the CGI and browser to a known state."""
- return '''
- --> -->
-
- '''
-
-__UNDEF__ = [] # a special sentinel object
-def small(text):
- if text:
- return '' + text + ''
- else:
- return ''
-
-def strong(text):
- if text:
- return '' + text + ''
- else:
- return ''
-
-def grey(text):
- if text:
- return '' + text + ''
- else:
- return ''
-
-def lookup(name, frame, locals):
- """Find the value for a given name in the given environment."""
- if name in locals:
- return 'local', locals[name]
- if name in frame.f_globals:
- return 'global', frame.f_globals[name]
- if '__builtins__' in frame.f_globals:
- builtins = frame.f_globals['__builtins__']
- if isinstance(builtins, dict):
- if name in builtins:
- return 'builtin', builtins[name]
- else:
- if hasattr(builtins, name):
- return 'builtin', getattr(builtins, name)
- return None, __UNDEF__
-
-def scanvars(reader, frame, locals):
- """Scan one logical line of Python and look up values of variables used."""
- vars, lasttoken, parent, prefix, value = [], None, None, '', __UNDEF__
- for ttype, token, start, end, line in tokenize.generate_tokens(reader):
- if ttype == tokenize.NEWLINE: break
- if ttype == tokenize.NAME and token not in keyword.kwlist:
- if lasttoken == '.':
- if parent is not __UNDEF__:
- value = getattr(parent, token, __UNDEF__)
- vars.append((prefix + token, prefix, value))
- else:
- where, value = lookup(token, frame, locals)
- vars.append((token, where, value))
- elif token == '.':
- prefix += lasttoken + '.'
- parent = value
- else:
- parent, prefix = None, ''
- lasttoken = token
- return vars
-
-def html(einfo, context=5):
- """Return a nice HTML document describing a given traceback."""
- etype, evalue, etb = einfo
- if isinstance(etype, type):
- etype = etype.__name__
- pyver = 'Python ' + sys.version.split()[0] + ': ' + sys.executable
- date = time.ctime(time.time())
- head = f'''
-
-
-
-
-
-{html_escape(str(etype))}
-
-{pyver} {date}
-
-
A problem occurred in a Python script. Here is the sequence of
-function calls leading up to the error, in the order they occurred.
' %
- (' ', link, call)]
- if index is not None:
- i = lnum - index
- for line in lines:
- num = small(' ' * (5-len(str(i))) + str(i)) + ' '
- if i in highlight:
- line = '=>%s%s' % (num, pydoc.html.preformat(line))
- rows.append('
' % grey(line))
- i += 1
-
- done, dump = {}, []
- for name, where, value in vars:
- if name in done: continue
- done[name] = 1
- if value is not __UNDEF__:
- if where in ('global', 'builtin'):
- name = ('%s ' % where) + strong(name)
- elif where == 'local':
- name = strong(name)
- else:
- name = where + strong(name.split('.')[-1])
- dump.append('%s = %s' % (name, pydoc.html.repr(value)))
- else:
- dump.append(name + ' undefined')
-
- rows.append('
A problem occurred in a Python script.\n')
-
- if self.logdir is not None:
- suffix = ['.txt', '.html'][self.format=="html"]
- (fd, path) = tempfile.mkstemp(suffix=suffix, dir=self.logdir)
-
- try:
- with os.fdopen(fd, 'w') as file:
- file.write(doc)
- msg = '%s contains the description of this error.' % path
- except:
- msg = 'Tried to save traceback to %s, but failed.' % path
-
- if self.format == 'html':
- self.file.write('
%s
\n' % msg)
- else:
- self.file.write(msg + '\n')
- try:
- self.file.flush()
- except: pass
-
-handler = Hook().handle
-def enable(display=1, logdir=None, context=5, format="html"):
- """Install an exception handler that formats tracebacks as HTML.
-
- The optional argument 'display' can be set to 0 to suppress sending the
- traceback to the browser, and 'logdir' can be set to a directory to cause
- tracebacks to be written to files there."""
- sys.excepthook = Hook(display=display, logdir=logdir,
- context=context, format=format)
diff --git a/PythonLib/full/chunk.py b/PythonLib/full/chunk.py
deleted file mode 100644
index 618781efd..000000000
--- a/PythonLib/full/chunk.py
+++ /dev/null
@@ -1,173 +0,0 @@
-"""Simple class to read IFF chunks.
-
-An IFF chunk (used in formats such as AIFF, TIFF, RMFF (RealMedia File
-Format)) has the following structure:
-
-+----------------+
-| ID (4 bytes) |
-+----------------+
-| size (4 bytes) |
-+----------------+
-| data |
-| ... |
-+----------------+
-
-The ID is a 4-byte string which identifies the type of chunk.
-
-The size field (a 32-bit value, encoded using big-endian byte order)
-gives the size of the whole chunk, including the 8-byte header.
-
-Usually an IFF-type file consists of one or more chunks. The proposed
-usage of the Chunk class defined here is to instantiate an instance at
-the start of each chunk and read from the instance until it reaches
-the end, after which a new instance can be instantiated. At the end
-of the file, creating a new instance will fail with an EOFError
-exception.
-
-Usage:
-while True:
- try:
- chunk = Chunk(file)
- except EOFError:
- break
- chunktype = chunk.getname()
- while True:
- data = chunk.read(nbytes)
- if not data:
- pass
- # do something with data
-
-The interface is file-like. The implemented methods are:
-read, close, seek, tell, isatty.
-Extra methods are: skip() (called by close, skips to the end of the chunk),
-getname() (returns the name (ID) of the chunk)
-
-The __init__ method has one required argument, a file-like object
-(including a chunk instance), and one optional argument, a flag which
-specifies whether or not chunks are aligned on 2-byte boundaries. The
-default is 1, i.e. aligned.
-"""
-
-import warnings
-
-warnings._deprecated(__name__, remove=(3, 13))
-
-class Chunk:
- def __init__(self, file, align=True, bigendian=True, inclheader=False):
- import struct
- self.closed = False
- self.align = align # whether to align to word (2-byte) boundaries
- if bigendian:
- strflag = '>'
- else:
- strflag = '<'
- self.file = file
- self.chunkname = file.read(4)
- if len(self.chunkname) < 4:
- raise EOFError
- try:
- self.chunksize = struct.unpack_from(strflag+'L', file.read(4))[0]
- except struct.error:
- raise EOFError from None
- if inclheader:
- self.chunksize = self.chunksize - 8 # subtract header
- self.size_read = 0
- try:
- self.offset = self.file.tell()
- except (AttributeError, OSError):
- self.seekable = False
- else:
- self.seekable = True
-
- def getname(self):
- """Return the name (ID) of the current chunk."""
- return self.chunkname
-
- def getsize(self):
- """Return the size of the current chunk."""
- return self.chunksize
-
- def close(self):
- if not self.closed:
- try:
- self.skip()
- finally:
- self.closed = True
-
- def isatty(self):
- if self.closed:
- raise ValueError("I/O operation on closed file")
- return False
-
- def seek(self, pos, whence=0):
- """Seek to specified position into the chunk.
- Default position is 0 (start of chunk).
- If the file is not seekable, this will result in an error.
- """
-
- if self.closed:
- raise ValueError("I/O operation on closed file")
- if not self.seekable:
- raise OSError("cannot seek")
- if whence == 1:
- pos = pos + self.size_read
- elif whence == 2:
- pos = pos + self.chunksize
- if pos < 0 or pos > self.chunksize:
- raise RuntimeError
- self.file.seek(self.offset + pos, 0)
- self.size_read = pos
-
- def tell(self):
- if self.closed:
- raise ValueError("I/O operation on closed file")
- return self.size_read
-
- def read(self, size=-1):
- """Read at most size bytes from the chunk.
- If size is omitted or negative, read until the end
- of the chunk.
- """
-
- if self.closed:
- raise ValueError("I/O operation on closed file")
- if self.size_read >= self.chunksize:
- return b''
- if size < 0:
- size = self.chunksize - self.size_read
- if size > self.chunksize - self.size_read:
- size = self.chunksize - self.size_read
- data = self.file.read(size)
- self.size_read = self.size_read + len(data)
- if self.size_read == self.chunksize and \
- self.align and \
- (self.chunksize & 1):
- dummy = self.file.read(1)
- self.size_read = self.size_read + len(dummy)
- return data
-
- def skip(self):
- """Skip the rest of the chunk.
- If you are not interested in the contents of the chunk,
- this method should be called so that the file points to
- the start of the next chunk.
- """
-
- if self.closed:
- raise ValueError("I/O operation on closed file")
- if self.seekable:
- try:
- n = self.chunksize - self.size_read
- # maybe fix alignment
- if self.align and (self.chunksize & 1):
- n = n + 1
- self.file.seek(n, 1)
- self.size_read = self.size_read + n
- return
- except OSError:
- pass
- while self.size_read < self.chunksize:
- n = min(8192, self.chunksize - self.size_read)
- dummy = self.read(n)
- if not dummy:
- raise EOFError
diff --git a/PythonLib/full/cmd.py b/PythonLib/full/cmd.py
index 88ee7d3dd..51495fb32 100644
--- a/PythonLib/full/cmd.py
+++ b/PythonLib/full/cmd.py
@@ -5,16 +5,16 @@
1. End of file on input is processed as the command 'EOF'.
2. A command is parsed out of each line by collecting the prefix composed
of characters in the identchars member.
-3. A command `foo' is dispatched to a method 'do_foo()'; the do_ method
+3. A command 'foo' is dispatched to a method 'do_foo()'; the do_ method
is passed a single argument consisting of the remainder of the line.
4. Typing an empty line repeats the last command. (Actually, it calls the
- method `emptyline', which may be overridden in a subclass.)
-5. There is a predefined `help' method. Given an argument `topic', it
- calls the command `help_topic'. With no arguments, it lists all topics
+ method 'emptyline', which may be overridden in a subclass.)
+5. There is a predefined 'help' method. Given an argument 'topic', it
+ calls the command 'help_topic'. With no arguments, it lists all topics
with defined help_ functions, broken into up to three topics; documented
commands, miscellaneous help topics, and undocumented commands.
-6. The command '?' is a synonym for `help'. The command '!' is a synonym
- for `shell', if a do_shell method exists.
+6. The command '?' is a synonym for 'help'. The command '!' is a synonym
+ for 'shell', if a do_shell method exists.
7. If completion is enabled, completing commands will be done automatically,
and completing of commands args is done by calling complete_foo() with
arguments text, line, begidx, endidx. text is string we are matching
@@ -23,31 +23,34 @@
indexes of the text being matched, which could be used to provide
different completion depending upon which position the argument is in.
-The `default' method may be overridden to intercept commands for which there
+The 'default' method may be overridden to intercept commands for which there
is no do_ method.
-The `completedefault' method may be overridden to intercept completions for
+The 'completedefault' method may be overridden to intercept completions for
commands that have no complete_ method.
-The data member `self.ruler' sets the character used to draw separator lines
+The data member 'self.ruler' sets the character used to draw separator lines
in the help messages. If empty, no ruler line is drawn. It defaults to "=".
-If the value of `self.intro' is nonempty when the cmdloop method is called,
+If the value of 'self.intro' is nonempty when the cmdloop method is called,
it is printed out on interpreter startup. This value may be overridden
via an optional argument to the cmdloop() method.
-The data members `self.doc_header', `self.misc_header', and
-`self.undoc_header' set the headers used for the help function's
+The data members 'self.doc_header', 'self.misc_header', and
+'self.undoc_header' set the headers used for the help function's
listings of documented functions, miscellaneous topics, and undocumented
functions respectively.
"""
-import string, sys
+import sys
__all__ = ["Cmd"]
PROMPT = '(Cmd) '
-IDENTCHARS = string.ascii_letters + string.digits + '_'
+IDENTCHARS = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ'
+ 'abcdefghijklmnopqrstuvwxyz'
+ '0123456789'
+ '_')
class Cmd:
"""A simple framework for writing line-oriented command interpreters.
@@ -108,7 +111,15 @@ def cmdloop(self, intro=None):
import readline
self.old_completer = readline.get_completer()
readline.set_completer(self.complete)
- readline.parse_and_bind(self.completekey+": complete")
+ if readline.backend == "editline":
+ if self.completekey == 'tab':
+ # libedit uses "^I" instead of "tab"
+ command_string = "bind ^I rl_complete"
+ else:
+ command_string = f"bind {self.completekey} rl_complete"
+ else:
+ command_string = f"{self.completekey}: complete"
+ readline.parse_and_bind(command_string)
except ImportError:
pass
try:
@@ -210,9 +221,8 @@ def onecmd(self, line):
if cmd == '':
return self.default(line)
else:
- try:
- func = getattr(self, 'do_' + cmd)
- except AttributeError:
+ func = getattr(self, 'do_' + cmd, None)
+ if func is None:
return self.default(line)
return func(arg)
@@ -263,7 +273,7 @@ def complete(self, text, state):
endidx = readline.get_endidx() - stripped
if begidx>0:
cmd, args, foo = self.parseline(line)
- if cmd == '':
+ if not cmd:
compfunc = self.completedefault
else:
try:
@@ -296,8 +306,11 @@ def do_help(self, arg):
try:
func = getattr(self, 'help_' + arg)
except AttributeError:
+ from inspect import cleandoc
+
try:
doc=getattr(self, 'do_' + arg).__doc__
+ doc = cleandoc(doc)
if doc:
self.stdout.write("%s\n"%str(doc))
return
diff --git a/PythonLib/full/code.py b/PythonLib/full/code.py
index cb7dd44b0..b134886dc 100644
--- a/PythonLib/full/code.py
+++ b/PythonLib/full/code.py
@@ -5,6 +5,7 @@
# Inspired by similar code by Jeff Epler and Fredrik Lundh.
+import builtins
import sys
import traceback
from codeop import CommandCompiler, compile_command
@@ -24,10 +25,10 @@ class InteractiveInterpreter:
def __init__(self, locals=None):
"""Constructor.
- The optional 'locals' argument specifies the dictionary in
- which code will be executed; it defaults to a newly created
- dictionary with key "__name__" set to "__console__" and key
- "__doc__" set to None.
+ The optional 'locals' argument specifies a mapping to use as the
+ namespace in which code will be executed; it defaults to a newly
+ created dictionary with key "__name__" set to "__console__" and
+ key "__doc__" set to None.
"""
if locals is None:
@@ -63,7 +64,7 @@ def runsource(self, source, filename="", symbol="single"):
code = self.compile(source, filename, symbol)
except (OverflowError, SyntaxError, ValueError):
# Case 1
- self.showsyntaxerror(filename)
+ self.showsyntaxerror(filename, source=source)
return False
if code is None:
@@ -93,7 +94,7 @@ def runcode(self, code):
except:
self.showtraceback()
- def showsyntaxerror(self, filename=None):
+ def showsyntaxerror(self, filename=None, **kwargs):
"""Display the syntax error that just occurred.
This doesn't display a stack trace because there isn't one.
@@ -107,17 +108,10 @@ def showsyntaxerror(self, filename=None):
"""
try:
typ, value, tb = sys.exc_info()
- if filename and typ is SyntaxError:
- # Work hard to stuff the correct filename in the exception
- try:
- msg, (dummy_filename, lineno, offset, line) = value.args
- except ValueError:
- # Not the format we expect; leave it alone
- pass
- else:
- # Stuff in the right filename
- value = SyntaxError(msg, (filename, lineno, offset, line))
- self._showtraceback(typ, value, None)
+ if filename and issubclass(typ, SyntaxError):
+ value.filename = filename
+ source = kwargs.pop('source', "")
+ self._showtraceback(typ, value, None, source)
finally:
typ = value = tb = None
@@ -131,17 +125,23 @@ def showtraceback(self):
"""
try:
typ, value, tb = sys.exc_info()
- self._showtraceback(typ, value, tb.tb_next)
+ self._showtraceback(typ, value, tb.tb_next, "")
finally:
typ = value = tb = None
- def _showtraceback(self, typ, value, tb):
+ def _showtraceback(self, typ, value, tb, source):
sys.last_type = typ
sys.last_traceback = tb
- sys.last_exc = sys.last_value = value = value.with_traceback(tb)
+ value = value.with_traceback(tb)
+ # Set the line of text that the exception refers to
+ lines = source.splitlines()
+ if (source and typ is SyntaxError
+ and not value.text and value.lineno is not None
+ and len(lines) >= value.lineno):
+ value.text = lines[value.lineno - 1]
+ sys.last_exc = sys.last_value = value
if sys.excepthook is sys.__excepthook__:
- lines = traceback.format_exception(typ, value, tb)
- self.write(''.join(lines))
+ self._excepthook(typ, value, tb)
else:
# If someone has set sys.excepthook, we let that take precedence
# over self.write
@@ -158,6 +158,12 @@ def _showtraceback(self, typ, value, tb):
print('Original exception was:', file=sys.stderr)
sys.__excepthook__(typ, value, tb)
+ def _excepthook(self, typ, value, tb):
+ # This method is being overwritten in
+ # _pyrepl.console.InteractiveColoredConsole
+ lines = traceback.format_exception(typ, value, tb)
+ self.write(''.join(lines))
+
def write(self, data):
"""Write a string.
@@ -176,7 +182,7 @@ class InteractiveConsole(InteractiveInterpreter):
"""
- def __init__(self, locals=None, filename=""):
+ def __init__(self, locals=None, filename="", *, local_exit=False):
"""Constructor.
The optional locals argument will be passed to the
@@ -188,6 +194,7 @@ def __init__(self, locals=None, filename=""):
"""
InteractiveInterpreter.__init__(self, locals)
self.filename = filename
+ self.local_exit = local_exit
self.resetbuffer()
def resetbuffer(self):
@@ -212,12 +219,17 @@ def interact(self, banner=None, exitmsg=None):
"""
try:
sys.ps1
+ delete_ps1_after = False
except AttributeError:
sys.ps1 = ">>> "
+ delete_ps1_after = True
try:
- sys.ps2
+ _ps2 = sys.ps2
+ delete_ps2_after = False
except AttributeError:
sys.ps2 = "... "
+ delete_ps2_after = True
+
cprt = 'Type "help", "copyright", "credits" or "license" for more information.'
if banner is None:
self.write("Python %s on %s\n%s\n(%s)\n" %
@@ -226,29 +238,72 @@ def interact(self, banner=None, exitmsg=None):
elif banner:
self.write("%s\n" % str(banner))
more = 0
- while 1:
- try:
- if more:
- prompt = sys.ps2
- else:
- prompt = sys.ps1
+
+ # When the user uses exit() or quit() in their interactive shell
+ # they probably just want to exit the created shell, not the whole
+ # process. exit and quit in builtins closes sys.stdin which makes
+ # it super difficult to restore
+ #
+ # When self.local_exit is True, we overwrite the builtins so
+ # exit() and quit() only raises SystemExit and we can catch that
+ # to only exit the interactive shell
+
+ _exit = None
+ _quit = None
+
+ if self.local_exit:
+ if hasattr(builtins, "exit"):
+ _exit = builtins.exit
+ builtins.exit = Quitter("exit")
+
+ if hasattr(builtins, "quit"):
+ _quit = builtins.quit
+ builtins.quit = Quitter("quit")
+
+ try:
+ while True:
try:
- line = self.raw_input(prompt)
- except EOFError:
- self.write("\n")
- break
- else:
- more = self.push(line)
- except KeyboardInterrupt:
- self.write("\nKeyboardInterrupt\n")
- self.resetbuffer()
- more = 0
- if exitmsg is None:
- self.write('now exiting %s...\n' % self.__class__.__name__)
- elif exitmsg != '':
- self.write('%s\n' % exitmsg)
-
- def push(self, line):
+ if more:
+ prompt = sys.ps2
+ else:
+ prompt = sys.ps1
+ try:
+ line = self.raw_input(prompt)
+ except EOFError:
+ self.write("\n")
+ break
+ else:
+ more = self.push(line)
+ except KeyboardInterrupt:
+ self.write("\nKeyboardInterrupt\n")
+ self.resetbuffer()
+ more = 0
+ except SystemExit as e:
+ if self.local_exit:
+ self.write("\n")
+ break
+ else:
+ raise e
+ finally:
+ # restore exit and quit in builtins if they were modified
+ if _exit is not None:
+ builtins.exit = _exit
+
+ if _quit is not None:
+ builtins.quit = _quit
+
+ if delete_ps1_after:
+ del sys.ps1
+
+ if delete_ps2_after:
+ del sys.ps2
+
+ if exitmsg is None:
+ self.write('now exiting %s...\n' % self.__class__.__name__)
+ elif exitmsg != '':
+ self.write('%s\n' % exitmsg)
+
+ def push(self, line, filename=None, _symbol="single"):
"""Push a line to the interpreter.
The line should not have a trailing newline; it may have
@@ -264,7 +319,9 @@ def push(self, line):
"""
self.buffer.append(line)
source = "\n".join(self.buffer)
- more = self.runsource(source, self.filename)
+ if filename is None:
+ filename = self.filename
+ more = self.runsource(source, filename, symbol=_symbol)
if not more:
self.resetbuffer()
return more
@@ -283,8 +340,22 @@ def raw_input(self, prompt=""):
return input(prompt)
+class Quitter:
+ def __init__(self, name):
+ self.name = name
+ if sys.platform == "win32":
+ self.eof = 'Ctrl-Z plus Return'
+ else:
+ self.eof = 'Ctrl-D (i.e. EOF)'
+
+ def __repr__(self):
+ return f'Use {self.name} or {self.eof} to exit'
+
+ def __call__(self, code=None):
+ raise SystemExit(code)
+
-def interact(banner=None, readfunc=None, local=None, exitmsg=None):
+def interact(banner=None, readfunc=None, local=None, exitmsg=None, local_exit=False):
"""Closely emulate the interactive Python interpreter.
This is a backwards compatible interface to the InteractiveConsole
@@ -297,14 +368,15 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None):
readfunc -- if not None, replaces InteractiveConsole.raw_input()
local -- passed to InteractiveInterpreter.__init__()
exitmsg -- passed to InteractiveConsole.interact()
+ local_exit -- passed to InteractiveConsole.__init__()
"""
- console = InteractiveConsole(local)
+ console = InteractiveConsole(local, local_exit=local_exit)
if readfunc is not None:
console.raw_input = readfunc
else:
try:
- import readline
+ import readline # noqa: F401
except ImportError:
pass
console.interact(banner, exitmsg)
@@ -313,7 +385,7 @@ def interact(banner=None, readfunc=None, local=None, exitmsg=None):
if __name__ == "__main__":
import argparse
- parser = argparse.ArgumentParser()
+ parser = argparse.ArgumentParser(color=True)
parser.add_argument('-q', action='store_true',
help="don't print version and copyright messages")
args = parser.parse_args()
diff --git a/PythonLib/full/codecs.py b/PythonLib/full/codecs.py
index 82f23983e..e4a8010ab 100644
--- a/PythonLib/full/codecs.py
+++ b/PythonLib/full/codecs.py
@@ -111,6 +111,9 @@ def __repr__(self):
(self.__class__.__module__, self.__class__.__qualname__,
self.name, id(self))
+ def __getnewargs__(self):
+ return tuple(self)
+
class Codec:
""" Defines the interface for stateless encoders/decoders.
@@ -615,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True):
method and are included in the list entries.
sizehint, if given, is ignored since there is no efficient
- way to finding the true end-of-line.
+ way of finding the true end-of-line.
"""
data = self.read()
@@ -706,13 +709,13 @@ def read(self, size=-1):
return self.reader.read(size)
- def readline(self, size=None):
+ def readline(self, size=None, keepends=True):
- return self.reader.readline(size)
+ return self.reader.readline(size, keepends)
- def readlines(self, sizehint=None):
+ def readlines(self, sizehint=None, keepends=True):
- return self.reader.readlines(sizehint)
+ return self.reader.readlines(sizehint, keepends)
def __next__(self):
@@ -881,7 +884,6 @@ def __reduce_ex__(self, proto):
### Shortcuts
def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
-
""" Open an encoded file using the given mode and return
a wrapped version providing transparent encoding/decoding.
@@ -909,8 +911,11 @@ def open(filename, mode='r', encoding=None, errors='strict', buffering=-1):
.encoding which allows querying the used encoding. This
attribute is only available if an encoding was specified as
parameter.
-
"""
+ import warnings
+ warnings.warn("codecs.open() is deprecated. Use open() instead.",
+ DeprecationWarning, stacklevel=2)
+
if encoding is not None and \
'b' not in mode:
# Force opening of the file in binary mode
@@ -1106,24 +1111,15 @@ def make_encoding_map(decoding_map):
### error handlers
-try:
- strict_errors = lookup_error("strict")
- ignore_errors = lookup_error("ignore")
- replace_errors = lookup_error("replace")
- xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace")
- backslashreplace_errors = lookup_error("backslashreplace")
- namereplace_errors = lookup_error("namereplace")
-except LookupError:
- # In --disable-unicode builds, these error handler are missing
- strict_errors = None
- ignore_errors = None
- replace_errors = None
- xmlcharrefreplace_errors = None
- backslashreplace_errors = None
- namereplace_errors = None
+strict_errors = lookup_error("strict")
+ignore_errors = lookup_error("ignore")
+replace_errors = lookup_error("replace")
+xmlcharrefreplace_errors = lookup_error("xmlcharrefreplace")
+backslashreplace_errors = lookup_error("backslashreplace")
+namereplace_errors = lookup_error("namereplace")
# Tell modulefinder that using codecs probably needs the encodings
# package
_false = 0
if _false:
- import encodings
+ import encodings # noqa: F401
diff --git a/PythonLib/full/codeop.py b/PythonLib/full/codeop.py
index 4dd096574..8cac00442 100644
--- a/PythonLib/full/codeop.py
+++ b/PythonLib/full/codeop.py
@@ -44,9 +44,10 @@
# Caveat emptor: These flags are undocumented on purpose and depending
# on their effect outside the standard library is **unsupported**.
PyCF_DONT_IMPLY_DEDENT = 0x200
+PyCF_ONLY_AST = 0x400
PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000
-def _maybe_compile(compiler, source, filename, symbol):
+def _maybe_compile(compiler, source, filename, symbol, flags):
# Check for source consisting of only blank lines and comments.
for line in source.split("\n"):
line = line.strip()
@@ -60,36 +61,26 @@ def _maybe_compile(compiler, source, filename, symbol):
with warnings.catch_warnings():
warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
try:
- compiler(source, filename, symbol)
+ compiler(source, filename, symbol, flags=flags)
except SyntaxError: # Let other compile() errors propagate.
try:
- compiler(source + "\n", filename, symbol)
+ compiler(source + "\n", filename, symbol, flags=flags)
+ return None
+ except _IncompleteInputError as e:
return None
except SyntaxError as e:
- if "incomplete input" in str(e):
- return None
+ pass
# fallthrough
return compiler(source, filename, symbol, incomplete_input=False)
-def _is_syntax_error(err1, err2):
- rep1 = repr(err1)
- rep2 = repr(err2)
- if "was never closed" in rep1 and "was never closed" in rep2:
- return False
- if rep1 == rep2:
- return True
- return False
-
-def _compile(source, filename, symbol, incomplete_input=True):
- flags = 0
+def _compile(source, filename, symbol, incomplete_input=True, *, flags=0):
if incomplete_input:
flags |= PyCF_ALLOW_INCOMPLETE_INPUT
flags |= PyCF_DONT_IMPLY_DEDENT
return compile(source, filename, symbol, flags)
-
-def compile_command(source, filename="", symbol="single"):
+def compile_command(source, filename="", symbol="single", flags=0):
r"""Compile a command and determine whether it is incomplete.
Arguments:
@@ -108,7 +99,7 @@ def compile_command(source, filename="", symbol="single"):
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
- return _maybe_compile(_compile, source, filename, symbol)
+ return _maybe_compile(_compile, source, filename, symbol, flags)
class Compile:
"""Instances of this class behave much like the built-in compile
@@ -118,12 +109,14 @@ class Compile:
def __init__(self):
self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT
- def __call__(self, source, filename, symbol, **kwargs):
- flags = self.flags
+ def __call__(self, source, filename, symbol, flags=0, **kwargs):
+ flags |= self.flags
if kwargs.get('incomplete_input', True) is False:
flags &= ~PyCF_DONT_IMPLY_DEDENT
flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT
codeob = compile(source, filename, symbol, flags, True)
+ if flags & PyCF_ONLY_AST:
+ return codeob # this is an ast.Module in this case
for feature in _features:
if codeob.co_flags & feature.compiler_flag:
self.flags |= feature.compiler_flag
@@ -158,4 +151,4 @@ def __call__(self, source, filename="", symbol="single"):
syntax error (OverflowError and ValueError can be produced by
malformed literals).
"""
- return _maybe_compile(self.compiler, source, filename, symbol)
+ return _maybe_compile(self.compiler, source, filename, symbol, flags=self.compiler.flags)
diff --git a/PythonLib/full/collections/__init__.py b/PythonLib/full/collections/__init__.py
index 5f000b5f2..803de0c67 100644
--- a/PythonLib/full/collections/__init__.py
+++ b/PythonLib/full/collections/__init__.py
@@ -29,6 +29,9 @@
import _collections_abc
import sys as _sys
+_sys.modules['collections.abc'] = _collections_abc
+abc = _collections_abc
+
from itertools import chain as _chain
from itertools import repeat as _repeat
from itertools import starmap as _starmap
@@ -46,7 +49,8 @@
_collections_abc.MutableSequence.register(deque)
try:
- from _collections import _deque_iterator
+ # Expose _deque_iterator to support pickling deque iterators
+ from _collections import _deque_iterator # noqa: F401
except ImportError:
pass
@@ -55,6 +59,8 @@
except ImportError:
pass
+heapq = None # Lazily imported
+
################################################################################
### OrderedDict
@@ -457,7 +463,7 @@ def _make(cls, iterable):
def _replace(self, /, **kwds):
result = self._make(_map(kwds.pop, field_names, self))
if kwds:
- raise ValueError(f'Got unexpected field names: {list(kwds)!r}')
+ raise TypeError(f'Got unexpected field names: {list(kwds)!r}')
return result
_replace.__doc__ = (f'Return a new {typename} object replacing specified '
@@ -495,6 +501,7 @@ def __getnewargs__(self):
'_field_defaults': field_defaults,
'__new__': __new__,
'_make': _make,
+ '__replace__': _replace,
'_replace': _replace,
'__repr__': __repr__,
'_asdict': _asdict,
@@ -588,7 +595,7 @@ class Counter(dict):
# References:
# http://en.wikipedia.org/wiki/Multiset
# http://www.gnu.org/software/smalltalk/manual-base/html_node/Bag.html
- # http://www.demo2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
+ # http://www.java2s.com/Tutorial/Cpp/0380__set-multiset/Catalog0380__set-multiset.htm
# http://code.activestate.com/recipes/259174/
# Knuth, TAOCP Vol. II section 4.6.3
@@ -628,7 +635,10 @@ def most_common(self, n=None):
return sorted(self.items(), key=_itemgetter(1), reverse=True)
# Lazy import to speedup Python startup time
- import heapq
+ global heapq
+ if heapq is None:
+ import heapq
+
return heapq.nlargest(n, self.items(), key=_itemgetter(1))
def elements(self):
@@ -1015,7 +1025,7 @@ def __getitem__(self, key):
return self.__missing__(key) # support subclasses that define __missing__
def get(self, key, default=None):
- return self[key] if key in self else default
+ return self[key] if key in self else default # needs to make use of __contains__
def __len__(self):
return len(set().union(*self.maps)) # reuses stored hash values if possible
@@ -1027,7 +1037,10 @@ def __iter__(self):
return iter(d)
def __contains__(self, key):
- return any(key in m for m in self.maps)
+ for mapping in self.maps:
+ if key in mapping:
+ return True
+ return False
def __bool__(self):
return any(self.maps)
@@ -1037,9 +1050,9 @@ def __repr__(self):
return f'{self.__class__.__name__}({", ".join(map(repr, self.maps))})'
@classmethod
- def fromkeys(cls, iterable, *args):
- 'Create a ChainMap with a single dict created from the iterable.'
- return cls(dict.fromkeys(iterable, *args))
+ def fromkeys(cls, iterable, value=None, /):
+ 'Create a new ChainMap with keys from iterable and values set to value.'
+ return cls(dict.fromkeys(iterable, value))
def copy(self):
'New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]'
@@ -1482,6 +1495,8 @@ def format_map(self, mapping):
return self.data.format_map(mapping)
def index(self, sub, start=0, end=_sys.maxsize):
+ if isinstance(sub, UserString):
+ sub = sub.data
return self.data.index(sub, start, end)
def isalpha(self):
@@ -1550,6 +1565,8 @@ def rfind(self, sub, start=0, end=_sys.maxsize):
return self.data.rfind(sub, start, end)
def rindex(self, sub, start=0, end=_sys.maxsize):
+ if isinstance(sub, UserString):
+ sub = sub.data
return self.data.rindex(sub, start, end)
def rjust(self, width, *args):
diff --git a/PythonLib/full/collections/abc.py b/PythonLib/full/collections/abc.py
deleted file mode 100644
index 86ca8b8a8..000000000
--- a/PythonLib/full/collections/abc.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from _collections_abc import *
-from _collections_abc import __all__
-from _collections_abc import _CallableGenericAlias
diff --git a/PythonLib/full/compileall.py b/PythonLib/full/compileall.py
index d394156ce..67fe37045 100644
--- a/PythonLib/full/compileall.py
+++ b/PythonLib/full/compileall.py
@@ -116,7 +116,8 @@ def compile_dir(dir, maxlevels=None, ddir=None, force=False,
prependdir=prependdir,
limit_sl_dest=limit_sl_dest,
hardlink_dupes=hardlink_dupes),
- files)
+ files,
+ chunksize=4)
success = min(results, default=True)
else:
for file in files:
@@ -172,13 +173,13 @@ def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0,
if stripdir is not None:
fullname_parts = fullname.split(os.path.sep)
stripdir_parts = stripdir.split(os.path.sep)
- ddir_parts = list(fullname_parts)
- for spart, opart in zip(stripdir_parts, fullname_parts):
- if spart == opart:
- ddir_parts.remove(spart)
-
- dfile = os.path.join(*ddir_parts)
+ if stripdir_parts != fullname_parts[:len(stripdir_parts)]:
+ if quiet < 2:
+ print("The stripdir path {!r} is not a valid prefix for "
+ "source path {!r}; ignoring".format(stripdir, fullname))
+ else:
+ dfile = os.path.join(*fullname_parts[len(stripdir_parts):])
if prependdir is not None:
if dfile is None:
@@ -316,7 +317,9 @@ def main():
import argparse
parser = argparse.ArgumentParser(
- description='Utilities to support installing Python libraries.')
+ description='Utilities to support installing Python libraries.',
+ color=True,
+ )
parser.add_argument('-l', action='store_const', const=0,
default=None, dest='maxlevels',
help="don't recurse into subdirectories")
diff --git a/PythonLib/full/compression/__init__.py b/PythonLib/full/compression/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/PythonLib/full/compression/_common/__init__.py b/PythonLib/full/compression/_common/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/PythonLib/full/compression/_common/_streams.py b/PythonLib/full/compression/_common/_streams.py
new file mode 100644
index 000000000..9f367d4e3
--- /dev/null
+++ b/PythonLib/full/compression/_common/_streams.py
@@ -0,0 +1,162 @@
+"""Internal classes used by compression modules"""
+
+import io
+import sys
+
+BUFFER_SIZE = io.DEFAULT_BUFFER_SIZE # Compressed data read chunk size
+
+
+class BaseStream(io.BufferedIOBase):
+ """Mode-checking helper functions."""
+
+ def _check_not_closed(self):
+ if self.closed:
+ raise ValueError("I/O operation on closed file")
+
+ def _check_can_read(self):
+ if not self.readable():
+ raise io.UnsupportedOperation("File not open for reading")
+
+ def _check_can_write(self):
+ if not self.writable():
+ raise io.UnsupportedOperation("File not open for writing")
+
+ def _check_can_seek(self):
+ if not self.readable():
+ raise io.UnsupportedOperation("Seeking is only supported "
+ "on files open for reading")
+ if not self.seekable():
+ raise io.UnsupportedOperation("The underlying file object "
+ "does not support seeking")
+
+
+class DecompressReader(io.RawIOBase):
+ """Adapts the decompressor API to a RawIOBase reader API"""
+
+ def readable(self):
+ return True
+
+ def __init__(self, fp, decomp_factory, trailing_error=(), **decomp_args):
+ self._fp = fp
+ self._eof = False
+ self._pos = 0 # Current offset in decompressed stream
+
+ # Set to size of decompressed stream once it is known, for SEEK_END
+ self._size = -1
+
+ # Save the decompressor factory and arguments.
+ # If the file contains multiple compressed streams, each
+ # stream will need a separate decompressor object. A new decompressor
+ # object is also needed when implementing a backwards seek().
+ self._decomp_factory = decomp_factory
+ self._decomp_args = decomp_args
+ self._decompressor = self._decomp_factory(**self._decomp_args)
+
+ # Exception class to catch from decompressor signifying invalid
+ # trailing data to ignore
+ self._trailing_error = trailing_error
+
+ def close(self):
+ self._decompressor = None
+ return super().close()
+
+ def seekable(self):
+ return self._fp.seekable()
+
+ def readinto(self, b):
+ with memoryview(b) as view, view.cast("B") as byte_view:
+ data = self.read(len(byte_view))
+ byte_view[:len(data)] = data
+ return len(data)
+
+ def read(self, size=-1):
+ if size < 0:
+ return self.readall()
+
+ if not size or self._eof:
+ return b""
+ data = None # Default if EOF is encountered
+ # Depending on the input data, our call to the decompressor may not
+ # return any data. In this case, try again after reading another block.
+ while True:
+ if self._decompressor.eof:
+ rawblock = (self._decompressor.unused_data or
+ self._fp.read(BUFFER_SIZE))
+ if not rawblock:
+ break
+ # Continue to next stream.
+ self._decompressor = self._decomp_factory(
+ **self._decomp_args)
+ try:
+ data = self._decompressor.decompress(rawblock, size)
+ except self._trailing_error:
+ # Trailing data isn't a valid compressed stream; ignore it.
+ break
+ else:
+ if self._decompressor.needs_input:
+ rawblock = self._fp.read(BUFFER_SIZE)
+ if not rawblock:
+ raise EOFError("Compressed file ended before the "
+ "end-of-stream marker was reached")
+ else:
+ rawblock = b""
+ data = self._decompressor.decompress(rawblock, size)
+ if data:
+ break
+ if not data:
+ self._eof = True
+ self._size = self._pos
+ return b""
+ self._pos += len(data)
+ return data
+
+ def readall(self):
+ chunks = []
+ # sys.maxsize means the max length of output buffer is unlimited,
+ # so that the whole input buffer can be decompressed within one
+ # .decompress() call.
+ while data := self.read(sys.maxsize):
+ chunks.append(data)
+
+ return b"".join(chunks)
+
+ # Rewind the file to the beginning of the data stream.
+ def _rewind(self):
+ self._fp.seek(0)
+ self._eof = False
+ self._pos = 0
+ self._decompressor = self._decomp_factory(**self._decomp_args)
+
+ def seek(self, offset, whence=io.SEEK_SET):
+ # Recalculate offset as an absolute file position.
+ if whence == io.SEEK_SET:
+ pass
+ elif whence == io.SEEK_CUR:
+ offset = self._pos + offset
+ elif whence == io.SEEK_END:
+ # Seeking relative to EOF - we need to know the file's size.
+ if self._size < 0:
+ while self.read(io.DEFAULT_BUFFER_SIZE):
+ pass
+ offset = self._size + offset
+ else:
+ raise ValueError("Invalid value for whence: {}".format(whence))
+
+ # Make it so that offset is the number of bytes to skip forward.
+ if offset < self._pos:
+ self._rewind()
+ else:
+ offset -= self._pos
+
+ # Read and discard data until we reach the desired position.
+ while offset > 0:
+ data = self.read(min(io.DEFAULT_BUFFER_SIZE, offset))
+ if not data:
+ break
+ offset -= len(data)
+
+ return self._pos
+
+ def tell(self):
+ """Return the current file position."""
+ return self._pos
diff --git a/PythonLib/full/compression/bz2.py b/PythonLib/full/compression/bz2.py
new file mode 100644
index 000000000..16815d6cd
--- /dev/null
+++ b/PythonLib/full/compression/bz2.py
@@ -0,0 +1,5 @@
+import bz2
+__doc__ = bz2.__doc__
+del bz2
+
+from bz2 import *
diff --git a/PythonLib/full/compression/gzip.py b/PythonLib/full/compression/gzip.py
new file mode 100644
index 000000000..552f48f94
--- /dev/null
+++ b/PythonLib/full/compression/gzip.py
@@ -0,0 +1,5 @@
+import gzip
+__doc__ = gzip.__doc__
+del gzip
+
+from gzip import *
diff --git a/PythonLib/full/compression/lzma.py b/PythonLib/full/compression/lzma.py
new file mode 100644
index 000000000..b4bc7ccb1
--- /dev/null
+++ b/PythonLib/full/compression/lzma.py
@@ -0,0 +1,5 @@
+import lzma
+__doc__ = lzma.__doc__
+del lzma
+
+from lzma import *
diff --git a/PythonLib/full/compression/zlib.py b/PythonLib/full/compression/zlib.py
new file mode 100644
index 000000000..3aa7e2db9
--- /dev/null
+++ b/PythonLib/full/compression/zlib.py
@@ -0,0 +1,5 @@
+import zlib
+__doc__ = zlib.__doc__
+del zlib
+
+from zlib import *
diff --git a/PythonLib/full/compression/zstd/__init__.py b/PythonLib/full/compression/zstd/__init__.py
new file mode 100644
index 000000000..84b25914b
--- /dev/null
+++ b/PythonLib/full/compression/zstd/__init__.py
@@ -0,0 +1,242 @@
+"""Python bindings to the Zstandard (zstd) compression library (RFC-8878)."""
+
+__all__ = (
+ # compression.zstd
+ 'COMPRESSION_LEVEL_DEFAULT',
+ 'compress',
+ 'CompressionParameter',
+ 'decompress',
+ 'DecompressionParameter',
+ 'finalize_dict',
+ 'get_frame_info',
+ 'Strategy',
+ 'train_dict',
+
+ # compression.zstd._zstdfile
+ 'open',
+ 'ZstdFile',
+
+ # _zstd
+ 'get_frame_size',
+ 'zstd_version',
+ 'zstd_version_info',
+ 'ZstdCompressor',
+ 'ZstdDecompressor',
+ 'ZstdDict',
+ 'ZstdError',
+)
+
+import _zstd
+import enum
+from _zstd import (ZstdCompressor, ZstdDecompressor, ZstdDict, ZstdError,
+ get_frame_size, zstd_version)
+from compression.zstd._zstdfile import ZstdFile, open, _nbytes
+
+# zstd_version_number is (MAJOR * 100 * 100 + MINOR * 100 + RELEASE)
+zstd_version_info = (*divmod(_zstd.zstd_version_number // 100, 100),
+ _zstd.zstd_version_number % 100)
+"""Version number of the runtime zstd library as a tuple of integers."""
+
+COMPRESSION_LEVEL_DEFAULT = _zstd.ZSTD_CLEVEL_DEFAULT
+"""The default compression level for Zstandard, currently '3'."""
+
+
+class FrameInfo:
+ """Information about a Zstandard frame."""
+
+ __slots__ = 'decompressed_size', 'dictionary_id'
+
+ def __init__(self, decompressed_size, dictionary_id):
+ super().__setattr__('decompressed_size', decompressed_size)
+ super().__setattr__('dictionary_id', dictionary_id)
+
+ def __repr__(self):
+ return (f'FrameInfo(decompressed_size={self.decompressed_size}, '
+ f'dictionary_id={self.dictionary_id})')
+
+ def __setattr__(self, name, _):
+ raise AttributeError(f"can't set attribute {name!r}")
+
+
+def get_frame_info(frame_buffer):
+ """Get Zstandard frame information from a frame header.
+
+ *frame_buffer* is a bytes-like object. It should start from the beginning
+ of a frame, and needs to include at least the frame header (6 to 18 bytes).
+
+ The returned FrameInfo object has two attributes.
+ 'decompressed_size' is the size in bytes of the data in the frame when
+ decompressed, or None when the decompressed size is unknown.
+ 'dictionary_id' is an int in the range (0, 2**32). The special value 0
+ means that the dictionary ID was not recorded in the frame header,
+ the frame may or may not need a dictionary to be decoded,
+ and the ID of such a dictionary is not specified.
+ """
+ return FrameInfo(*_zstd.get_frame_info(frame_buffer))
+
+
+def train_dict(samples, dict_size):
+ """Return a ZstdDict representing a trained Zstandard dictionary.
+
+ *samples* is an iterable of samples, where a sample is a bytes-like
+ object representing a file.
+
+ *dict_size* is the dictionary's maximum size, in bytes.
+ """
+ if not isinstance(dict_size, int):
+ ds_cls = type(dict_size).__qualname__
+ raise TypeError(f'dict_size must be an int object, not {ds_cls!r}.')
+
+ samples = tuple(samples)
+ chunks = b''.join(samples)
+ chunk_sizes = tuple(_nbytes(sample) for sample in samples)
+ if not chunks:
+ raise ValueError("samples contained no data; can't train dictionary.")
+ dict_content = _zstd.train_dict(chunks, chunk_sizes, dict_size)
+ return ZstdDict(dict_content)
+
+
+def finalize_dict(zstd_dict, /, samples, dict_size, level):
+ """Return a ZstdDict representing a finalized Zstandard dictionary.
+
+ Given a custom content as a basis for dictionary, and a set of samples,
+ finalize *zstd_dict* by adding headers and statistics according to the
+ Zstandard dictionary format.
+
+ You may compose an effective dictionary content by hand, which is used as
+ basis dictionary, and use some samples to finalize a dictionary. The basis
+ dictionary may be a "raw content" dictionary. See *is_raw* in ZstdDict.
+
+ *samples* is an iterable of samples, where a sample is a bytes-like object
+ representing a file.
+ *dict_size* is the dictionary's maximum size, in bytes.
+ *level* is the expected compression level. The statistics for each
+ compression level differ, so tuning the dictionary to the compression level
+ can provide improvements.
+ """
+
+ if not isinstance(zstd_dict, ZstdDict):
+ raise TypeError('zstd_dict argument should be a ZstdDict object.')
+ if not isinstance(dict_size, int):
+ raise TypeError('dict_size argument should be an int object.')
+ if not isinstance(level, int):
+ raise TypeError('level argument should be an int object.')
+
+ samples = tuple(samples)
+ chunks = b''.join(samples)
+ chunk_sizes = tuple(_nbytes(sample) for sample in samples)
+ if not chunks:
+ raise ValueError("The samples are empty content, can't finalize the "
+ "dictionary.")
+ dict_content = _zstd.finalize_dict(zstd_dict.dict_content, chunks,
+ chunk_sizes, dict_size, level)
+ return ZstdDict(dict_content)
+
+
+def compress(data, level=None, options=None, zstd_dict=None):
+ """Return Zstandard compressed *data* as bytes.
+
+ *level* is an int specifying the compression level to use, defaulting to
+ COMPRESSION_LEVEL_DEFAULT ('3').
+ *options* is a dict object that contains advanced compression
+ parameters. See CompressionParameter for more on options.
+ *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See
+ the function train_dict for how to train a ZstdDict on sample data.
+
+ For incremental compression, use a ZstdCompressor instead.
+ """
+ comp = ZstdCompressor(level=level, options=options, zstd_dict=zstd_dict)
+ return comp.compress(data, mode=ZstdCompressor.FLUSH_FRAME)
+
+
+def decompress(data, zstd_dict=None, options=None):
+ """Decompress one or more frames of Zstandard compressed *data*.
+
+ *zstd_dict* is a ZstdDict object, a pre-trained Zstandard dictionary. See
+ the function train_dict for how to train a ZstdDict on sample data.
+ *options* is a dict object that contains advanced compression
+ parameters. See DecompressionParameter for more on options.
+
+ For incremental decompression, use a ZstdDecompressor instead.
+ """
+ results = []
+ while True:
+ decomp = ZstdDecompressor(options=options, zstd_dict=zstd_dict)
+ results.append(decomp.decompress(data))
+ if not decomp.eof:
+ raise ZstdError('Compressed data ended before the '
+ 'end-of-stream marker was reached')
+ data = decomp.unused_data
+ if not data:
+ break
+ return b''.join(results)
+
+
+class CompressionParameter(enum.IntEnum):
+ """Compression parameters."""
+
+ compression_level = _zstd.ZSTD_c_compressionLevel
+ window_log = _zstd.ZSTD_c_windowLog
+ hash_log = _zstd.ZSTD_c_hashLog
+ chain_log = _zstd.ZSTD_c_chainLog
+ search_log = _zstd.ZSTD_c_searchLog
+ min_match = _zstd.ZSTD_c_minMatch
+ target_length = _zstd.ZSTD_c_targetLength
+ strategy = _zstd.ZSTD_c_strategy
+
+ enable_long_distance_matching = _zstd.ZSTD_c_enableLongDistanceMatching
+ ldm_hash_log = _zstd.ZSTD_c_ldmHashLog
+ ldm_min_match = _zstd.ZSTD_c_ldmMinMatch
+ ldm_bucket_size_log = _zstd.ZSTD_c_ldmBucketSizeLog
+ ldm_hash_rate_log = _zstd.ZSTD_c_ldmHashRateLog
+
+ content_size_flag = _zstd.ZSTD_c_contentSizeFlag
+ checksum_flag = _zstd.ZSTD_c_checksumFlag
+ dict_id_flag = _zstd.ZSTD_c_dictIDFlag
+
+ nb_workers = _zstd.ZSTD_c_nbWorkers
+ job_size = _zstd.ZSTD_c_jobSize
+ overlap_log = _zstd.ZSTD_c_overlapLog
+
+ def bounds(self):
+ """Return the (lower, upper) int bounds of a compression parameter.
+
+ Both the lower and upper bounds are inclusive.
+ """
+ return _zstd.get_param_bounds(self.value, is_compress=True)
+
+
+class DecompressionParameter(enum.IntEnum):
+ """Decompression parameters."""
+
+ window_log_max = _zstd.ZSTD_d_windowLogMax
+
+ def bounds(self):
+ """Return the (lower, upper) int bounds of a decompression parameter.
+
+ Both the lower and upper bounds are inclusive.
+ """
+ return _zstd.get_param_bounds(self.value, is_compress=False)
+
+
+class Strategy(enum.IntEnum):
+ """Compression strategies, listed from fastest to strongest.
+
+ Note that new strategies might be added in the future.
+ Only the order (from fast to strong) is guaranteed,
+ the numeric value might change.
+ """
+
+ fast = _zstd.ZSTD_fast
+ dfast = _zstd.ZSTD_dfast
+ greedy = _zstd.ZSTD_greedy
+ lazy = _zstd.ZSTD_lazy
+ lazy2 = _zstd.ZSTD_lazy2
+ btlazy2 = _zstd.ZSTD_btlazy2
+ btopt = _zstd.ZSTD_btopt
+ btultra = _zstd.ZSTD_btultra
+ btultra2 = _zstd.ZSTD_btultra2
+
+
+# Check validity of the CompressionParameter & DecompressionParameter types
+_zstd.set_parameter_types(CompressionParameter, DecompressionParameter)
diff --git a/PythonLib/full/compression/zstd/_zstdfile.py b/PythonLib/full/compression/zstd/_zstdfile.py
new file mode 100644
index 000000000..d709f5efc
--- /dev/null
+++ b/PythonLib/full/compression/zstd/_zstdfile.py
@@ -0,0 +1,345 @@
+import io
+from os import PathLike
+from _zstd import ZstdCompressor, ZstdDecompressor, ZSTD_DStreamOutSize
+from compression._common import _streams
+
+__all__ = ('ZstdFile', 'open')
+
+_MODE_CLOSED = 0
+_MODE_READ = 1
+_MODE_WRITE = 2
+
+
+def _nbytes(dat, /):
+ if isinstance(dat, (bytes, bytearray)):
+ return len(dat)
+ with memoryview(dat) as mv:
+ return mv.nbytes
+
+
+class ZstdFile(_streams.BaseStream):
+ """A file-like object providing transparent Zstandard (de)compression.
+
+ A ZstdFile can act as a wrapper for an existing file object, or refer
+ directly to a named file on disk.
+
+ ZstdFile provides a *binary* file interface. Data is read and returned as
+ bytes, and may only be written to objects that support the Buffer Protocol.
+ """
+
+ FLUSH_BLOCK = ZstdCompressor.FLUSH_BLOCK
+ FLUSH_FRAME = ZstdCompressor.FLUSH_FRAME
+
+ def __init__(self, file, /, mode='r', *,
+ level=None, options=None, zstd_dict=None):
+ """Open a Zstandard compressed file in binary mode.
+
+ *file* can be either an file-like object, or a file name to open.
+
+ *mode* can be 'r' for reading (default), 'w' for (over)writing, 'x' for
+ creating exclusively, or 'a' for appending. These can equivalently be
+ given as 'rb', 'wb', 'xb' and 'ab' respectively.
+
+ *level* is an optional int specifying the compression level to use,
+ or COMPRESSION_LEVEL_DEFAULT if not given.
+
+ *options* is an optional dict for advanced compression parameters.
+ See CompressionParameter and DecompressionParameter for the possible
+ options.
+
+ *zstd_dict* is an optional ZstdDict object, a pre-trained Zstandard
+ dictionary. See train_dict() to train ZstdDict on sample data.
+ """
+ self._fp = None
+ self._close_fp = False
+ self._mode = _MODE_CLOSED
+ self._buffer = None
+
+ if not isinstance(mode, str):
+ raise ValueError('mode must be a str')
+ if options is not None and not isinstance(options, dict):
+ raise TypeError('options must be a dict or None')
+ mode = mode.removesuffix('b') # handle rb, wb, xb, ab
+ if mode == 'r':
+ if level is not None:
+ raise TypeError('level is illegal in read mode')
+ self._mode = _MODE_READ
+ elif mode in {'w', 'a', 'x'}:
+ if level is not None and not isinstance(level, int):
+ raise TypeError('level must be int or None')
+ self._mode = _MODE_WRITE
+ self._compressor = ZstdCompressor(level=level, options=options,
+ zstd_dict=zstd_dict)
+ self._pos = 0
+ else:
+ raise ValueError(f'Invalid mode: {mode!r}')
+
+ if isinstance(file, (str, bytes, PathLike)):
+ self._fp = io.open(file, f'{mode}b')
+ self._close_fp = True
+ elif ((mode == 'r' and hasattr(file, 'read'))
+ or (mode != 'r' and hasattr(file, 'write'))):
+ self._fp = file
+ else:
+ raise TypeError('file must be a file-like object '
+ 'or a str, bytes, or PathLike object')
+
+ if self._mode == _MODE_READ:
+ raw = _streams.DecompressReader(
+ self._fp,
+ ZstdDecompressor,
+ zstd_dict=zstd_dict,
+ options=options,
+ )
+ self._buffer = io.BufferedReader(raw)
+
+ def close(self):
+ """Flush and close the file.
+
+ May be called multiple times. Once the file has been closed,
+ any other operation on it will raise ValueError.
+ """
+ if self._fp is None:
+ return
+ try:
+ if self._mode == _MODE_READ:
+ if getattr(self, '_buffer', None):
+ self._buffer.close()
+ self._buffer = None
+ elif self._mode == _MODE_WRITE:
+ self.flush(self.FLUSH_FRAME)
+ self._compressor = None
+ finally:
+ self._mode = _MODE_CLOSED
+ try:
+ if self._close_fp:
+ self._fp.close()
+ finally:
+ self._fp = None
+ self._close_fp = False
+
+ def write(self, data, /):
+ """Write a bytes-like object *data* to the file.
+
+ Returns the number of uncompressed bytes written, which is
+ always the length of data in bytes. Note that due to buffering,
+ the file on disk may not reflect the data written until .flush()
+ or .close() is called.
+ """
+ self._check_can_write()
+
+ length = _nbytes(data)
+
+ compressed = self._compressor.compress(data)
+ self._fp.write(compressed)
+ self._pos += length
+ return length
+
+ def flush(self, mode=FLUSH_BLOCK):
+ """Flush remaining data to the underlying stream.
+
+ The mode argument can be FLUSH_BLOCK or FLUSH_FRAME. Abuse of this
+ method will reduce compression ratio, use it only when necessary.
+
+ If the program is interrupted afterwards, all data can be recovered.
+ To ensure saving to disk, also need to use os.fsync(fd).
+
+ This method does nothing in reading mode.
+ """
+ if self._mode == _MODE_READ:
+ return
+ self._check_not_closed()
+ if mode not in {self.FLUSH_BLOCK, self.FLUSH_FRAME}:
+ raise ValueError('Invalid mode argument, expected either '
+ 'ZstdFile.FLUSH_FRAME or '
+ 'ZstdFile.FLUSH_BLOCK')
+ if self._compressor.last_mode == mode:
+ return
+ # Flush zstd block/frame, and write.
+ data = self._compressor.flush(mode)
+ self._fp.write(data)
+ if hasattr(self._fp, 'flush'):
+ self._fp.flush()
+
+ def read(self, size=-1):
+ """Read up to size uncompressed bytes from the file.
+
+ If size is negative or omitted, read until EOF is reached.
+ Returns b'' if the file is already at EOF.
+ """
+ if size is None:
+ size = -1
+ self._check_can_read()
+ return self._buffer.read(size)
+
+ def read1(self, size=-1):
+ """Read up to size uncompressed bytes, while trying to avoid
+ making multiple reads from the underlying stream. Reads up to a
+ buffer's worth of data if size is negative.
+
+ Returns b'' if the file is at EOF.
+ """
+ self._check_can_read()
+ if size < 0:
+ # Note this should *not* be io.DEFAULT_BUFFER_SIZE.
+ # ZSTD_DStreamOutSize is the minimum amount to read guaranteeing
+ # a full block is read.
+ size = ZSTD_DStreamOutSize
+ return self._buffer.read1(size)
+
+ def readinto(self, b):
+ """Read bytes into b.
+
+ Returns the number of bytes read (0 for EOF).
+ """
+ self._check_can_read()
+ return self._buffer.readinto(b)
+
+ def readinto1(self, b):
+ """Read bytes into b, while trying to avoid making multiple reads
+ from the underlying stream.
+
+ Returns the number of bytes read (0 for EOF).
+ """
+ self._check_can_read()
+ return self._buffer.readinto1(b)
+
+ def readline(self, size=-1):
+ """Read a line of uncompressed bytes from the file.
+
+ The terminating newline (if present) is retained. If size is
+ non-negative, no more than size bytes will be read (in which
+ case the line may be incomplete). Returns b'' if already at EOF.
+ """
+ self._check_can_read()
+ return self._buffer.readline(size)
+
+ def seek(self, offset, whence=io.SEEK_SET):
+ """Change the file position.
+
+ The new position is specified by offset, relative to the
+ position indicated by whence. Possible values for whence are:
+
+ 0: start of stream (default): offset must not be negative
+ 1: current stream position
+ 2: end of stream; offset must not be positive
+
+ Returns the new file position.
+
+ Note that seeking is emulated, so depending on the arguments,
+ this operation may be extremely slow.
+ """
+ self._check_can_read()
+
+ # BufferedReader.seek() checks seekable
+ return self._buffer.seek(offset, whence)
+
+ def peek(self, size=-1):
+ """Return buffered data without advancing the file position.
+
+ Always returns at least one byte of data, unless at EOF.
+ The exact number of bytes returned is unspecified.
+ """
+ # Relies on the undocumented fact that BufferedReader.peek() always
+ # returns at least one byte (except at EOF)
+ self._check_can_read()
+ return self._buffer.peek(size)
+
+ def __next__(self):
+ if ret := self._buffer.readline():
+ return ret
+ raise StopIteration
+
+ def tell(self):
+ """Return the current file position."""
+ self._check_not_closed()
+ if self._mode == _MODE_READ:
+ return self._buffer.tell()
+ elif self._mode == _MODE_WRITE:
+ return self._pos
+
+ def fileno(self):
+ """Return the file descriptor for the underlying file."""
+ self._check_not_closed()
+ return self._fp.fileno()
+
+ @property
+ def name(self):
+ self._check_not_closed()
+ return self._fp.name
+
+ @property
+ def mode(self):
+ return 'wb' if self._mode == _MODE_WRITE else 'rb'
+
+ @property
+ def closed(self):
+ """True if this file is closed."""
+ return self._mode == _MODE_CLOSED
+
+ def seekable(self):
+ """Return whether the file supports seeking."""
+ return self.readable() and self._buffer.seekable()
+
+ def readable(self):
+ """Return whether the file was opened for reading."""
+ self._check_not_closed()
+ return self._mode == _MODE_READ
+
+ def writable(self):
+ """Return whether the file was opened for writing."""
+ self._check_not_closed()
+ return self._mode == _MODE_WRITE
+
+
+def open(file, /, mode='rb', *, level=None, options=None, zstd_dict=None,
+ encoding=None, errors=None, newline=None):
+ """Open a Zstandard compressed file in binary or text mode.
+
+ file can be either a file name (given as a str, bytes, or PathLike object),
+ in which case the named file is opened, or it can be an existing file object
+ to read from or write to.
+
+ The mode parameter can be 'r', 'rb' (default), 'w', 'wb', 'x', 'xb', 'a',
+ 'ab' for binary mode, or 'rt', 'wt', 'xt', 'at' for text mode.
+
+ The level, options, and zstd_dict parameters specify the settings the same
+ as ZstdFile.
+
+ When using read mode (decompression), the options parameter is a dict
+ representing advanced decompression options. The level parameter is not
+ supported in this case. When using write mode (compression), only one of
+ level, an int representing the compression level, or options, a dict
+ representing advanced compression options, may be passed. In both modes,
+ zstd_dict is a ZstdDict instance containing a trained Zstandard dictionary.
+
+ For binary mode, this function is equivalent to the ZstdFile constructor:
+ ZstdFile(filename, mode, ...). In this case, the encoding, errors and
+ newline parameters must not be provided.
+
+ For text mode, an ZstdFile object is created, and wrapped in an
+ io.TextIOWrapper instance with the specified encoding, error handling
+ behavior, and line ending(s).
+ """
+
+ text_mode = 't' in mode
+ mode = mode.replace('t', '')
+
+ if text_mode:
+ if 'b' in mode:
+ raise ValueError(f'Invalid mode: {mode!r}')
+ else:
+ if encoding is not None:
+ raise ValueError('Argument "encoding" not supported in binary mode')
+ if errors is not None:
+ raise ValueError('Argument "errors" not supported in binary mode')
+ if newline is not None:
+ raise ValueError('Argument "newline" not supported in binary mode')
+
+ binary_file = ZstdFile(file, mode, level=level, options=options,
+ zstd_dict=zstd_dict)
+
+ if text_mode:
+ return io.TextIOWrapper(binary_file, encoding, errors, newline)
+ else:
+ return binary_file
diff --git a/PythonLib/full/concurrent/futures/__init__.py b/PythonLib/full/concurrent/futures/__init__.py
index 72de617a5..d6ac4b3e0 100644
--- a/PythonLib/full/concurrent/futures/__init__.py
+++ b/PythonLib/full/concurrent/futures/__init__.py
@@ -17,7 +17,7 @@
wait,
as_completed)
-__all__ = (
+__all__ = [
'FIRST_COMPLETED',
'FIRST_EXCEPTION',
'ALL_COMPLETED',
@@ -31,24 +31,35 @@
'as_completed',
'ProcessPoolExecutor',
'ThreadPoolExecutor',
-)
+]
+
+
+try:
+ import _interpreters
+except ImportError:
+ _interpreters = None
+
+if _interpreters:
+ __all__.append('InterpreterPoolExecutor')
def __dir__():
- return __all__ + ('__author__', '__doc__')
+ return __all__ + ['__author__', '__doc__']
def __getattr__(name):
- global ProcessPoolExecutor, ThreadPoolExecutor
+ global ProcessPoolExecutor, ThreadPoolExecutor, InterpreterPoolExecutor
if name == 'ProcessPoolExecutor':
- from .process import ProcessPoolExecutor as pe
- ProcessPoolExecutor = pe
- return pe
+ from .process import ProcessPoolExecutor
+ return ProcessPoolExecutor
if name == 'ThreadPoolExecutor':
- from .thread import ThreadPoolExecutor as te
- ThreadPoolExecutor = te
- return te
+ from .thread import ThreadPoolExecutor
+ return ThreadPoolExecutor
+
+ if _interpreters and name == 'InterpreterPoolExecutor':
+ from .interpreter import InterpreterPoolExecutor
+ return InterpreterPoolExecutor
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/PythonLib/full/concurrent/futures/_base.py b/PythonLib/full/concurrent/futures/_base.py
index 6742a0775..d98b1ebdd 100644
--- a/PythonLib/full/concurrent/futures/_base.py
+++ b/PythonLib/full/concurrent/futures/_base.py
@@ -8,6 +8,8 @@
import threading
import time
import types
+import weakref
+from itertools import islice
FIRST_COMPLETED = 'FIRST_COMPLETED'
FIRST_EXCEPTION = 'FIRST_EXCEPTION'
@@ -23,14 +25,6 @@
CANCELLED_AND_NOTIFIED = 'CANCELLED_AND_NOTIFIED'
FINISHED = 'FINISHED'
-_FUTURE_STATES = [
- PENDING,
- RUNNING,
- CANCELLED,
- CANCELLED_AND_NOTIFIED,
- FINISHED
-]
-
_STATE_TO_DESCRIPTION_MAP = {
PENDING: "pending",
RUNNING: "running",
@@ -396,7 +390,7 @@ def done(self):
return self._state in [CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED]
def __get_result(self):
- if self._exception:
+ if self._exception is not None:
try:
raise self._exception
finally:
@@ -580,7 +574,7 @@ def submit(self, fn, /, *args, **kwargs):
"""
raise NotImplementedError()
- def map(self, fn, *iterables, timeout=None, chunksize=1):
+ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
"""Returns an iterator equivalent to map(fn, iter).
Args:
@@ -592,6 +586,11 @@ def map(self, fn, *iterables, timeout=None, chunksize=1):
before being passed to a child process. This argument is only
used by ProcessPoolExecutor; it is ignored by
ThreadPoolExecutor.
+ buffersize: The number of submitted tasks whose results have not
+ yet been yielded. If the buffer is full, iteration over the
+ iterables pauses until a result is yielded from the buffer.
+ If None, all input elements are eagerly collected, and a task is
+ submitted for each.
Returns:
An iterator equivalent to: map(func, *iterables) but the calls may
@@ -602,10 +601,25 @@ def map(self, fn, *iterables, timeout=None, chunksize=1):
before the given timeout.
Exception: If fn(*args) raises for any values.
"""
+ if buffersize is not None and not isinstance(buffersize, int):
+ raise TypeError("buffersize must be an integer or None")
+ if buffersize is not None and buffersize < 1:
+ raise ValueError("buffersize must be None or > 0")
+
if timeout is not None:
end_time = timeout + time.monotonic()
- fs = [self.submit(fn, *args) for args in zip(*iterables)]
+ zipped_iterables = zip(*iterables)
+ if buffersize:
+ fs = collections.deque(
+ self.submit(fn, *args) for args in islice(zipped_iterables, buffersize)
+ )
+ else:
+ fs = [self.submit(fn, *args) for args in zipped_iterables]
+
+ # Use a weak reference to ensure that the executor can be garbage
+ # collected independently of the result_iterator closure.
+ executor_weakref = weakref.ref(self)
# Yield must be hidden in closure so that the futures are submitted
# before the first iterator value is required.
@@ -614,6 +628,12 @@ def result_iterator():
# reverse to keep finishing order
fs.reverse()
while fs:
+ if (
+ buffersize
+ and (executor := executor_weakref())
+ and (args := next(zipped_iterables, None))
+ ):
+ fs.appendleft(executor.submit(fn, *args))
# Careful not to keep a reference to the popped future
if timeout is None:
yield _result_or_cancel(fs.pop())
diff --git a/PythonLib/full/concurrent/futures/interpreter.py b/PythonLib/full/concurrent/futures/interpreter.py
new file mode 100644
index 000000000..85c1da2c7
--- /dev/null
+++ b/PythonLib/full/concurrent/futures/interpreter.py
@@ -0,0 +1,123 @@
+"""Implements InterpreterPoolExecutor."""
+
+from concurrent import interpreters
+import sys
+from . import thread as _thread
+import traceback
+
+
+def do_call(results, func, args, kwargs):
+ try:
+ return func(*args, **kwargs)
+ except BaseException as exc:
+ # Send the captured exception out on the results queue,
+ # but still leave it unhandled for the interpreter to handle.
+ try:
+ results.put(exc)
+ except interpreters.NotShareableError:
+ # The exception is not shareable.
+ print('exception is not shareable:', file=sys.stderr)
+ traceback.print_exception(exc)
+ results.put(None)
+ raise # re-raise
+
+
+class WorkerContext(_thread.WorkerContext):
+
+ @classmethod
+ def prepare(cls, initializer, initargs):
+ def resolve_task(fn, args, kwargs):
+ if isinstance(fn, str):
+ # XXX Circle back to this later.
+ raise TypeError('scripts not supported')
+ else:
+ task = (fn, args, kwargs)
+ return task
+
+ if initializer is not None:
+ try:
+ initdata = resolve_task(initializer, initargs, {})
+ except ValueError:
+ if isinstance(initializer, str) and initargs:
+ raise ValueError(f'an initializer script does not take args, got {initargs!r}')
+ raise # re-raise
+ else:
+ initdata = None
+ def create_context():
+ return cls(initdata)
+ return create_context, resolve_task
+
+ def __init__(self, initdata):
+ self.initdata = initdata
+ self.interp = None
+ self.results = None
+
+ def __del__(self):
+ if self.interp is not None:
+ self.finalize()
+
+ def initialize(self):
+ assert self.interp is None, self.interp
+ self.interp = interpreters.create()
+ try:
+ maxsize = 0
+ self.results = interpreters.create_queue(maxsize)
+
+ if self.initdata:
+ self.run(self.initdata)
+ except BaseException:
+ self.finalize()
+ raise # re-raise
+
+ def finalize(self):
+ interp = self.interp
+ results = self.results
+ self.results = None
+ self.interp = None
+ if results is not None:
+ del results
+ if interp is not None:
+ interp.close()
+
+ def run(self, task):
+ try:
+ return self.interp.call(do_call, self.results, *task)
+ except interpreters.ExecutionFailed as wrapper:
+ # Wait for the exception data to show up.
+ exc = self.results.get()
+ if exc is None:
+ # The exception must have been not shareable.
+ raise # re-raise
+ raise exc from wrapper
+
+
+class BrokenInterpreterPool(_thread.BrokenThreadPool):
+ """
+ Raised when a worker thread in an InterpreterPoolExecutor failed initializing.
+ """
+
+
+class InterpreterPoolExecutor(_thread.ThreadPoolExecutor):
+
+ BROKEN = BrokenInterpreterPool
+
+ @classmethod
+ def prepare_context(cls, initializer, initargs):
+ return WorkerContext.prepare(initializer, initargs)
+
+ def __init__(self, max_workers=None, thread_name_prefix='',
+ initializer=None, initargs=()):
+ """Initializes a new InterpreterPoolExecutor instance.
+
+ Args:
+ max_workers: The maximum number of interpreters that can be used to
+ execute the given calls.
+ thread_name_prefix: An optional name prefix to give our threads.
+ initializer: A callable or script used to initialize
+ each worker interpreter.
+ initargs: A tuple of arguments to pass to the initializer.
+ """
+ thread_name_prefix = (thread_name_prefix or
+ (f"InterpreterPoolExecutor-{self._counter()}"))
+ super().__init__(max_workers, thread_name_prefix,
+ initializer, initargs)
diff --git a/PythonLib/full/concurrent/futures/process.py b/PythonLib/full/concurrent/futures/process.py
index ff7c17efa..a14650bf5 100644
--- a/PythonLib/full/concurrent/futures/process.py
+++ b/PythonLib/full/concurrent/futures/process.py
@@ -191,16 +191,6 @@ def _on_queue_feeder_error(self, e, obj):
super()._on_queue_feeder_error(e, obj)
-def _get_chunks(*iterables, chunksize):
- """ Iterates over zip()ed iterables in chunks. """
- it = zip(*iterables)
- while True:
- chunk = tuple(itertools.islice(it, chunksize))
- if not chunk:
- return
- yield chunk
-
-
def _process_chunk(fn, chunk):
""" Processes a chunk of an iterable passed to map.
@@ -306,8 +296,9 @@ def __init__(self, executor):
# will wake up the queue management thread so that it can terminate
# if there is no pending work item.
def weakref_cb(_,
- thread_wakeup=self.thread_wakeup):
- mp.util.debug('Executor collected: triggering callback for'
+ thread_wakeup=self.thread_wakeup,
+ mp_util_debug=mp.util.debug):
+ mp_util_debug('Executor collected: triggering callback for'
' QueueManager wakeup')
thread_wakeup.wakeup()
@@ -445,24 +436,14 @@ def process_result_item(self, result_item):
# Process the received a result_item. This can be either the PID of a
# worker that exited gracefully or a _ResultItem
- if isinstance(result_item, int):
- # Clean shutdown of a worker using its PID
- # (avoids marking the executor broken)
- assert self.is_shutting_down()
- p = self.processes.pop(result_item)
- p.join()
- if not self.processes:
- self.join_executor_internals()
- return
- else:
- # Received a _ResultItem so mark the future as completed.
- work_item = self.pending_work_items.pop(result_item.work_id, None)
- # work_item can be None if another process terminated (see above)
- if work_item is not None:
- if result_item.exception:
- work_item.future.set_exception(result_item.exception)
- else:
- work_item.future.set_result(result_item.result)
+ # Received a _ResultItem so mark the future as completed.
+ work_item = self.pending_work_items.pop(result_item.work_id, None)
+ # work_item can be None if another process terminated (see above)
+ if work_item is not None:
+ if result_item.exception is not None:
+ work_item.future.set_exception(result_item.exception)
+ else:
+ work_item.future.set_result(result_item.result)
def is_shutting_down(self):
# Check whether we should start shutting down the executor.
@@ -602,7 +583,7 @@ def _check_system_limits():
raise NotImplementedError(_system_limited)
_system_limits_checked = True
try:
- import multiprocessing.synchronize
+ import multiprocessing.synchronize # noqa: F401
except ImportError:
_system_limited = (
"This Python build lacks multiprocessing.synchronize, usually due "
@@ -645,6 +626,14 @@ class BrokenProcessPool(_base.BrokenExecutor):
while a future was in the running state.
"""
+_TERMINATE = "terminate"
+_KILL = "kill"
+
+_SHUTDOWN_CALLBACK_OPERATION = {
+ _TERMINATE,
+ _KILL
+}
+
class ProcessPoolExecutor(_base.Executor):
def __init__(self, max_workers=None, mp_context=None,
@@ -670,7 +659,7 @@ def __init__(self, max_workers=None, mp_context=None,
_check_system_limits()
if max_workers is None:
- self._max_workers = os.cpu_count() or 1
+ self._max_workers = os.process_cpu_count() or 1
if sys.platform == 'win32':
self._max_workers = min(_MAX_WINDOWS_WORKERS,
self._max_workers)
@@ -766,6 +755,11 @@ def _start_executor_manager_thread(self):
self._executor_manager_thread_wakeup
def _adjust_process_count(self):
+ # gh-132969: avoid error when state is reset and executor is still running,
+ # which will happen when shutdown(wait=False) is called.
+ if self._processes is None:
+ return
+
# if there's an idle process, we don't need to spawn a new one.
if self._idle_worker_semaphore.acquire(blocking=False):
return
@@ -824,7 +818,7 @@ def submit(self, fn, /, *args, **kwargs):
return f
submit.__doc__ = _base.Executor.submit.__doc__
- def map(self, fn, *iterables, timeout=None, chunksize=1):
+ def map(self, fn, *iterables, timeout=None, chunksize=1, buffersize=None):
"""Returns an iterator equivalent to map(fn, iter).
Args:
@@ -835,6 +829,11 @@ def map(self, fn, *iterables, timeout=None, chunksize=1):
chunksize: If greater than one, the iterables will be chopped into
chunks of size chunksize and submitted to the process pool.
If set to one, the items in the list will be sent one at a time.
+ buffersize: The number of submitted tasks whose results have not
+ yet been yielded. If the buffer is full, iteration over the
+ iterables pauses until a result is yielded from the buffer.
+ If None, all input elements are eagerly collected, and a task is
+ submitted for each.
Returns:
An iterator equivalent to: map(func, *iterables) but the calls may
@@ -849,8 +848,9 @@ def map(self, fn, *iterables, timeout=None, chunksize=1):
raise ValueError("chunksize must be >= 1.")
results = super().map(partial(_process_chunk, fn),
- _get_chunks(*iterables, chunksize=chunksize),
- timeout=timeout)
+ itertools.batched(zip(*iterables), chunksize),
+ timeout=timeout,
+ buffersize=buffersize)
return _chain_from_iterable_of_lists(results)
def shutdown(self, wait=True, *, cancel_futures=False):
@@ -874,3 +874,66 @@ def shutdown(self, wait=True, *, cancel_futures=False):
self._executor_manager_thread_wakeup = None
shutdown.__doc__ = _base.Executor.shutdown.__doc__
+
+ def _force_shutdown(self, operation):
+ """Attempts to terminate or kill the executor's workers based off the
+ given operation. Iterates through all of the current processes and
+ performs the relevant task if the process is still alive.
+
+ After terminating workers, the pool will be in a broken state
+ and no longer usable (for instance, new tasks should not be
+ submitted).
+ """
+ if operation not in _SHUTDOWN_CALLBACK_OPERATION:
+ raise ValueError(f"Unsupported operation: {operation!r}")
+
+ processes = {}
+ if self._processes:
+ processes = self._processes.copy()
+
+ # shutdown will invalidate ._processes, so we copy it right before
+ # calling. If we waited here, we would deadlock if a process decides not
+ # to exit.
+ self.shutdown(wait=False, cancel_futures=True)
+
+ if not processes:
+ return
+
+ for proc in processes.values():
+ try:
+ if not proc.is_alive():
+ continue
+ except ValueError:
+ # The process is already exited/closed out.
+ continue
+
+ try:
+ if operation == _TERMINATE:
+ proc.terminate()
+ elif operation == _KILL:
+ proc.kill()
+ except ProcessLookupError:
+ # The process just ended before our signal
+ continue
+
+ def terminate_workers(self):
+ """Attempts to terminate the executor's workers.
+ Iterates through all of the current worker processes and terminates
+ each one that is still alive.
+
+ After terminating workers, the pool will be in a broken state
+ and no longer usable (for instance, new tasks should not be
+ submitted).
+ """
+ return self._force_shutdown(operation=_TERMINATE)
+
+ def kill_workers(self):
+ """Attempts to kill the executor's workers.
+ Iterates through all of the current worker processes and kills
+ each one that is still alive.
+
+ After killing workers, the pool will be in a broken state
+ and no longer usable (for instance, new tasks should not be
+ submitted).
+ """
+ return self._force_shutdown(operation=_KILL)
diff --git a/PythonLib/full/concurrent/futures/thread.py b/PythonLib/full/concurrent/futures/thread.py
index 61dbff8a4..909359b64 100644
--- a/PythonLib/full/concurrent/futures/thread.py
+++ b/PythonLib/full/concurrent/futures/thread.py
@@ -44,19 +44,46 @@ def _python_exit():
os.register_at_fork(after_in_child=_threads_queues.clear)
+class WorkerContext:
+
+ @classmethod
+ def prepare(cls, initializer, initargs):
+ if initializer is not None:
+ if not callable(initializer):
+ raise TypeError("initializer must be a callable")
+ def create_context():
+ return cls(initializer, initargs)
+ def resolve_task(fn, args, kwargs):
+ return (fn, args, kwargs)
+ return create_context, resolve_task
+
+ def __init__(self, initializer, initargs):
+ self.initializer = initializer
+ self.initargs = initargs
+
+ def initialize(self):
+ if self.initializer is not None:
+ self.initializer(*self.initargs)
+
+ def finalize(self):
+ pass
+
+ def run(self, task):
+ fn, args, kwargs = task
+ return fn(*args, **kwargs)
+
+
class _WorkItem:
- def __init__(self, future, fn, args, kwargs):
+ def __init__(self, future, task):
self.future = future
- self.fn = fn
- self.args = args
- self.kwargs = kwargs
+ self.task = task
- def run(self):
+ def run(self, ctx):
if not self.future.set_running_or_notify_cancel():
return
try:
- result = self.fn(*self.args, **self.kwargs)
+ result = ctx.run(self.task)
except BaseException as exc:
self.future.set_exception(exc)
# Break a reference cycle with the exception 'exc'
@@ -67,16 +94,15 @@ def run(self):
__class_getitem__ = classmethod(types.GenericAlias)
-def _worker(executor_reference, work_queue, initializer, initargs):
- if initializer is not None:
- try:
- initializer(*initargs)
- except BaseException:
- _base.LOGGER.critical('Exception in initializer:', exc_info=True)
- executor = executor_reference()
- if executor is not None:
- executor._initializer_failed()
- return
+def _worker(executor_reference, ctx, work_queue):
+ try:
+ ctx.initialize()
+ except BaseException:
+ _base.LOGGER.critical('Exception in initializer:', exc_info=True)
+ executor = executor_reference()
+ if executor is not None:
+ executor._initializer_failed()
+ return
try:
while True:
try:
@@ -90,7 +116,7 @@ def _worker(executor_reference, work_queue, initializer, initargs):
work_item = work_queue.get(block=True)
if work_item is not None:
- work_item.run()
+ work_item.run(ctx)
# Delete references to object. See GH-60488
del work_item
continue
@@ -111,6 +137,8 @@ def _worker(executor_reference, work_queue, initializer, initargs):
del executor
except BaseException:
_base.LOGGER.critical('Exception in worker', exc_info=True)
+ finally:
+ ctx.finalize()
class BrokenThreadPool(_base.BrokenExecutor):
@@ -121,11 +149,17 @@ class BrokenThreadPool(_base.BrokenExecutor):
class ThreadPoolExecutor(_base.Executor):
+ BROKEN = BrokenThreadPool
+
# Used to assign unique thread names when thread_name_prefix is not supplied.
_counter = itertools.count().__next__
+ @classmethod
+ def prepare_context(cls, initializer, initargs):
+ return WorkerContext.prepare(initializer, initargs)
+
def __init__(self, max_workers=None, thread_name_prefix='',
- initializer=None, initargs=()):
+ initializer=None, initargs=(), **ctxkwargs):
"""Initializes a new ThreadPoolExecutor instance.
Args:
@@ -134,21 +168,23 @@ def __init__(self, max_workers=None, thread_name_prefix='',
thread_name_prefix: An optional name prefix to give our threads.
initializer: A callable used to initialize worker threads.
initargs: A tuple of arguments to pass to the initializer.
+ ctxkwargs: Additional arguments to cls.prepare_context().
"""
if max_workers is None:
# ThreadPoolExecutor is often used to:
# * CPU bound task which releases GIL
# * I/O bound task (which releases GIL, of course)
#
- # We use cpu_count + 4 for both types of tasks.
+ # We use process_cpu_count + 4 for both types of tasks.
# But we limit it to 32 to avoid consuming surprisingly large resource
# on many core machine.
- max_workers = min(32, (os.cpu_count() or 1) + 4)
+ max_workers = min(32, (os.process_cpu_count() or 1) + 4)
if max_workers <= 0:
raise ValueError("max_workers must be greater than 0")
- if initializer is not None and not callable(initializer):
- raise TypeError("initializer must be a callable")
+ (self._create_worker_context,
+ self._resolve_work_item_task,
+ ) = type(self).prepare_context(initializer, initargs, **ctxkwargs)
self._max_workers = max_workers
self._work_queue = queue.SimpleQueue()
@@ -159,13 +195,11 @@ def __init__(self, max_workers=None, thread_name_prefix='',
self._shutdown_lock = threading.Lock()
self._thread_name_prefix = (thread_name_prefix or
("ThreadPoolExecutor-%d" % self._counter()))
- self._initializer = initializer
- self._initargs = initargs
def submit(self, fn, /, *args, **kwargs):
with self._shutdown_lock, _global_shutdown_lock:
if self._broken:
- raise BrokenThreadPool(self._broken)
+ raise self.BROKEN(self._broken)
if self._shutdown:
raise RuntimeError('cannot schedule new futures after shutdown')
@@ -174,7 +208,8 @@ def submit(self, fn, /, *args, **kwargs):
'interpreter shutdown')
f = _base.Future()
- w = _WorkItem(f, fn, args, kwargs)
+ task = self._resolve_work_item_task(fn, args, kwargs)
+ w = _WorkItem(f, task)
self._work_queue.put(w)
self._adjust_thread_count()
@@ -197,9 +232,8 @@ def weakref_cb(_, q=self._work_queue):
num_threads)
t = threading.Thread(name=thread_name, target=_worker,
args=(weakref.ref(self, weakref_cb),
- self._work_queue,
- self._initializer,
- self._initargs))
+ self._create_worker_context(),
+ self._work_queue))
t.start()
self._threads.add(t)
_threads_queues[t] = self._work_queue
@@ -215,7 +249,7 @@ def _initializer_failed(self):
except queue.Empty:
break
if work_item is not None:
- work_item.future.set_exception(BrokenThreadPool(self._broken))
+ work_item.future.set_exception(self.BROKEN(self._broken))
def shutdown(self, wait=True, *, cancel_futures=False):
with self._shutdown_lock:
diff --git a/PythonLib/full/concurrent/interpreters/__init__.py b/PythonLib/full/concurrent/interpreters/__init__.py
new file mode 100644
index 000000000..ea4147ee9
--- /dev/null
+++ b/PythonLib/full/concurrent/interpreters/__init__.py
@@ -0,0 +1,247 @@
+"""Subinterpreters High Level Module."""
+
+import threading
+import weakref
+import _interpreters
+
+# aliases:
+from _interpreters import (
+ InterpreterError, InterpreterNotFoundError, NotShareableError,
+ is_shareable,
+)
+from ._queues import (
+ create as create_queue,
+ Queue, QueueEmpty, QueueFull,
+)
+
+
+__all__ = [
+ 'get_current', 'get_main', 'create', 'list_all', 'is_shareable',
+ 'Interpreter',
+ 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed',
+ 'NotShareableError',
+ 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull',
+]
+
+
+_EXEC_FAILURE_STR = """
+{superstr}
+
+Uncaught in the interpreter:
+
+{formatted}
+""".strip()
+
+class ExecutionFailed(InterpreterError):
+ """An unhandled exception happened during execution.
+
+ This is raised from Interpreter.exec() and Interpreter.call().
+ """
+
+ def __init__(self, excinfo):
+ msg = excinfo.formatted
+ if not msg:
+ if excinfo.type and excinfo.msg:
+ msg = f'{excinfo.type.__name__}: {excinfo.msg}'
+ else:
+ msg = excinfo.type.__name__ or excinfo.msg
+ super().__init__(msg)
+ self.excinfo = excinfo
+
+ def __str__(self):
+ try:
+ formatted = self.excinfo.errdisplay
+ except Exception:
+ return super().__str__()
+ else:
+ return _EXEC_FAILURE_STR.format(
+ superstr=super().__str__(),
+ formatted=formatted,
+ )
+
+
+def create():
+ """Return a new (idle) Python interpreter."""
+ id = _interpreters.create(reqrefs=True)
+ return Interpreter(id, _ownsref=True)
+
+
+def list_all():
+ """Return all existing interpreters."""
+ return [Interpreter(id, _whence=whence)
+ for id, whence in _interpreters.list_all(require_ready=True)]
+
+
+def get_current():
+ """Return the currently running interpreter."""
+ id, whence = _interpreters.get_current()
+ return Interpreter(id, _whence=whence)
+
+
+def get_main():
+ """Return the main interpreter."""
+ id, whence = _interpreters.get_main()
+ assert whence == _interpreters.WHENCE_RUNTIME, repr(whence)
+ return Interpreter(id, _whence=whence)
+
+
+_known = weakref.WeakValueDictionary()
+
+class Interpreter:
+ """A single Python interpreter.
+
+ Attributes:
+
+ "id" - the unique process-global ID number for the interpreter
+ "whence" - indicates where the interpreter was created
+
+ If the interpreter wasn't created by this module
+ then any method that modifies the interpreter will fail,
+ i.e. .close(), .prepare_main(), .exec(), and .call()
+ """
+
+ _WHENCE_TO_STR = {
+ _interpreters.WHENCE_UNKNOWN: 'unknown',
+ _interpreters.WHENCE_RUNTIME: 'runtime init',
+ _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API',
+ _interpreters.WHENCE_CAPI: 'C-API',
+ _interpreters.WHENCE_XI: 'cross-interpreter C-API',
+ _interpreters.WHENCE_STDLIB: '_interpreters module',
+ }
+
+ def __new__(cls, id, /, _whence=None, _ownsref=None):
+ # There is only one instance for any given ID.
+ if not isinstance(id, int):
+ raise TypeError(f'id must be an int, got {id!r}')
+ id = int(id)
+ if _whence is None:
+ if _ownsref:
+ _whence = _interpreters.WHENCE_STDLIB
+ else:
+ _whence = _interpreters.whence(id)
+ assert _whence in cls._WHENCE_TO_STR, repr(_whence)
+ if _ownsref is None:
+ _ownsref = (_whence == _interpreters.WHENCE_STDLIB)
+ try:
+ self = _known[id]
+ assert hasattr(self, '_ownsref')
+ except KeyError:
+ self = super().__new__(cls)
+ _known[id] = self
+ self._id = id
+ self._whence = _whence
+ self._ownsref = _ownsref
+ if _ownsref:
+ # This may raise InterpreterNotFoundError:
+ _interpreters.incref(id)
+ return self
+
+ def __repr__(self):
+ return f'{type(self).__name__}({self.id})'
+
+ def __hash__(self):
+ return hash(self._id)
+
+ def __del__(self):
+ self._decref()
+
+ # for pickling:
+ def __reduce__(self):
+ return (type(self), (self._id,))
+
+ # gh-135729: Globals might be destroyed by the time this is called, so we
+ # need to keep references ourself
+ def _decref(self, *,
+ InterpreterNotFoundError=InterpreterNotFoundError,
+ _interp_decref=_interpreters.decref,
+ ):
+ if not self._ownsref:
+ return
+ self._ownsref = False
+ try:
+ _interp_decref(self._id)
+ except InterpreterNotFoundError:
+ pass
+
+ @property
+ def id(self):
+ return self._id
+
+ @property
+ def whence(self):
+ return self._WHENCE_TO_STR[self._whence]
+
+ def is_running(self):
+ """Return whether or not the identified interpreter is running."""
+ return _interpreters.is_running(self._id)
+
+ # Everything past here is available only to interpreters created by
+ # interpreters.create().
+
+ def close(self):
+ """Finalize and destroy the interpreter.
+
+ Attempting to destroy the current interpreter results
+ in an InterpreterError.
+ """
+ return _interpreters.destroy(self._id, restrict=True)
+
+ def prepare_main(self, ns=None, /, **kwargs):
+ """Bind the given values into the interpreter's __main__.
+
+ The values must be shareable.
+ """
+ ns = dict(ns, **kwargs) if ns is not None else kwargs
+ _interpreters.set___main___attrs(self._id, ns, restrict=True)
+
+ def exec(self, code, /):
+ """Run the given source code in the interpreter.
+
+ This is essentially the same as calling the builtin "exec"
+ with this interpreter, using the __dict__ of its __main__
+ module as both globals and locals.
+
+ There is no return value.
+
+ If the code raises an unhandled exception then an ExecutionFailed
+ exception is raised, which summarizes the unhandled exception.
+ The actual exception is discarded because objects cannot be
+ shared between interpreters.
+
+ This blocks the current Python thread until done. During
+ that time, the previous interpreter is allowed to run
+ in other threads.
+ """
+ excinfo = _interpreters.exec(self._id, code, restrict=True)
+ if excinfo is not None:
+ raise ExecutionFailed(excinfo)
+
+ def _call(self, callable, args, kwargs):
+ res, excinfo = _interpreters.call(self._id, callable, args, kwargs, restrict=True)
+ if excinfo is not None:
+ raise ExecutionFailed(excinfo)
+ return res
+
+ def call(self, callable, /, *args, **kwargs):
+ """Call the object in the interpreter with given args/kwargs.
+
+ Nearly all callables, args, kwargs, and return values are
+ supported. All "shareable" objects are supported, as are
+ "stateless" functions (meaning non-closures that do not use
+ any globals). This method will fall back to pickle.
+
+ If the callable raises an exception then the error display
+ (including full traceback) is sent back between the interpreters
+ and an ExecutionFailed exception is raised, much like what
+ happens with Interpreter.exec().
+ """
+ return self._call(callable, args, kwargs)
+
+ def call_in_thread(self, callable, /, *args, **kwargs):
+ """Return a new thread that calls the object in the interpreter.
+
+ The return value and any raised exception are discarded.
+ """
+ t = threading.Thread(target=self._call, args=(callable, args, kwargs))
+ t.start()
+ return t
diff --git a/PythonLib/full/concurrent/interpreters/_crossinterp.py b/PythonLib/full/concurrent/interpreters/_crossinterp.py
new file mode 100644
index 000000000..a5f46b20f
--- /dev/null
+++ b/PythonLib/full/concurrent/interpreters/_crossinterp.py
@@ -0,0 +1,107 @@
+"""Common code between queues and channels."""
+
+
+class ItemInterpreterDestroyed(Exception):
+ """Raised when trying to get an item whose interpreter was destroyed."""
+
+
+class classonly:
+ """A non-data descriptor that makes a value only visible on the class.
+
+ This is like the "classmethod" builtin, but does not show up on
+ instances of the class. It may be used as a decorator.
+ """
+
+ def __init__(self, value):
+ self.value = value
+ self.getter = classmethod(value).__get__
+ self.name = None
+
+ def __set_name__(self, cls, name):
+ if self.name is not None:
+ raise TypeError('already used')
+ self.name = name
+
+ def __get__(self, obj, cls):
+ if obj is not None:
+ raise AttributeError(self.name)
+ # called on the class
+ return self.getter(None, cls)
+
+
+class UnboundItem:
+ """Represents a cross-interpreter item no longer bound to an interpreter.
+
+ An item is unbound when the interpreter that added it to the
+ cross-interpreter container is destroyed.
+ """
+
+ __slots__ = ()
+
+ @classonly
+ def singleton(cls, kind, module, name='UNBOUND'):
+ doc = cls.__doc__
+ if doc:
+ doc = doc.replace(
+ 'cross-interpreter container', kind,
+ ).replace(
+ 'cross-interpreter', kind,
+ )
+ subclass = type(
+ f'Unbound{kind.capitalize()}Item',
+ (cls,),
+ {
+ "_MODULE": module,
+ "_NAME": name,
+ "__doc__": doc,
+ },
+ )
+ return object.__new__(subclass)
+
+ _MODULE = __name__
+ _NAME = 'UNBOUND'
+
+ def __new__(cls):
+ raise Exception(f'use {cls._MODULE}.{cls._NAME}')
+
+ def __repr__(self):
+ return f'{self._MODULE}.{self._NAME}'
+# return f'interpreters._queues.UNBOUND'
+
+
+UNBOUND = object.__new__(UnboundItem)
+UNBOUND_ERROR = object()
+UNBOUND_REMOVE = object()
+
+_UNBOUND_CONSTANT_TO_FLAG = {
+ UNBOUND_REMOVE: 1,
+ UNBOUND_ERROR: 2,
+ UNBOUND: 3,
+}
+_UNBOUND_FLAG_TO_CONSTANT = {v: k
+ for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()}
+
+
+def serialize_unbound(unbound):
+ op = unbound
+ try:
+ flag = _UNBOUND_CONSTANT_TO_FLAG[op]
+ except KeyError:
+ raise NotImplementedError(f'unsupported unbound replacement op {op!r}')
+ return flag,
+
+
+def resolve_unbound(flag, exctype_destroyed):
+ try:
+ op = _UNBOUND_FLAG_TO_CONSTANT[flag]
+ except KeyError:
+ raise NotImplementedError(f'unsupported unbound replacement op {flag!r}')
+ if op is UNBOUND_REMOVE:
+ # "remove" not possible here
+ raise NotImplementedError
+ elif op is UNBOUND_ERROR:
+ raise exctype_destroyed("item's original interpreter destroyed")
+ elif op is UNBOUND:
+ return UNBOUND
+ else:
+ raise NotImplementedError(repr(op))
diff --git a/PythonLib/full/concurrent/interpreters/_queues.py b/PythonLib/full/concurrent/interpreters/_queues.py
new file mode 100644
index 000000000..81ea1098d
--- /dev/null
+++ b/PythonLib/full/concurrent/interpreters/_queues.py
@@ -0,0 +1,289 @@
+"""Cross-interpreter Queues High Level Module."""
+
+import pickle
+import queue
+import time
+import weakref
+import _interpqueues as _queues
+from . import _crossinterp
+
+# aliases:
+from _interpqueues import (
+ QueueError, QueueNotFoundError,
+)
+from ._crossinterp import (
+ UNBOUND_ERROR, UNBOUND_REMOVE,
+)
+
+__all__ = [
+ 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE',
+ 'create', 'list_all',
+ 'Queue',
+ 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull',
+ 'ItemInterpreterDestroyed',
+]
+
+
+class QueueEmpty(QueueError, queue.Empty):
+ """Raised from get_nowait() when the queue is empty.
+
+ It is also raised from get() if it times out.
+ """
+
+
+class QueueFull(QueueError, queue.Full):
+ """Raised from put_nowait() when the queue is full.
+
+ It is also raised from put() if it times out.
+ """
+
+
+class ItemInterpreterDestroyed(QueueError,
+ _crossinterp.ItemInterpreterDestroyed):
+ """Raised from get() and get_nowait()."""
+
+
+_SHARED_ONLY = 0
+_PICKLED = 1
+
+
+UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__)
+
+
+def _serialize_unbound(unbound):
+ if unbound is UNBOUND:
+ unbound = _crossinterp.UNBOUND
+ return _crossinterp.serialize_unbound(unbound)
+
+
+def _resolve_unbound(flag):
+ resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed)
+ if resolved is _crossinterp.UNBOUND:
+ resolved = UNBOUND
+ return resolved
+
+
+def create(maxsize=0, *, unbounditems=UNBOUND):
+ """Return a new cross-interpreter queue.
+
+ The queue may be used to pass data safely between interpreters.
+
+ "unbounditems" sets the default for Queue.put(); see that method for
+ supported values. The default value is UNBOUND, which replaces
+ the unbound item.
+ """
+ unbound = _serialize_unbound(unbounditems)
+ unboundop, = unbound
+ qid = _queues.create(maxsize, unboundop, -1)
+ self = Queue(qid)
+ self._set_unbound(unboundop, unbounditems)
+ return self
+
+
+def list_all():
+ """Return a list of all open queues."""
+ queues = []
+ for qid, unboundop, _ in _queues.list_all():
+ self = Queue(qid)
+ if not hasattr(self, '_unbound'):
+ self._set_unbound(unboundop)
+ else:
+ assert self._unbound[0] == unboundop
+ queues.append(self)
+ return queues
+
+
+_known_queues = weakref.WeakValueDictionary()
+
+class Queue:
+ """A cross-interpreter queue."""
+
+ def __new__(cls, id, /):
+ # There is only one instance for any given ID.
+ if isinstance(id, int):
+ id = int(id)
+ else:
+ raise TypeError(f'id must be an int, got {id!r}')
+ try:
+ self = _known_queues[id]
+ except KeyError:
+ self = super().__new__(cls)
+ self._id = id
+ _known_queues[id] = self
+ _queues.bind(id)
+ return self
+
+ def __del__(self):
+ try:
+ _queues.release(self._id)
+ except QueueNotFoundError:
+ pass
+ try:
+ del _known_queues[self._id]
+ except KeyError:
+ pass
+
+ def __repr__(self):
+ return f'{type(self).__name__}({self.id})'
+
+ def __hash__(self):
+ return hash(self._id)
+
+ # for pickling:
+ def __reduce__(self):
+ return (type(self), (self._id,))
+
+ def _set_unbound(self, op, items=None):
+ assert not hasattr(self, '_unbound')
+ if items is None:
+ items = _resolve_unbound(op)
+ unbound = (op, items)
+ self._unbound = unbound
+ return unbound
+
+ @property
+ def id(self):
+ return self._id
+
+ @property
+ def unbounditems(self):
+ try:
+ _, items = self._unbound
+ except AttributeError:
+ op, _ = _queues.get_queue_defaults(self._id)
+ _, items = self._set_unbound(op)
+ return items
+
+ @property
+ def maxsize(self):
+ try:
+ return self._maxsize
+ except AttributeError:
+ self._maxsize = _queues.get_maxsize(self._id)
+ return self._maxsize
+
+ def empty(self):
+ return self.qsize() == 0
+
+ def full(self):
+ return _queues.is_full(self._id)
+
+ def qsize(self):
+ return _queues.get_count(self._id)
+
+ def put(self, obj, block=True, timeout=None, *,
+ unbounditems=None,
+ _delay=10 / 1000, # 10 milliseconds
+ ):
+ """Add the object to the queue.
+
+ If "block" is true, this blocks while the queue is full.
+
+ For most objects, the object received through Queue.get() will
+ be a new one, equivalent to the original and not sharing any
+ actual underlying data. The notable exceptions include
+ cross-interpreter types (like Queue) and memoryview, where the
+ underlying data is actually shared. Furthermore, some types
+ can be sent through a queue more efficiently than others. This
+ group includes various immutable types like int, str, bytes, and
+ tuple (if the items are likewise efficiently shareable). See interpreters.is_shareable().
+
+ "unbounditems" controls the behavior of Queue.get() for the given
+ object if the current interpreter (calling put()) is later
+ destroyed.
+
+ If "unbounditems" is None (the default) then it uses the
+ queue's default, set with create_queue(),
+ which is usually UNBOUND.
+
+ If "unbounditems" is UNBOUND_ERROR then get() will raise an
+ ItemInterpreterDestroyed exception if the original interpreter
+ has been destroyed. This does not otherwise affect the queue;
+ the next call to put() will work like normal, returning the next
+ item in the queue.
+
+ If "unbounditems" is UNBOUND_REMOVE then the item will be removed
+ from the queue as soon as the original interpreter is destroyed.
+ Be aware that this will introduce an imbalance between put()
+ and get() calls.
+
+ If "unbounditems" is UNBOUND then it is returned by get() in place
+ of the unbound item.
+ """
+ if not block:
+ return self.put_nowait(obj, unbounditems=unbounditems)
+ if unbounditems is None:
+ unboundop = -1
+ else:
+ unboundop, = _serialize_unbound(unbounditems)
+ if timeout is not None:
+ timeout = int(timeout)
+ if timeout < 0:
+ raise ValueError(f'timeout value must be non-negative')
+ end = time.time() + timeout
+ while True:
+ try:
+ _queues.put(self._id, obj, unboundop)
+ except QueueFull as exc:
+ if timeout is not None and time.time() >= end:
+ raise # re-raise
+ time.sleep(_delay)
+ else:
+ break
+
+ def put_nowait(self, obj, *, unbounditems=None):
+ if unbounditems is None:
+ unboundop = -1
+ else:
+ unboundop, = _serialize_unbound(unbounditems)
+ _queues.put(self._id, obj, unboundop)
+
+ def get(self, block=True, timeout=None, *,
+ _delay=10 / 1000, # 10 milliseconds
+ ):
+ """Return the next object from the queue.
+
+ If "block" is true, this blocks while the queue is empty.
+
+ If the next item's original interpreter has been destroyed
+ then the "next object" is determined by the value of the
+ "unbounditems" argument to put().
+ """
+ if not block:
+ return self.get_nowait()
+ if timeout is not None:
+ timeout = int(timeout)
+ if timeout < 0:
+ raise ValueError(f'timeout value must be non-negative')
+ end = time.time() + timeout
+ while True:
+ try:
+ obj, unboundop = _queues.get(self._id)
+ except QueueEmpty as exc:
+ if timeout is not None and time.time() >= end:
+ raise # re-raise
+ time.sleep(_delay)
+ else:
+ break
+ if unboundop is not None:
+ assert obj is None, repr(obj)
+ return _resolve_unbound(unboundop)
+ return obj
+
+ def get_nowait(self):
+ """Return the next object from the channel.
+
+ If the queue is empty then raise QueueEmpty. Otherwise this
+ is the same as get().
+ """
+ try:
+ obj, unboundop = _queues.get(self._id)
+ except QueueEmpty as exc:
+ raise # re-raise
+ if unboundop is not None:
+ assert obj is None, repr(obj)
+ return _resolve_unbound(unboundop)
+ return obj
+
+
+_queues._register_heap_types(Queue, QueueEmpty, QueueFull)
diff --git a/PythonLib/full/configparser.py b/PythonLib/full/configparser.py
index f96704eb4..a53ac8727 100644
--- a/PythonLib/full/configparser.py
+++ b/PythonLib/full/configparser.py
@@ -18,8 +18,8 @@
delimiters=('=', ':'), comment_prefixes=('#', ';'),
inline_comment_prefixes=None, strict=True,
empty_lines_in_values=True, default_section='DEFAULT',
- interpolation=, converters=):
-
+ interpolation=, converters=,
+ allow_unnamed_section=False):
Create the parser. When `defaults` is given, it is initialized into the
dictionary or intrinsic defaults. The keys must be strings, the values
must be appropriate for %()s string interpolation.
@@ -68,6 +68,10 @@
converter gets its corresponding get*() method on the parser object and
section proxies.
+ When `allow_unnamed_section` is True (default: False), options
+ without section are accepted: the section for these is
+ ``configparser.UNNAMED_SECTION``.
+
sections()
Return all the configuration section names, sans DEFAULT.
@@ -139,24 +143,27 @@
between keys and values are surrounded by spaces.
"""
-from collections.abc import MutableMapping
+# Do not import dataclasses; overhead is unacceptable (gh-117703)
+
+from collections.abc import Iterable, MutableMapping
from collections import ChainMap as _ChainMap
+import contextlib
import functools
import io
import itertools
import os
import re
import sys
-import warnings
__all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError",
"NoOptionError", "InterpolationError", "InterpolationDepthError",
"InterpolationMissingOptionError", "InterpolationSyntaxError",
"ParsingError", "MissingSectionHeaderError",
- "ConfigParser", "RawConfigParser",
+ "MultilineContinuationError", "UnnamedSectionDisabledError",
+ "InvalidWriteError", "ConfigParser", "RawConfigParser",
"Interpolation", "BasicInterpolation", "ExtendedInterpolation",
- "LegacyInterpolation", "SectionProxy", "ConverterMapping",
- "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH")
+ "SectionProxy", "ConverterMapping",
+ "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION")
_default_dict = dict
DEFAULTSECT = "DEFAULT"
@@ -298,15 +305,36 @@ def __init__(self, option, section, rawval):
class ParsingError(Error):
"""Raised when a configuration file does not follow legal syntax."""
- def __init__(self, source):
+ def __init__(self, source, *args):
super().__init__(f'Source contains parsing errors: {source!r}')
self.source = source
self.errors = []
self.args = (source, )
+ if args:
+ self.append(*args)
def append(self, lineno, line):
self.errors.append((lineno, line))
- self.message += '\n\t[line %2d]: %s' % (lineno, line)
+ self.message += f'\n\t[line {lineno:2d}]: {line!r}'
+
+ def combine(self, others):
+ messages = [self.message]
+ for other in others:
+ for lineno, line in other.errors:
+ self.errors.append((lineno, line))
+ messages.append(f'\n\t[line {lineno:2d}]: {line!r}')
+ self.message = "".join(messages)
+ return self
+
+ @staticmethod
+ def _raise_all(exceptions: Iterable['ParsingError']):
+ """
+ Combine any number of ParsingErrors into one and raise it.
+ """
+ exceptions = iter(exceptions)
+ with contextlib.suppress(StopIteration):
+ raise next(exceptions).combine(exceptions)
+
class MissingSectionHeaderError(ParsingError):
@@ -323,6 +351,44 @@ def __init__(self, filename, lineno, line):
self.args = (filename, lineno, line)
+class MultilineContinuationError(ParsingError):
+ """Raised when a key without value is followed by continuation line"""
+ def __init__(self, filename, lineno, line):
+ Error.__init__(
+ self,
+ "Key without value continued with an indented line.\n"
+ "file: %r, line: %d\n%r"
+ %(filename, lineno, line))
+ self.source = filename
+ self.lineno = lineno
+ self.line = line
+ self.args = (filename, lineno, line)
+
+
+class UnnamedSectionDisabledError(Error):
+ """Raised when an attempt to use UNNAMED_SECTION is made with the
+ feature disabled."""
+ def __init__(self):
+ Error.__init__(self, "Support for UNNAMED_SECTION is disabled.")
+
+
+class _UnnamedSection:
+
+ def __repr__(self):
+ return ""
+
+class InvalidWriteError(Error):
+ """Raised when attempting to write data that the parser would read back differently.
+ ex: writing a key which begins with the section header pattern would read back as a
+ new section """
+
+ def __init__(self, msg=''):
+ Error.__init__(self, msg)
+
+
+UNNAMED_SECTION = _UnnamedSection()
+
+
# Used in parser getters to indicate the default behaviour when a specific
# option is not found it to raise an exception. Created to enable `None` as
# a valid fallback value.
@@ -478,6 +544,8 @@ def _interpolate_some(self, parser, option, accum, rest, section, map,
except (KeyError, NoSectionError, NoOptionError):
raise InterpolationMissingOptionError(
option, section, rawval, ":".join(path)) from None
+ if v is None:
+ continue
if "$" in v:
self._interpolate_some(parser, opt, accum, v, sect,
dict(parser.items(sect, raw=True)),
@@ -491,51 +559,51 @@ def _interpolate_some(self, parser, option, accum, rest, section, map,
"found: %r" % (rest,))
-class LegacyInterpolation(Interpolation):
- """Deprecated interpolation used in old versions of ConfigParser.
- Use BasicInterpolation or ExtendedInterpolation instead."""
+class _ReadState:
+ elements_added : set[str]
+ cursect : dict[str, str] | None = None
+ sectname : str | None = None
+ optname : str | None = None
+ lineno : int = 0
+ indent_level : int = 0
+ errors : list[ParsingError]
- _KEYCRE = re.compile(r"%\(([^)]*)\)s|.")
+ def __init__(self):
+ self.elements_added = set()
+ self.errors = list()
- def __init__(self, *args, **kwargs):
- super().__init__(*args, **kwargs)
- warnings.warn(
- "LegacyInterpolation has been deprecated since Python 3.2 "
- "and will be removed from the configparser module in Python 3.13. "
- "Use BasicInterpolation or ExtendedInterpolation instead.",
- DeprecationWarning, stacklevel=2
- )
- def before_get(self, parser, section, option, value, vars):
- rawval = value
- depth = MAX_INTERPOLATION_DEPTH
- while depth: # Loop through this until it's done
- depth -= 1
- if value and "%(" in value:
- replace = functools.partial(self._interpolation_replace,
- parser=parser)
- value = self._KEYCRE.sub(replace, value)
- try:
- value = value % vars
- except KeyError as e:
- raise InterpolationMissingOptionError(
- option, section, rawval, e.args[0]) from None
- else:
- break
- if value and "%(" in value:
- raise InterpolationDepthError(option, section, rawval)
- return value
+class _Line(str):
+ __slots__ = 'clean', 'has_comments'
- def before_set(self, parser, section, option, value):
- return value
+ def __new__(cls, val, *args, **kwargs):
+ return super().__new__(cls, val)
- @staticmethod
- def _interpolation_replace(match, parser):
- s = match.group(1)
- if s is None:
- return match.group()
- else:
- return "%%(%s)s" % parser.optionxform(s)
+ def __init__(self, val, comments):
+ trimmed = val.strip()
+ self.clean = comments.strip(trimmed)
+ self.has_comments = trimmed != self.clean
+
+
+class _CommentSpec:
+ def __init__(self, full_prefixes, inline_prefixes):
+ full_patterns = (
+ # prefix at the beginning of a line
+ fr'^({re.escape(prefix)}).*'
+ for prefix in full_prefixes
+ )
+ inline_patterns = (
+ # prefix at the beginning of the line or following a space
+ fr'(^|\s)({re.escape(prefix)}.*)'
+ for prefix in inline_prefixes
+ )
+ self.pattern = re.compile('|'.join(itertools.chain(full_patterns, inline_patterns)))
+
+ def strip(self, text):
+ return self.pattern.sub('', text).rstrip()
+
+ def wrap(self, text):
+ return _Line(text, self)
class RawConfigParser(MutableMapping):
@@ -548,7 +616,9 @@ class RawConfigParser(MutableMapping):
\] # ]
"""
_OPT_TMPL = r"""
- (?P