Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions Doc/library/audit_events.rst
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public API of CPython:
+----------------------------+-------------------------------------------+
| _winapi.OpenProcess | ``process_id``, ``desired_access`` |
+----------------------------+-------------------------------------------+
| _winapi.OpenThread | ``thread_id``, ``desired_access`` |
+----------------------------+-------------------------------------------+
| _winapi.TerminateProcess | ``handle``, ``exit_code`` |
+----------------------------+-------------------------------------------+
| _posixsubprocess.fork_exec | ``exec_list``, ``args``, ``env`` |
Expand All @@ -50,3 +52,6 @@ public API of CPython:

.. versionadded:: 3.14
The ``_posixsubprocess.fork_exec`` internal audit event.

.. versionadded:: next
The ``_winapi.OpenThread`` internal audit event.
72 changes: 64 additions & 8 deletions Lib/subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -672,11 +672,13 @@ def run(*popenargs,
except TimeoutExpired as exc:
process.kill()
if _mswindows:
# Windows accumulates the output in a single blocking
# read() call run on child threads, with the timeout
# being done in a join() on those threads. communicate()
# _after_ kill() is required to collect that and add it
# to the exception.
# Windows accumulates the output on child threads, with the
# timeout being done in a join() on those threads.
# communicate() _after_ kill() is required to collect that
# and add it to the exception. The pipes can be inherited
# by other processes, so cancel pending reads to not wait
# for them (gh-87512).
process._cancel_io()
exc.stdout, exc.stderr = process.communicate()
else:
# POSIX _communicate already populated the output so
Expand Down Expand Up @@ -1277,6 +1279,8 @@ def __enter__(self):
return self

def __exit__(self, exc_type, value, traceback):
if _mswindows:
self._cancel_io()
if self.stdout:
self.stdout.close()
if self.stderr:
Expand Down Expand Up @@ -1776,8 +1780,46 @@ def _wait(self, timeout):
return self.returncode


def _cancel_io(self):
# The communication threads can be blocked in a synchronous read
# or write on a pipe which is not closed by the child process.
# Closing such pipe blocks too, so cancel the I/O first.
for name in ('stdout_thread', 'stderr_thread', '_stdin_thread'):
thread = getattr(self, name, None)
if thread is None or not thread.is_alive():
continue
try:
handle = _winapi.OpenThread(_winapi.THREAD_TERMINATE,
False, thread.ident)
except OSError:
continue
try:
_winapi.CancelSynchronousIo(handle)
except OSError:
# ERROR_NOT_FOUND if there is no pending I/O.
pass
finally:
_winapi.CloseHandle(handle)


def _readerthread(self, fh, buffer):
buffer.append(fh.read())
# Read what is available, or block for a single byte, so that
# the read can be canceled and does not wait for the pipe to be
# closed by all processes which inherited it (gh-87512).
handle = msvcrt.get_osfhandle(fh.fileno())
while True:
try:
size = _winapi.PeekNamedPipe(handle)[0] or 1
data = _winapi.ReadFile(handle, size)[0]
except BrokenPipeError:
break
except OSError as exc:
if exc.winerror == _winapi.ERROR_OPERATION_ABORTED:
break
raise
if not data:
break
buffer.append(data)
fh.close()


Expand Down Expand Up @@ -1845,8 +1887,22 @@ def _communicate(self, input, endtime, orig_timeout):
self.stderr.close()

# All data exchanged. Translate lists into strings.
stdout = stdout[0] if stdout else None
stderr = stderr[0] if stderr else None
if stdout is not None:
stdout = b''.join(stdout)
if stderr is not None:
stderr = b''.join(stderr)

# Translate newlines, if requested.
# This also turns bytes into strings.
if self.text_mode:
if stdout is not None:
stdout = self._translate_newlines(stdout,
self.stdout.encoding,
self.stdout.errors)
if stderr is not None:
stderr = self._translate_newlines(stderr,
self.stderr.encoding,
self.stderr.errors)

return (stdout, stderr)

Expand Down
70 changes: 70 additions & 0 deletions Lib/test/test_subprocess.py
Original file line number Diff line number Diff line change
Expand Up @@ -3982,6 +3982,76 @@ def test_kill_dead(self):
def test_terminate_dead(self):
self._kill_dead_process('terminate')

# gh-87512: the pipes can be inherited by a process which outlives the
# child process, e.g. with shell=True. Reading them should not wait for
# that process, neither in communicate() nor when closing them.

def _inherited_pipe_cmd(self, sentinel):
# Run the code in a grandchild process which lives while the
# sentinel file exists, but not longer than SHORT_TIMEOUT.
code = ("import os, sys, time; "
"print('spam'); sys.stdout.flush(); "
"print('eggs', file=sys.stderr); sys.stderr.flush(); "
"[time.sleep(0.05) for _ in range(%s) "
"if os.path.exists(%a)]"
% (int(support.SHORT_TIMEOUT / 0.05), sentinel))
cmd = '"%s" -c "%s"' % (sys.executable, code)
return cmd

def test_run_timeout_inherited_pipe(self):
with os_helper.temp_dir() as dirname:
sentinel = os.path.join(dirname, 'sentinel')
open(sentinel, 'wb').close()
cmd = self._inherited_pipe_cmd(sentinel)
try:
start = time.monotonic()
with self.assertRaises(subprocess.TimeoutExpired) as cm:
subprocess.run(cmd, shell=True, capture_output=True,
timeout=0.5)
self.assertLess(time.monotonic() - start, support.LOOPBACK_TIMEOUT)
# The output written before the timeout is preserved.
self.assertEqual(cm.exception.stdout, b'spam\r\n')
self.assertEqual(cm.exception.stderr, b'eggs\r\n')
finally:
os.unlink(sentinel)

def test_run_timeout_inherited_pipe_text(self):
with os_helper.temp_dir() as dirname:
sentinel = os.path.join(dirname, 'sentinel')
open(sentinel, 'wb').close()
cmd = self._inherited_pipe_cmd(sentinel)
try:
start = time.monotonic()
with self.assertRaises(subprocess.TimeoutExpired) as cm:
subprocess.run(cmd, shell=True, capture_output=True,
text=True, timeout=0.5)
self.assertLess(time.monotonic() - start,
support.LOOPBACK_TIMEOUT)
self.assertEqual(cm.exception.stdout, 'spam\n')
self.assertEqual(cm.exception.stderr, 'eggs\n')
finally:
os.unlink(sentinel)

def test_exit_inherited_pipe(self):
# Closing the pipes should not block on a pending read.
with os_helper.temp_dir() as dirname:
sentinel = os.path.join(dirname, 'sentinel')
open(sentinel, 'wb').close()
cmd = self._inherited_pipe_cmd(sentinel)
try:
proc = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE)
with self.assertRaises(subprocess.TimeoutExpired):
proc.communicate(timeout=0.5)
proc.kill() # kills the shell, not the grandchild
start = time.monotonic()
with proc:
pass
self.assertLess(time.monotonic() - start, support.LOOPBACK_TIMEOUT)
finally:
os.unlink(sentinel)


class MiscTests(unittest.TestCase):

class RecordingPopen(subprocess.Popen):
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Fix :func:`subprocess.run` on Windows: the *timeout* was ignored if the pipes
were inherited by a process which outlives the child process (for example
with ``shell=True``). The output read before the timeout is now set on the
:exc:`~subprocess.TimeoutExpired` exception, as on other platforms. Closing
:class:`subprocess.Popen` no longer blocks in such case either.
60 changes: 60 additions & 0 deletions Modules/_winapi.c
Original file line number Diff line number Diff line change
Expand Up @@ -1933,6 +1933,63 @@ _winapi_OpenProcess_impl(PyObject *module, DWORD desired_access,
return handle;
}

/*[clinic input]
_winapi.OpenThread -> HANDLE

desired_access: DWORD
inherit_handle: BOOL
thread_id: DWORD
/
[clinic start generated code]*/

static HANDLE
_winapi_OpenThread_impl(PyObject *module, DWORD desired_access,
BOOL inherit_handle, DWORD thread_id)
/*[clinic end generated code: output=4eac45e975925ec0 input=ee54a437060f9b59]*/
{
HANDLE handle;

if (PySys_Audit("_winapi.OpenThread", "kk",
thread_id, desired_access) < 0) {
return INVALID_HANDLE_VALUE;
}

Py_BEGIN_ALLOW_THREADS
handle = OpenThread(desired_access, inherit_handle, thread_id);
Py_END_ALLOW_THREADS
if (handle == NULL) {
PyErr_SetFromWindowsErr(0);
handle = INVALID_HANDLE_VALUE;
}

return handle;
}

/*[clinic input]
_winapi.CancelSynchronousIo

thread: HANDLE
/

Cancel pending synchronous I/O issued by the specified thread.
[clinic start generated code]*/

static PyObject *
_winapi_CancelSynchronousIo_impl(PyObject *module, HANDLE thread)
/*[clinic end generated code: output=a15680598ffc526b input=40cdf5a637ed95da]*/
{
BOOL result;

Py_BEGIN_ALLOW_THREADS
result = CancelSynchronousIo(thread);
Py_END_ALLOW_THREADS

if (!result) {
return PyErr_SetFromWindowsErr(0);
}
Py_RETURN_NONE;
}

/*[clinic input]
_winapi.PeekNamedPipe

Expand Down Expand Up @@ -3212,6 +3269,8 @@ static PyMethodDef winapi_functions[] = {
_WINAPI_RESETEVENT_METHODDEF
_WINAPI_SETEVENT_METHODDEF
_WINAPI_SETNAMEDPIPEHANDLESTATE_METHODDEF
_WINAPI_OPENTHREAD_METHODDEF
_WINAPI_CANCELSYNCHRONOUSIO_METHODDEF
_WINAPI_TERMINATEPROCESS_METHODDEF
_WINAPI_UNMAPVIEWOFFILE_METHODDEF
_WINAPI_VIRTUALQUERYSIZE_METHODDEF
Expand Down Expand Up @@ -3323,6 +3382,7 @@ static int winapi_exec(PyObject *m)
WINAPI_CONSTANT(F_DWORD, PIPE_WAIT);
WINAPI_CONSTANT(F_DWORD, PROCESS_ALL_ACCESS);
WINAPI_CONSTANT(F_DWORD, SYNCHRONIZE);
WINAPI_CONSTANT(F_DWORD, THREAD_TERMINATE);
WINAPI_CONSTANT(F_DWORD, PROCESS_DUP_HANDLE);
WINAPI_CONSTANT(F_DWORD, PROCESS_QUERY_LIMITED_INFORMATION);
WINAPI_CONSTANT(F_DWORD, SEC_COMMIT);
Expand Down
67 changes: 66 additions & 1 deletion Modules/clinic/_winapi.c.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading