gh-109276: libregrtest: WASM use filename for JSON

On Emscripten and WASI platforms, libregrtest now uses a filename for the JSON file. Passing a file descriptor to a child process doesn't work on these platforms.
python · vstinner · Sep 12, 2023 · Sep 12, 2023 · Sep 12, 2023 · Sep 12, 2023
commit 2004fb6be321033a85e163c9e99d85aadb59f9d0
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
@@ -406,7 +406,7 @@ def create_run_tests(self, tests: TestTuple):
             python_cmd=self.python_cmd,
             randomize=self.randomize,
             random_seed=self.random_seed,
-            json_fd=None,
+            json_file=None,
         )
 
     def _run_tests(self, selected: TestTuple, tests: TestList | None) -> int:

diff --git a/Lib/test/libregrtest/run_workers.py b/Lib/test/libregrtest/run_workers.py
@@ -17,7 +17,7 @@
 from .logger import Logger
 from .result import TestResult, State
 from .results import TestResults
-from .runtests import RunTests
+from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
 from .single import PROGRESS_MIN_TIME
 from .utils import (
     StrPath, StrJSON, TestName, MS_WINDOWS,
@@ -155,10 +155,11 @@ def mp_result_error(
     ) -> MultiprocessResult:
         return MultiprocessResult(test_result, stdout, err_msg)
 
-    def _run_process(self, runtests: RunTests, output_fd: int, json_fd: int,
+    def _run_process(self, runtests: RunTests, output_fd: int,
+                     json_file: JsonFileType,
                      tmp_dir: StrPath | None = None) -> int:
         try:
-            popen = create_worker_process(runtests, output_fd, json_fd,
+            popen = create_worker_process(runtests, output_fd, json_file,
                                           tmp_dir)
 
             self._killed = False
@@ -226,21 +227,29 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
             match_tests = None
         err_msg = None
 
+        stdout_file = tempfile.TemporaryFile('w+', encoding=encoding)
+        if JSON_FILE_USE_FILENAME:
+            json_tmpfile = tempfile.NamedTemporaryFile('w+', encoding='utf8')
+        else:
+            json_tmpfile = tempfile.TemporaryFile('w+', encoding='utf8')
+
         # gh-94026: Write stdout+stderr to a tempfile as workaround for
         # non-blocking pipes on Emscripten with NodeJS.
-        with (tempfile.TemporaryFile('w+', encoding=encoding) as stdout_file,
-              tempfile.TemporaryFile('w+', encoding='utf8') as json_file):
+        with (stdout_file, json_tmpfile):
             stdout_fd = stdout_file.fileno()
-            json_fd = json_file.fileno()
-            if MS_WINDOWS:
-                json_fd = msvcrt.get_osfhandle(json_fd)
+            if JSON_FILE_USE_FILENAME:
+                json_file = json_tmpfile.name
+            else:
+                json_file = json_tmpfile.fileno()
+                if MS_WINDOWS:
+                    json_file = msvcrt.get_osfhandle(json_file)
 
             kwargs = {}
             if match_tests:
                 kwargs['match_tests'] = match_tests
             worker_runtests = self.runtests.copy(
                 tests=tests,
-                json_fd=json_fd,
+                json_file=json_file,
                 **kwargs)
 
             # gh-93353: Check for leaked temporary files in the parent process,
@@ -254,13 +263,13 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
                 tmp_dir = os.path.abspath(tmp_dir)
                 try:
                     retcode = self._run_process(worker_runtests,
-                                                stdout_fd, json_fd, tmp_dir)
+                                                stdout_fd, json_file, tmp_dir)
                 finally:
                     tmp_files = os.listdir(tmp_dir)
                     os_helper.rmtree(tmp_dir)
             else:
                 retcode = self._run_process(worker_runtests,
-                                            stdout_fd, json_fd)
+                                            stdout_fd, json_file)
                 tmp_files = ()
             stdout_file.seek(0)
 
@@ -275,8 +284,8 @@ def _runtest(self, test_name: TestName) -> MultiprocessResult:
 
             try:
                 # deserialize run_tests_worker() output
-                json_file.seek(0)
-                worker_json: StrJSON = json_file.read()
+                json_tmpfile.seek(0)
+                worker_json: StrJSON = json_tmpfile.read()
                 if worker_json:
                     result = TestResult.from_json(worker_json)
                 else:

diff --git a/Lib/test/libregrtest/runtests.py b/Lib/test/libregrtest/runtests.py
@@ -2,10 +2,25 @@
 import json
 from typing import Any
 
+from test import support
+
 from .utils import (
     StrPath, StrJSON, TestTuple, FilterTuple, FilterDict)
 
 
+if support.is_emscripten or support.is_wasi:
+    # On Emscripten/WASI, it's a filename. Passing a file descriptor to a
+    # worker process fails with "OSError: [Errno 8] Bad file descriptor" in the
+    # worker process.
+    JsonFileType = StrPath
+    JSON_FILE_USE_FILENAME = True
+else:
+    # On Unix, it's a file descriptor.
+    # On Windows, it's a handle.
+    JsonFileType = int
+    JSON_FILE_USE_FILENAME = False
+
+
 @dataclasses.dataclass(slots=True, frozen=True)
 class HuntRefleak:
     warmups: int
@@ -38,9 +53,7 @@ class RunTests:
     python_cmd: tuple[str] | None
     randomize: bool
     random_seed: int | None
-    # On Unix, it's a file descriptor.
-    # On Windows, it's a handle.
-    json_fd: int | None
+    json_file: JsonFileType  | None
 
     def copy(self, **override):
         state = dataclasses.asdict(self)

diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py
@@ -387,7 +387,7 @@ def get_work_dir(parent_dir: StrPath, worker: bool = False) -> StrPath:
     # testing (see the -j option).
     # Emscripten and WASI have stubbed getpid(), Emscripten has only
     # milisecond clock resolution. Use randint() instead.
-    if sys.platform in {"emscripten", "wasi"}:
+    if support.is_emscripten or support.is_wasi:
         nounce = random.randint(0, 1_000_000)
     else:
         nounce = os.getpid()

diff --git a/Lib/test/libregrtest/worker.py b/Lib/test/libregrtest/worker.py
@@ -7,7 +7,7 @@
 from test.support import os_helper
 
 from .setup import setup_process, setup_test_dir
-from .runtests import RunTests
+from .runtests import RunTests, JsonFileType, JSON_FILE_USE_FILENAME
 from .single import run_single_test
 from .utils import (
     StrPath, StrJSON, FilterTuple, MS_WINDOWS,
@@ -18,7 +18,7 @@
 
 
 def create_worker_process(runtests: RunTests,
-                          output_fd: int, json_fd: int,
+                          output_fd: int, json_file: JsonFileType,
                           tmp_dir: StrPath | None = None) -> subprocess.Popen:
     python_cmd = runtests.python_cmd
     worker_json = runtests.as_json()
@@ -55,33 +55,42 @@ def create_worker_process(runtests: RunTests,
         close_fds=True,
         cwd=work_dir,
     )
-    if not MS_WINDOWS:
-        kwargs['pass_fds'] = [json_fd]
-    else:
+    if JSON_FILE_USE_FILENAME:
+        # Nothing to do to pass the JSON filename to the worker process
+        pass
+    elif MS_WINDOWS:
+        # Pass the JSON handle to the worker process
         startupinfo = subprocess.STARTUPINFO()
-        startupinfo.lpAttributeList = {"handle_list": [json_fd]}
+        startupinfo.lpAttributeList = {"handle_list": [json_file]}
         kwargs['startupinfo'] = startupinfo
+    else:
+        # Pass the JSON file descriptor to the worker process
+        kwargs['pass_fds'] = [json_file]
     if USE_PROCESS_GROUP:
         kwargs['start_new_session'] = True
 
     if MS_WINDOWS:
-        os.set_handle_inheritable(json_fd, True)
+        os.set_handle_inheritable(json_file, True)
     try:
         return subprocess.Popen(cmd, **kwargs)
     finally:
         if MS_WINDOWS:
-            os.set_handle_inheritable(json_fd, False)
+            os.set_handle_inheritable(json_file, False)
 
 
 def worker_process(worker_json: StrJSON) -> NoReturn:
     runtests = RunTests.from_json(worker_json)
     test_name = runtests.tests[0]
     match_tests: FilterTuple | None = runtests.match_tests
-    json_fd: int = runtests.json_fd
+    # On Unix, it's a file descriptor.
+    # On Windows, it's a handle.
+    # On Emscripten/WASI, it's a filename.
+    json_file: JsonFileType = runtests.json_file
 
     if MS_WINDOWS:
         import msvcrt
-        json_fd = msvcrt.open_osfhandle(json_fd, os.O_WRONLY)
+        # Create a file descriptor from the handle
+        json_file = msvcrt.open_osfhandle(json_file, os.O_WRONLY)
 
 
     setup_test_dir(runtests.test_dir)
@@ -96,8 +105,8 @@ def worker_process(worker_json: StrJSON) -> NoReturn:
 
     result = run_single_test(test_name, runtests)
 
-    with open(json_fd, 'w', encoding='utf-8') as json_file:
-        result.write_json_into(json_file)
+    with open(json_file, 'w', encoding='utf-8') as fp:
+        result.write_json_into(fp)
 
     sys.exit(0)