Add no-GIL interpreter support

Add `pytest-run-parallel` as dependency, test no-GIL interpreters in CI, and mark Cython module as safe for freethreaded interpreters.
msgpack · clin1234 · Jul 20, 2025 · Jul 20, 2025 · Jul 20, 2025 · Jul 25, 2025
commit 6ced81761612fe33e339a09362042cb65ac388cb
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -10,7 +10,7 @@ jobs:
     strategy:
       matrix:
         os: ["ubuntu-latest", "windows-latest", "macos-latest"]
-        py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"]
+        py: ["3.14", "3.14t", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"]
 
     runs-on: ${{ matrix.os }}
     name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}
@@ -41,12 +41,12 @@ jobs:
       - name: Test (C extension)
         shell: bash
         run: |
-          pytest -v test
+          PYTHON_GIL=0 pytest -v test
 
       - name: Test (pure Python fallback)
         shell: bash
         run: |
-          MSGPACK_PUREPYTHON=1 pytest -v test
+          PYTHON_GIL=0 MSGPACK_PUREPYTHON=1 pytest -v test
 
       - name: build packages
         shell: bash

diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml
@@ -32,8 +32,9 @@ jobs:
         uses: pypa/cibuildwheel@v2.23.3
         env:
           CIBW_TEST_REQUIRES: "pytest"
-          CIBW_TEST_COMMAND: "pytest {package}/test"
+          CIBW_TEST_COMMAND: "PYTHON_GIL=0 pytest {package}/test"
           CIBW_SKIP: "pp* cp38-macosx_*"
+          CIBW_ENABLE: cpython-freerelease
 
       - name: Build sdist
         if: runner.os == 'Linux' && runner.arch == 'X64'

diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx
@@ -1,5 +1,5 @@
 # coding: utf-8
-#cython: embedsignature=True, c_string_encoding=ascii, language_level=3
+#cython: embedsignature=True, c_string_encoding=ascii, language_level=3, freethreading_compatible=True
 from cpython.datetime cimport import_datetime, datetime_new
 import_datetime()
 

diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx
@@ -1,5 +1,5 @@
 # coding: utf-8
-
+# cython: freethreading_compatible = True 
 from cpython cimport *
 from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact
 from cpython.datetime cimport (

diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx
@@ -1,5 +1,5 @@
 # coding: utf-8
-
+# cython: freethreading_compatible = True 
 from cpython cimport *
 cdef extern from "Python.h":
     ctypedef struct PyObject

diff --git a/requirements.txt b/requirements.txt
@@ -1 +1,2 @@
 Cython~=3.1.1
+pytest-run-parallel[psutil]
diff --git a/test/test_multithreading.py b/test/test_multithreading.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+from concurrent.futures import ThreadPoolExecutor
+from msgpack import Packer
+import threading
+
+
+def run_threaded(
+    func,
+    num_threads=8,
+    pass_count=False,
+    pass_barrier=False,
+    outer_iterations=1,
+    prepare_args=None,
+):
+    """Runs a function many times in parallel"""
+    for _ in range(outer_iterations):
+        with ThreadPoolExecutor(max_workers=num_threads) as tpe:
+            if prepare_args is None:
+                args = []
+            else:
+                args = prepare_args()
+            if pass_barrier:
+                barrier = threading.Barrier(num_threads)
+                args.append(barrier)
+            if pass_count:
+                all_args = [(func, i, *args) for i in range(num_threads)]
+            else:
+                all_args = [(func, *args) for i in range(num_threads)]
+            try:
+                futures = []
+                for arg in all_args:
+                    futures.append(tpe.submit(*arg))
+            finally:
+                if len(futures) < num_threads and pass_barrier:
+                    barrier.abort()
+            for f in futures:
+                f.result()
+
+
+def test_multithread_packing():
+    output = []
+    test_data = "abcd" * 10_000_000
+    packer = Packer()
+
+    def closure(b):
+        data = packer.pack(test_data)
+        output.append(data)
+        b.wait()
+
+    run_threaded(closure, num_threads=10, pass_barrier=True, pass_count=False)
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		Cython~=3.1.1
		pytest-run-parallel[psutil]
Comment thread clin1234 marked this conversation as resolved. Outdated