From be4b2a92feb2a2db91bd627db2ed49cdc3ad6521 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 26 Jan 2025 18:28:09 +0000 Subject: [PATCH 01/36] Fix publishing to pypi --- .github/workflows/build_dist.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1e815c1..3189d4a 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -80,8 +80,8 @@ jobs: - name: Save wheels uses: actions/upload-artifact@v4 with: - name: cibw-wheels-${{ strategy.job-index }}.whl - path: wheelhouse/*.whl + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl upload_pypi: name: Upload to PyPI @@ -93,6 +93,7 @@ jobs: with: pattern: cibw-* path: dist + merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} From b5e5217b1a783da004efef0b0d2ea0ce64241b84 Mon Sep 17 00:00:00 2001 From: "Jonathan G. Underwood" Date: Sun, 26 Jan 2025 19:42:04 +0000 Subject: [PATCH 02/36] Disable aarch64 builds due to compiler failure --- .github/workflows/build_dist.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 3189d4a..879db7d 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,7 +69,8 @@ jobs: uses: pypa/cibuildwheel@v2.21 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" - CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" + # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" + CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86" CIBW_BUILD: ${{ matrix.cibw_build }} From b20fdf35e659d697bcc82a81ac0aec6bf95bdbc3 Mon Sep 17 00:00:00 2001 From: Rudolf Kolbe Date: Wed, 2 Apr 2025 00:03:43 +0200 Subject: [PATCH 03/36] build_dist workflow - (re)add arm support for Linux and Windows (#300) This enables Arm support for Linux and Windows. --- .github/workflows/build_dist.yml | 46 +++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 879db7d..ac70871 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -26,10 +26,8 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x - - name: Install setuptools - run: pip install setuptools - name: Build sdist - run: python setup.py sdist + run: pipx run build --sdist - name: Save sdist uses: actions/upload-artifact@v4 with: @@ -72,11 +70,47 @@ jobs: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches - CIBW_ARCHS_WINDOWS: "AMD64 x86" + CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_BUILD: ${{ matrix.cibw_build }} + CIBW_SKIP: "cp*-musllinux*" + CIBW_TEST_COMMAND: "tox -c {project}" + CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" + CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" + - name: Save wheels + uses: actions/upload-artifact@v4 + with: + name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} + path: ./wheelhouse/*.whl + + wheels_linux_arm: + name: Build wheels on ${{ matrix.os }} CIBW_BUILD=${{ matrix.cibw_build }} + runs-on: ${{ matrix.os }} + strategy: + # since multiple builds run at the same time, cancelling them all when one + # fails is wasteful and forces handling build problems one by one instead + # of showing a "full picture" + fail-fast: false + matrix: + os: + - ubuntu-24.04-arm + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + steps: + - name: Check out repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + - name: Install Python 3.x + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: pypa/cibuildwheel@v2.21 + env: + CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" + CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_TEST_COMMAND: "tox -c {project}" - CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{aarch64,ppc64le,s390x}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v4 @@ -86,7 +120,7 @@ jobs: upload_pypi: name: Upload to PyPI - needs: [sdist, wheels] + needs: [sdist, wheels, wheels_linux_arm] runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: From 7633d8d48f4b8ff6da495e0c426ee292d00b97e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Mon, 17 Mar 2025 16:09:57 -0500 Subject: [PATCH 04/36] Enable CPython free-threaded wheel builds --- .github/workflows/build_dist.yml | 11 ++++++++- lz4/_version.c | 4 +++ lz4/block/_block.c | 4 +++ lz4/frame/_frame.c | 4 +++ lz4/stream/_stream.c | 4 +++ tests/block/conftest.py | 18 ++++++++++++++ tests/block/test_block_0.py | 14 ++++++++++- tests/block/test_block_3.py | 1 + tests/frame/test_frame_2.py | 7 ++++++ tests/frame/test_frame_5.py | 8 +++--- tests/frame/test_frame_6.py | 42 +++++++++++++++++++------------- tests/frame/test_frame_8.py | 8 +++--- tests/frame/test_frame_9.py | 25 +++++++++++-------- tests/stream/test_stream_0.py | 1 + tests/stream/test_stream_3.py | 1 + 15 files changed, 117 insertions(+), 35 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index ac70871..b1e325c 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-13 # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -63,6 +63,14 @@ jobs: uses: docker/setup-qemu-action@v3 with: platforms: all + + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" + echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: @@ -110,6 +118,7 @@ jobs: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" + CIBW_ENABLE: cpython-freethreading CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels diff --git a/lz4/_version.c b/lz4/_version.c index c611f0b..7795241 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -113,5 +113,9 @@ PyInit__version(void) if (module == NULL) return NULL; + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/block/_block.c b/lz4/block/_block.c index 3e904a0..daa9fd5 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -518,5 +518,9 @@ PyInit__block(void) Py_INCREF(LZ4BlockError); PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index 3460665..d081cce 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1677,5 +1677,9 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX1MB", LZ4F_max1MB); PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 522fded..6351d96 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1649,5 +1649,9 @@ PyInit__stream(void) Py_INCREF (LZ4StreamError); PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); + #ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + #endif + return module; } diff --git a/tests/block/conftest.py b/tests/block/conftest.py index 0b3578e..b25c239 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,6 +3,24 @@ import sys +class EmptyMemoryView(): + def __init__(self): + self.data = b'' + self.view = None + + def __buffer__(self, flags: int, /) -> memoryview: + if self.view is None: + self.view = memoryview(self.data) + return self.view + + def __release_buffer__(self, buffer: memoryview, /): + breakpoint() + buffer.release() + + def __len__(self): + return 0 + + test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8fc0f48..31513b4 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -1,6 +1,9 @@ import lz4.block from multiprocessing.pool import ThreadPool import sys +import copy +import inspect +import pytest from functools import partial if sys.version_info <= (3, 2): import struct @@ -79,10 +82,19 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti # Test multi threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) - data_in = [data for i in range(32)] + def copy_buf(data): + data_x = data + if isinstance(data, memoryview): + data_x = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + return data_x + + data_in = [copy_buf(data) for i in range(32)] pool = ThreadPool(2) rt = partial(roundtrip, c_kwargs=c_kwargs, diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 3fcb175..88461b7 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,6 +18,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 80b44b8..14a3f6b 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -1,6 +1,8 @@ import lz4.frame as lz4frame import pytest import os +import copy +import inspect import sys from . helpers import ( get_chunked, @@ -41,6 +43,11 @@ def test_roundtrip_chunked(data, block_size, block_linked, data, c_chunks, d_chunks = data + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + c_context = lz4frame.create_compression_context() kwargs = {} diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 05daf28..4083646 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,6 +8,8 @@ (b'a' * 1024 * 1024), ] +pytestmark = pytest.mark.thread_unsafe + @pytest.fixture( params=test_data, @@ -66,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(data): +def test_frame_open_decompress_mem_usage(tmp_path, data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open('test.lz4', 'w') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open('test.lz4', 'r') as f: + with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: diff --git a/tests/frame/test_frame_6.py b/tests/frame/test_frame_6.py index c20a4f3..4f4185e 100644 --- a/tests/frame/test_frame_6.py +++ b/tests/frame/test_frame_6.py @@ -1,5 +1,6 @@ import os import pytest +import threading import lz4.frame as lz4frame test_data = [ @@ -33,40 +34,45 @@ def compression_level(request): return request.param -def test_lz4frame_open_write(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) -def test_lz4frame_open_write_read_defaults(data): - with lz4frame.open('testfile', mode='wb') as fp: +def test_lz4frame_open_write_read_defaults(tmp_path, data): + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wb') as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text(): +def test_lz4frame_open_write_read_text(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data def test_lz4frame_open_write_read( + tmp_path, data, compression_level, block_linked, @@ -91,29 +97,31 @@ def test_lz4frame_open_write_read( kwargs['return_bytearray'] = return_bytearray kwargs['mode'] = 'wb' - with lz4frame.open('testfile', **kwargs) as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', **kwargs) as fp: fp.write(data) - with lz4frame.open('testfile', mode='r') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='r') as fp: data_out = fp.read() assert data_out == data -def test_lz4frame_flush(): +def test_lz4frame_flush(tmp_path): data_1 = b"This is a..." data_2 = b" test string!" + thread_id = threading.get_native_id() - with lz4frame.open("testfile", mode="w") as fp_write: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="w") as fp_write: fp_write.write(data_1) fp_write.flush() fp_write.write(data_2) - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 fp_write.flush() - with lz4frame.open("testfile", mode="r") as fp_read: + with lz4frame.open(tmp_path / f"testfile_{thread_id}", mode="r") as fp_read: assert fp_read.read() == data_1 + data_2 diff --git a/tests/frame/test_frame_8.py b/tests/frame/test_frame_8.py index 159534a..cfaeaac 100644 --- a/tests/frame/test_frame_8.py +++ b/tests/frame/test_frame_8.py @@ -1,12 +1,14 @@ +import threading import lz4.frame as lz4frame -def test_lz4frame_open_write_read_text_iter(): +def test_lz4frame_open_write_read_text_iter(tmp_path): data = u'This is a test string' - with lz4frame.open('testfile', mode='wt') as fp: + thread_id = threading.get_native_id() + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='wt') as fp: fp.write(data) data_out = '' - with lz4frame.open('testfile', mode='rt') as fp: + with lz4frame.open(tmp_path / f'testfile_{thread_id}', mode='rt') as fp: for line in fp: data_out += line assert data_out == data diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index 5143393..c5335ae 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -3,11 +3,12 @@ import io import pickle import sys +import threading import lz4.frame import pytest -def test_issue_172_1(): +def test_issue_172_1(tmp_path): """Test reproducer for issue 172 Issue 172 is a reported failure occurring on Windows 10 only. This bug was @@ -16,34 +17,38 @@ def test_issue_172_1(): """ input_data = 8 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_2(): +def test_issue_172_2(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'w') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'w') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'r') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'r') as fp: data = fp.read(10) assert len(data) == 10 -def test_issue_172_3(): +def test_issue_172_3(tmp_path): input_data = 9 * os.urandom(1024) - with lz4.frame.open('testfile_small', 'wb') as fp: + thread_id = threading.get_native_id() + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'wb') as fp: bytes_written = fp.write(input_data) # noqa: F841 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(10) assert len(data) == 10 - with lz4.frame.open('testfile_small', 'rb') as fp: + with lz4.frame.open(tmp_path / f'testfile_small_{thread_id}', 'rb') as fp: data = fp.read(16 * 1024 - 1) assert len(data) == 9 * 1024 assert data == input_data diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index 03b19f3..cac07bd 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -96,6 +96,7 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, # Test single threaded usage with all valid variations of input +@pytest.mark.thread_unsafe def test_1(data, strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray, dictionary): if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 2b52d6b..fed93d2 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,6 +71,7 @@ def data(request): return request.param +@pytest.mark.thread_unsafe def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From 0f5c05e7ccfc35f5f1d2d61cebb9f40480116444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 11:42:55 -0500 Subject: [PATCH 05/36] Update cibuildwheel version --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b1e325c..38f40dd 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -72,7 +72,7 @@ jobs: echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" From c0b3bb5c236cec8e1563411228d5133893c1a7f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 12:32:44 -0500 Subject: [PATCH 06/36] Variable misspell fix --- lz4/_version.c | 2 +- lz4/block/_block.c | 2 +- lz4/frame/_frame.c | 2 +- lz4/stream/_stream.c | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lz4/_version.c b/lz4/_version.c index 7795241..af606ab 100644 --- a/lz4/_version.c +++ b/lz4/_version.c @@ -114,7 +114,7 @@ PyInit__version(void) return NULL; #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/block/_block.c b/lz4/block/_block.c index daa9fd5..993cc44 100644 --- a/lz4/block/_block.c +++ b/lz4/block/_block.c @@ -519,7 +519,7 @@ PyInit__block(void) PyModule_AddObject(module, "LZ4BlockError", LZ4BlockError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index d081cce..e62c72c 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1678,7 +1678,7 @@ PyInit__frame(void) PyModule_AddIntConstant (module, "BLOCKSIZE_MAX4MB", LZ4F_max4MB); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index 6351d96..f0dfad5 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -1650,7 +1650,7 @@ PyInit__stream(void) PyModule_AddObject (module, "LZ4StreamError", LZ4StreamError); #ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(mod, Py_MOD_GIL_NOT_USED); + PyUnstable_Module_SetGIL(module, Py_MOD_GIL_NOT_USED); #endif return module; From 99321aa0166e6398276988a4d8c800568239177b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:38:33 -0500 Subject: [PATCH 07/36] Update environment variables --- .github/workflows/build_dist.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 38f40dd..77862f1 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,15 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "TOX_OVERRIDE=testenv.deps+=pytest-run-parallel" >> "$GITHUB_ENV" - echo "PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches From 2258e28190e3f037617ae48fe329035b60c28ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 14:44:07 -0500 Subject: [PATCH 08/36] Ensure CIBW_ENVIRONMENT is set --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 77862f1..0c659e3 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,12 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4="False" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From a20d167678f4fec73d41e811f809ec2f1799bab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:28:12 -0500 Subject: [PATCH 09/36] Pass tox overrides through CLI --- .github/workflows/build_dist.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 0c659e3..62d84da 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -69,12 +69,14 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4 TOX_OVERRIDE=testenv.deps+=pytest-run-parallel;testenv.pass_env=PYTEST_ADDOPTS" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: From cc5c9b18002863ebb4fcff7ff58db05cdd3e0db3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:39:14 -0500 Subject: [PATCH 10/36] Ensure test command is applied --- .github/workflows/build_dist.yml | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 62d84da..e6541dd 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -84,9 +84,9 @@ jobs: CIBW_ARCHS_LINUX: "x86_64 i686" CIBW_ARCHS_MACOS: "auto64" # since we have both runner arches CIBW_ARCHS_WINDOWS: "AMD64 x86 ARM64" + CIBW_ENABLE: cpython-freethreading CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] steps: - name: Check out repository uses: actions/checkout@v4 @@ -116,15 +116,26 @@ jobs: uses: actions/setup-python@v5 with: python-version: 3.x + - name: Setup free-threading variables + if: ${{ endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" + - name: Setup environment + if: ${{ !endsWith(matrix.cibw_build, 't-*') }} + shell: bash -l {0} + run: | + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" + echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels uses: pypa/cibuildwheel@v2.21 env: - CIBW_ENVIRONMENT: PYLZ4_USE_SYSTEM_LZ4="False" CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading - CIBW_TEST_COMMAND: "tox -c {project}" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels uses: actions/upload-artifact@v4 From 40bcbd8c88c27416b9dc430e65e7f07741c7dfa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Tue, 1 Apr 2025 15:57:10 -0500 Subject: [PATCH 11/36] Check why using temp paths increase memory usage --- tests/frame/test_frame_5.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 4083646..dcbe4ae 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -68,17 +68,17 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot -def test_frame_open_decompress_mem_usage(tmp_path, data): +def test_frame_open_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() - with lz4.frame.open(tmp_path / 'test.lz4', 'w') as f: + with lz4.frame.open('test.lz4', 'w') as f: f.write(data) prev_snapshot = None for i in range(1000): - with lz4.frame.open(tmp_path / 'test.lz4', 'r') as f: + with lz4.frame.open('test.lz4', 'r') as f: decompressed = f.read() # noqa: F841 if i % 100 == 0: From ad3ec0d5810615afc9dff3d2b18f6d29d2066223 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:13:03 -0500 Subject: [PATCH 12/36] Copy bytearrays directly --- tests/block/conftest.py | 18 ------------------ tests/block/test_block_0.py | 3 ++- tests/frame/test_frame_2.py | 4 +++- 3 files changed, 5 insertions(+), 20 deletions(-) diff --git a/tests/block/conftest.py b/tests/block/conftest.py index b25c239..0b3578e 100644 --- a/tests/block/conftest.py +++ b/tests/block/conftest.py @@ -3,24 +3,6 @@ import sys -class EmptyMemoryView(): - def __init__(self): - self.data = b'' - self.view = None - - def __buffer__(self, flags: int, /) -> memoryview: - if self.view is None: - self.view = memoryview(self.data) - return self.view - - def __release_buffer__(self, buffer: memoryview, /): - breakpoint() - buffer.release() - - def __len__(self): - return 0 - - test_data = [ (b''), (os.urandom(8 * 1024)), diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 31513b4..8a22e57 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -91,7 +91,8 @@ def copy_buf(data): if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data_x = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_x = bytearray() + data_x[:] = data return data_x data_in = [copy_buf(data) for i in range(32)] diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index 14a3f6b..f7e6fa3 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -46,7 +46,9 @@ def test_roundtrip_chunked(data, block_size, block_linked, if isinstance(data, memoryview): data = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): - data = bytearray(copy.deepcopy(data.__buffer__(inspect.BufferFlags.FULL_RO).obj)) + data_2 = bytearray() + data_2[:] = data + data = data_2 c_context = lz4frame.create_compression_context() From 7a88fb9351ddfd65936dab78b79e41e910add51c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:22:17 -0500 Subject: [PATCH 13/36] Remove unused import --- tests/block/test_block_0.py | 1 - tests/frame/test_frame_2.py | 1 - 2 files changed, 2 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 8a22e57..f2c3cd1 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -2,7 +2,6 @@ from multiprocessing.pool import ThreadPool import sys import copy -import inspect import pytest from functools import partial if sys.version_info <= (3, 2): diff --git a/tests/frame/test_frame_2.py b/tests/frame/test_frame_2.py index f7e6fa3..230867e 100644 --- a/tests/frame/test_frame_2.py +++ b/tests/frame/test_frame_2.py @@ -2,7 +2,6 @@ import pytest import os import copy -import inspect import sys from . helpers import ( get_chunked, From 737bac1a663c4d15347e98359b7a11c7ddc96f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 2 Apr 2025 17:41:35 -0500 Subject: [PATCH 14/36] Copy memoryview on test_1 --- tests/block/test_block_0.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index f2c3cd1..561b185 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -70,6 +70,13 @@ def setup_kwargs(mode, store_size, c_return_bytearray=None, d_return_bytearray=N # Test single threaded usage with all valid variations of input def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dictionary): + if isinstance(data, memoryview): + data = memoryview(copy.deepcopy(data.obj)) + elif isinstance(data, bytearray): + data_x = bytearray() + data_x[:] = data + data = data_x + (c_kwargs, d_kwargs) = setup_kwargs( mode, store_size, c_return_bytearray, d_return_bytearray) From b7c0b665eb70706a1c6e334e54a1b9fa18c96cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 12:26:53 -0500 Subject: [PATCH 15/36] Upgrade pypa/cibuildwheel action in arm64 job --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e6541dd..4af62b6 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -130,7 +130,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.21 + uses: pypa/cibuildwheel@v2.23.2 env: CIBW_ARCHS_LINUX: "aarch64 armv7l" CIBW_BUILD: ${{ matrix.cibw_build }} From 61957cb0dacf51b4cf0f351b740119bb576428b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 14:45:57 -0500 Subject: [PATCH 16/36] CI: set parallelism to 2 in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 4af62b6..e327680 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=4" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From 689440cdabe92a8a407cf96e43847eec1b956ec0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:11:26 -0500 Subject: [PATCH 17/36] CI: disable pytest-run-parallel altogeogether in aarch64 --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e327680..6765e99 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -121,7 +121,7 @@ jobs: shell: bash -l {0} run: | echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=2" >> "$GITHUB_ENV" + echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment if: ${{ !endsWith(matrix.cibw_build, 't-*') }} From e54341fac7984222faec6e789ddc97486e6e88d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Thu, 3 Apr 2025 15:24:42 -0500 Subject: [PATCH 18/36] CI: disable armv7l --- .github/workflows/build_dist.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 6765e99..9b9964b 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -132,7 +132,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.2 env: - CIBW_ARCHS_LINUX: "aarch64 armv7l" + CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_SKIP: "cp*-musllinux*" CIBW_ENABLE: cpython-freethreading From 022d53815abf97e6eb49d0be3a27c74541c782ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Andr=C3=A9s=20Margffoy=20Tuay?= Date: Wed, 9 Apr 2025 16:43:32 -0500 Subject: [PATCH 19/36] Address review comments --- .github/workflows/build_dist.yml | 2 ++ tests/block/test_block_0.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 9b9964b..b907acc 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -120,6 +120,8 @@ jobs: if: ${{ endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | + # Variables are set in order to be passed down to both cibuildwheel and the + # Docker image spawned by that action echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index 561b185..a7731c3 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -93,12 +93,13 @@ def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) def copy_buf(data): - data_x = data if isinstance(data, memoryview): data_x = memoryview(copy.deepcopy(data.obj)) elif isinstance(data, bytearray): data_x = bytearray() data_x[:] = data + else: + data_x = data return data_x data_in = [copy_buf(data) for i in range(32)] From 708e6d49edf09079936c444c0516236dbe406426 Mon Sep 17 00:00:00 2001 From: Karolina Surma Date: Thu, 29 May 2025 12:06:30 +0200 Subject: [PATCH 20/36] Correct the import of _compression for Python 3.14 This is backwards compatible with all supported versions of Python. --- lz4/frame/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 00f3e64..60bf7dc 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -25,9 +25,9 @@ __doc__ = _doc try: - import _compression # Python 3.6 and later + import compression._common._streams as _compression # Python 3.14 except ImportError: - from . import _compression + import _compression # Python 3.6 - 3.13 BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT From d8387509dd17dcefec6041bf242598ada30a505f Mon Sep 17 00:00:00 2001 From: Karolina Surma <33810531+befeleme@users.noreply.github.com> Date: Tue, 7 Oct 2025 09:19:38 +0200 Subject: [PATCH 21/36] Update lz4/frame/__init__.py Co-authored-by: Christian Clauss --- lz4/frame/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 60bf7dc..91ad50b 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -27,7 +27,7 @@ try: import compression._common._streams as _compression # Python 3.14 except ImportError: - import _compression # Python 3.6 - 3.13 + import _compression # Python 3.9 - 3.13 BLOCKSIZE_DEFAULT = _BLOCKSIZE_DEFAULT From 5c9f5561e4353978746b2af6fac1338668e5473b Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 13:13:08 +0200 Subject: [PATCH 22/36] Keep GitHub Actions up to date with GitHub's Dependabot * [Keeping your software supply chain secure with Dependabot](https://docs.github.com/en/code-security/dependabot) * [Keeping your actions up to date with Dependabot](https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot) * [Configuration options for the `dependabot.yml` file - package-ecosystem](https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem) --- .github/dependabot.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..be006de --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,13 @@ +# Keep GitHub Actions up to date with GitHub's Dependabot... +# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot +# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + groups: + github-actions: + patterns: + - "*" # Group all Actions updates into a single larger pull request + schedule: + interval: weekly From 8c82a0df102fed1221bb5edf8dfa3f72017160a6 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 12:46:16 +0200 Subject: [PATCH 23/36] Build wheels for Python 3.14 Caution: The new `compression` module in Python >= 3.14 Standard Library causes compatibility problems. * https://docs.python.org/3.14/library/compression.html * #302 * #303 * #308 --- Python v3.14 -- October 7th * https://www.python.org/download/pre-releases * https://www.python.org/downloads/release/python-3140rc3 * https://docs.python.org/3.14/whatsnew/3.14.html What's new in Python 3.14: ___PEP 784: Adding Zstandard to the standard library___ * https://docs.python.org/3.14/whatsnew/3.14.html#whatsnew314-pep784 --- .github/workflows/build_dist.yml | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index b907acc..836e216 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -19,11 +19,11 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Build sdist @@ -45,17 +45,17 @@ jobs: matrix: os: - ubuntu-latest - - macos-13 # x86 + - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Set up QEMU # Needed to build aarch64 wheels @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.23.2 + uses: pypa/cibuildwheel@v3.2.0 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -106,14 +106,14 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: - fetch-depth: 0 # To ensure tags are retrieved to enabe setuptools_scm to work + fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: 3.x - name: Setup free-threading variables @@ -132,7 +132,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v2.23.2 + uses: pypa/cibuildwheel@v3.2.0 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} @@ -151,7 +151,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v5 with: pattern: cibw-* path: dist From 438df39de27dc1d44fdc45c6fd3256611758d099 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Mon, 20 Oct 2025 17:21:16 +0200 Subject: [PATCH 24/36] Remove cp314t from cibw_build matrix --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 836e216..1556839 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] steps: - name: Check out repository uses: actions/checkout@v5 From 849e1545f36ec5036052dad80e402c2e4d0b2ed2 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Wed, 24 Sep 2025 14:32:23 +0200 Subject: [PATCH 25/36] Fix typos discovered by codespell --- lz4/frame/__init__.py | 6 +++--- lz4/frame/_frame.c | 4 ++-- lz4/stream/__init__.py | 10 +++++----- lz4/stream/_stream.c | 2 +- tests/frame/test_frame_9.py | 2 +- tests/stream/test_stream_0.py | 2 +- tests/stream/test_stream_1.py | 10 +++++----- 7 files changed, 18 insertions(+), 18 deletions(-) diff --git a/lz4/frame/__init__.py b/lz4/frame/__init__.py index 91ad50b..2a82cd0 100644 --- a/lz4/frame/__init__.py +++ b/lz4/frame/__init__.py @@ -268,7 +268,7 @@ def flush(self): This returns a ``bytes`` or ``bytearray`` object containing any data stored in the compressor's internal buffers and a frame footer. - The LZ4FrameCompressor instance may be re-used after this method has + The LZ4FrameCompressor instance may be reused after this method has been called to create a new frame of compressed data. Returns: @@ -287,7 +287,7 @@ def flush(self): def reset(self): """Reset the `LZ4FrameCompressor` instance. - This allows the `LZ4FrameCompression` instance to be re-used after an + This allows the `LZ4FrameCompression` instance to be reused after an error. """ @@ -360,7 +360,7 @@ def __exit__(self, exception_type, exception, traceback): def reset(self): """Reset the decompressor state. - This is useful after an error occurs, allowing re-use of the instance. + This is useful after an error occurs, allowing reuse of the instance. """ reset_decompression_context(self._context) diff --git a/lz4/frame/_frame.c b/lz4/frame/_frame.c index e62c72c..440b0b5 100644 --- a/lz4/frame/_frame.c +++ b/lz4/frame/_frame.c @@ -1330,7 +1330,7 @@ PyDoc_STRVAR( ); #define COMPRESS_KWARGS_DOCSTRING \ - " block_size (int): Sepcifies the maximum blocksize to use.\n" \ + " block_size (int): Specifies the maximum blocksize to use.\n" \ " Options:\n\n" \ " - `lz4.frame.BLOCKSIZE_DEFAULT`: the lz4 library default\n" \ " - `lz4.frame.BLOCKSIZE_MAX64KB`: 64 kB\n" \ @@ -1466,7 +1466,7 @@ PyDoc_STRVAR "data will also be included in the returned data.\n" \ "\n" \ "If the ``end_frame`` argument is ``True``, the compression context will be\n" \ - "reset and can be re-used.\n" \ + "reset and can be reused.\n" \ "\n" \ "Args:\n" \ " context (cCtx): Compression context\n" \ diff --git a/lz4/stream/__init__.py b/lz4/stream/__init__.py index 4b00746..58ce298 100644 --- a/lz4/stream/__init__.py +++ b/lz4/stream/__init__.py @@ -32,7 +32,7 @@ def __init__(self, strategy, buffer_size, return_bytearray=False, store_comp_siz perform decompression using this initial dictionary. Raises: - Exceptions occuring during the context initialization. + Exceptions occurring during the context initialization. OverflowError: raised if the ``dictionary`` parameter is too large for the LZ4 context. @@ -73,7 +73,7 @@ def decompress(self, chunk): bytes or bytearray: Decompressed data. Raises: - Exceptions occuring during decompression. + Exceptions occurring during decompression. ValueError: raised if the source is inconsistent with a finite LZ4 stream block chain. @@ -96,7 +96,7 @@ def get_block(self, stream): bytes or bytearray: LZ4 compressed data block. Raises: - Exceptions occuring while getting the first block from ``stream``. + Exceptions occurring while getting the first block from ``stream``. BufferError: raised if the function cannot return a complete LZ4 compressed block from the stream (i.e. the stream does not hold @@ -150,7 +150,7 @@ def __init__(self, strategy, buffer_size, mode="default", acceleration=True, com perform compression using this initial dictionary. Raises: - Exceptions occuring during the context initialization. + Exceptions occurring during the context initialization. OverflowError: raised if the ``dictionary`` parameter is too large for the LZ4 context. @@ -194,7 +194,7 @@ def compress(self, chunk): bytes or bytearray: Compressed data. Raises: - Exceptions occuring during compression. + Exceptions occurring during compression. OverflowError: raised if the source is too large for being compressed in the given context. diff --git a/lz4/stream/_stream.c b/lz4/stream/_stream.c index f0dfad5..4c51d89 100644 --- a/lz4/stream/_stream.c +++ b/lz4/stream/_stream.c @@ -931,7 +931,7 @@ _create_context (PyObject * Py_UNUSED (self), PyObject * args, PyObject * kwds) context->output.len = buffer_size; total_size = context->output.len; - /* Here we cannot assert the maximal theorical decompressed chunk length + /* Here we cannot assert the maximal theoretical decompressed chunk length * will fit in one page of the double_buffer, i.e.: * assert( !(double_buffer.page_size < _LZ4_inputBound(store_max_size)) ) * diff --git a/tests/frame/test_frame_9.py b/tests/frame/test_frame_9.py index c5335ae..6f7fc0d 100644 --- a/tests/frame/test_frame_9.py +++ b/tests/frame/test_frame_9.py @@ -65,7 +65,7 @@ def test_issue_227_1(): @pytest.mark.skipif( sys.version_info < (3, 8), - reason="PickleBuffer only availiable in Python 3.8 or greater" + reason="PickleBuffer only available in Python 3.8 or greater" ) def test_issue_227_2(): q = array.array('Q', [1, 2, 3, 4, 5]) diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index cac07bd..3cd05d6 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -112,6 +112,6 @@ def test_1(data, strategy, mode, buffer_size, store_comp_size, # Test multi threaded: # Not relevant in the lz4.stream case (the process is highly sequential, -# and re-use/share the same context from one input chunk to the next one). +# and reuse/share the same context from one input chunk to the next one). def test_2(data, strategy, mode, buffer_size, store_comp_size, dictionary): # noqa pass diff --git a/tests/stream/test_stream_1.py b/tests/stream/test_stream_1.py index 6b49267..481de8a 100644 --- a/tests/stream/test_stream_1.py +++ b/tests/stream/test_stream_1.py @@ -136,7 +136,7 @@ def test_invalid_config_c_4(store_comp_size): c_kwargs.update(store_comp_size) if store_comp_size['store_comp_size'] >= 4: - # No need for skiping this test case, since arguments check is + # No need for skipping this test case, since arguments check is # expecting to raise an error. # Make sure the page size is larger than what the input bound will be, @@ -169,7 +169,7 @@ def test_invalid_config_d_4(store_comp_size): # but still fit in 4 bytes d_kwargs['buffer_size'] -= 1 - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization lz4.stream.LZ4StreamDecompressor(**d_kwargs) @@ -199,7 +199,7 @@ def test_invalid_config_d_5(): d_kwargs = {} d_kwargs['strategy'] = "double_buffer" - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE if sys.maxsize < 0xffffffff: @@ -207,7 +207,7 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = lz4.stream.LZ4_MAX_INPUT_SIZE + 1 if sys.maxsize < 0xffffffff: @@ -215,7 +215,7 @@ def test_invalid_config_d_5(): lz4.stream.LZ4StreamDecompressor(**d_kwargs) - # No failure expected during instanciation/initialization + # No failure expected during instantiation/initialization d_kwargs['buffer_size'] = _4GB - 1 # 4GB - 1 (to fit in 4 bytes) if sys.maxsize < 0xffffffff: From 59b2d8176072bdee50d38cc68ec65c33b928a980 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 17:15:14 +0000 Subject: [PATCH 26/36] Bump pypa/cibuildwheel in the github-actions group across 1 directory Bumps the github-actions group with 1 update in the / directory: [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel). Updates `pypa/cibuildwheel` from 3.2.0 to 3.2.1 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v3.2.0...v3.2.1) --- updated-dependencies: - dependency-name: pypa/cibuildwheel dependency-version: 3.2.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/build_dist.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1556839..1af3020 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.2.1 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -132,7 +132,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.2.1 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} From d5daffd8356e71cb075fef5071bbbe75b274d109 Mon Sep 17 00:00:00 2001 From: Christian Clauss Date: Thu, 23 Oct 2025 14:26:19 +0200 Subject: [PATCH 27/36] setup.py: Add support for Python 3.14 and Free Threading https://pypi.org/classifiers --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 9bc91f0..d9d8191 100644 --- a/setup.py +++ b/setup.py @@ -205,5 +205,7 @@ def pkgconfig_installed_check(lib, required_version, default): 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', + 'Programming Language :: Python :: 3.14', + 'Programming Language :: Python :: Free Threading', ], ) From 00e0ae93e7e7f81fcc9e9d9768a9b95c01360adb Mon Sep 17 00:00:00 2001 From: Hannes Braun Date: Tue, 3 Feb 2026 15:06:43 +0100 Subject: [PATCH 28/36] Switch to SPDX license expression License classifiers are deprecated. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d9d8191..350b551 100644 --- a/setup.py +++ b/setup.py @@ -180,6 +180,7 @@ def pkgconfig_installed_check(lib, required_version, default): long_description=open('README.rst', 'r').read(), author='Jonathan Underwood', author_email='jonathan.underwood@gmail.com', + license='BSD-3-Clause', url='https://github.com/python-lz4/python-lz4', packages=packages, ext_modules=ext_modules, @@ -196,7 +197,6 @@ def pkgconfig_installed_check(lib, required_version, default): }, classifiers=[ 'Development Status :: 5 - Production/Stable', - 'License :: OSI Approved :: BSD License', 'Intended Audience :: Developers', 'Programming Language :: C', 'Programming Language :: Python', From e000ef843a760998462f9a7b10b52d6bddbb8532 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 08:31:48 -0700 Subject: [PATCH 29/36] Enable cp314t wheel builds --- .github/workflows/build_dist.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index 1af3020..e2fd4fc 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -48,7 +48,7 @@ jobs: - macos-15-intel # x86 - macos-latest # arm - windows-latest - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -106,7 +106,7 @@ jobs: matrix: os: - ubuntu-24.04-arm - cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*] + cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository uses: actions/checkout@v5 @@ -116,17 +116,7 @@ jobs: uses: actions/setup-python@v6 with: python-version: 3.x - - name: Setup free-threading variables - if: ${{ endsWith(matrix.cibw_build, 't-*') }} - shell: bash -l {0} - run: | - # Variables are set in order to be passed down to both cibuildwheel and the - # Docker image spawned by that action - echo "CIBW_BEFORE_TEST=pip install pytest pytest-run-parallel" >> "$GITHUB_ENV" - echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False PYTEST_ADDOPTS=--parallel-threads=1" >> "$GITHUB_ENV" - echo "CIBW_TEST_COMMAND=tox -x testenv.deps+=pytest-run-parallel -x testenv.pass_env+=PYTEST_ADDOPTS -c {project}" >> "$GITHUB_ENV" - name: Setup environment - if: ${{ !endsWith(matrix.cibw_build, 't-*') }} shell: bash -l {0} run: | echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" From 8427b5691dea5909768c805bc4fc3b1935549fa1 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 09:20:20 -0700 Subject: [PATCH 30/36] mark tests that use tracemalloc as thread-unsafe --- tests/block/test_block_2.py | 4 ++++ tests/block/test_block_3.py | 4 +++- tests/frame/test_frame_5.py | 9 +++++++++ tests/stream/test_stream_3.py | 4 +++- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/block/test_block_2.py b/tests/block/test_block_2.py index a2aea4d..a75e92f 100644 --- a/tests/block/test_block_2.py +++ b/tests/block/test_block_2.py @@ -19,6 +19,10 @@ # fragile. +@pytest.mark.thread_unsafe( + reason=("Large multithreaded allocations will likely exhaust " + "system memory.") +) @pytest.mark.skipif( os.environ.get('TRAVIS') is not None, reason='Skipping test on Travis due to insufficient memory' diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 88461b7..07f8db2 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,7 +18,9 @@ def data(request): return request.param -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_block_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index dcbe4ae..85e6fa0 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -21,6 +21,9 @@ def data(request): return request.param +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') @@ -43,6 +46,9 @@ def test_frame_decompress_mem_usage(data): prev_snapshot = snapshot +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_decompress_chunk_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() @@ -68,6 +74,9 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_frame_open_decompress_mem_usage(data): tracemalloc = pytest.importorskip('tracemalloc') tracemalloc.start() diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index fed93d2..58094e1 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,7 +71,9 @@ def data(request): return request.param -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason="tracemalloc captures global snapshots" +) def test_block_decompress_mem_usage(data, buffer_size): kwargs = { 'strategy': "double_buffer", From 30a7a7eac52b222f2842710727eba40222cef22e Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 09:38:16 -0700 Subject: [PATCH 31/36] delete unnecessary module-scope mark --- tests/frame/test_frame_5.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 85e6fa0..497c8a1 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,9 +8,6 @@ (b'a' * 1024 * 1024), ] -pytestmark = pytest.mark.thread_unsafe - - @pytest.fixture( params=test_data, ids=[ From c5c5130b2ed1cf276b9f01d7a4e6fe8c6cff9a36 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 10:30:37 -0700 Subject: [PATCH 32/36] add missing thread-unsafe reasons --- tests/block/test_block_0.py | 5 ++++- tests/frame/test_frame_5.py | 1 + tests/stream/test_stream_0.py | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/block/test_block_0.py b/tests/block/test_block_0.py index a7731c3..f39da18 100644 --- a/tests/block/test_block_0.py +++ b/tests/block/test_block_0.py @@ -88,7 +88,10 @@ def test_1(data, mode, store_size, c_return_bytearray, d_return_bytearray, dicti # Test multi threaded usage with all valid variations of input -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason=("Test is already explicitly multithreaded, should " + "not be run in a thread pool.") +) def test_2(data, mode, store_size, dictionary): (c_kwargs, d_kwargs) = setup_kwargs(mode, store_size) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 497c8a1..9e4b83a 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -8,6 +8,7 @@ (b'a' * 1024 * 1024), ] + @pytest.fixture( params=test_data, ids=[ diff --git a/tests/stream/test_stream_0.py b/tests/stream/test_stream_0.py index 3cd05d6..c7db76e 100644 --- a/tests/stream/test_stream_0.py +++ b/tests/stream/test_stream_0.py @@ -96,7 +96,10 @@ def setup_kwargs(strategy, mode, buffer_size, store_comp_size, # Test single threaded usage with all valid variations of input -@pytest.mark.thread_unsafe +@pytest.mark.thread_unsafe( + reason=("test modifies fixtures that would be shared, see " + "https://github.com/Quansight-Labs/pytest-run-parallel/issues/14") +) def test_1(data, strategy, mode, buffer_size, store_comp_size, c_return_bytearray, d_return_bytearray, dictionary): if buffer_size >= (1 << (8 * store_comp_size['store_comp_size'])): From ebb04a3525e7d146471bd2c394ba4aa63053db41 Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 10:30:55 -0700 Subject: [PATCH 33/36] suppress warnings when pytest-run-parallel isn't available --- tests/conftest.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/conftest.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..0fefd8e --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,13 @@ +try: + import pytest_run_parallel # noqa: F401 + PARALLEL_RUN_AVALIABLE = True +except ModuleNotFoundError: + PARALLEL_RUN_AVALIABLE = False + + +def pytest_configure(config): + if not PARALLEL_RUN_AVALIABLE: + config.addinivalue_line( + "markers", + "thread_unsafe: mark the test function as single-threaded", + ) From 43b672d78f728cc2ea33cd5f0a5949cfcf88572f Mon Sep 17 00:00:00 2001 From: Nathan Goldbaum Date: Mon, 3 Nov 2025 13:09:23 -0700 Subject: [PATCH 34/36] disable coverage for tests that measure memory usage --- tests/block/test_block_3.py | 2 ++ tests/frame/test_frame_5.py | 6 ++++++ tests/stream/test_stream_3.py | 2 ++ 3 files changed, 10 insertions(+) diff --git a/tests/block/test_block_3.py b/tests/block/test_block_3.py index 07f8db2..86bd993 100644 --- a/tests/block/test_block_3.py +++ b/tests/block/test_block_3.py @@ -18,6 +18,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) diff --git a/tests/frame/test_frame_5.py b/tests/frame/test_frame_5.py index 9e4b83a..29e7765 100644 --- a/tests/frame/test_frame_5.py +++ b/tests/frame/test_frame_5.py @@ -19,6 +19,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) @@ -44,6 +46,8 @@ def test_frame_decompress_mem_usage(data): prev_snapshot = snapshot +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) @@ -72,6 +76,8 @@ def test_frame_decompress_chunk_mem_usage(data): prev_snapshot = snapshot +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) diff --git a/tests/stream/test_stream_3.py b/tests/stream/test_stream_3.py index 58094e1..a18ed56 100644 --- a/tests/stream/test_stream_3.py +++ b/tests/stream/test_stream_3.py @@ -71,6 +71,8 @@ def data(request): return request.param +# coverage might allocate and cause this test to fail +@pytest.mark.no_cover @pytest.mark.thread_unsafe( reason="tracemalloc captures global snapshots" ) From 33612bddc2fa9df7471e01e07bf597159aba2e07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:37:01 +0000 Subject: [PATCH 35/36] Bump the github-actions group across 1 directory with 4 updates Bumps the github-actions group with 4 updates in the / directory: [actions/checkout](https://github.com/actions/checkout), [actions/upload-artifact](https://github.com/actions/upload-artifact), [pypa/cibuildwheel](https://github.com/pypa/cibuildwheel) and [actions/download-artifact](https://github.com/actions/download-artifact). Updates `actions/checkout` from 5 to 6 - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v5...v6) Updates `actions/upload-artifact` from 4 to 5 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v4...v5) Updates `pypa/cibuildwheel` from 3.2.1 to 3.3.0 - [Release notes](https://github.com/pypa/cibuildwheel/releases) - [Changelog](https://github.com/pypa/cibuildwheel/blob/main/docs/changelog.md) - [Commits](https://github.com/pypa/cibuildwheel/compare/v3.2.1...v3.3.0) Updates `actions/download-artifact` from 5 to 6 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/upload-artifact dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: pypa/cibuildwheel dependency-version: 3.3.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: actions/download-artifact dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/build_dist.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_dist.yml b/.github/workflows/build_dist.yml index e2fd4fc..d33b352 100644 --- a/.github/workflows/build_dist.yml +++ b/.github/workflows/build_dist.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -29,7 +29,7 @@ jobs: - name: Build sdist run: pipx run build --sdist - name: Save sdist - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-sdist.tar.gz path: dist/*.tar.gz @@ -51,7 +51,7 @@ jobs: cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -78,7 +78,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.1 + uses: pypa/cibuildwheel@v3.3.0 env: # CIBW_ARCHS_LINUX: "x86_64 i686 aarch64" CIBW_ARCHS_LINUX: "x86_64 i686" @@ -90,7 +90,7 @@ jobs: CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64 *-*linux_{ppc64le,s390x} *-win_arm64" CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl @@ -109,7 +109,7 @@ jobs: cibw_build: [cp39-*, cp310-*, cp311-*, cp312-*, cp313-*, cp313t-*, cp314-*, cp314t-*] steps: - name: Check out repository - uses: actions/checkout@v5 + uses: actions/checkout@v6 with: fetch-depth: 0 # To ensure tags are retrieved to enable setuptools_scm to work - name: Install Python 3.x @@ -122,7 +122,7 @@ jobs: echo "CIBW_ENVIRONMENT=PYLZ4_USE_SYSTEM_LZ4=False" >> "$GITHUB_ENV" echo "CIBW_TEST_COMMAND=tox -c {project}" >> "$GITHUB_ENV" - name: Build wheels - uses: pypa/cibuildwheel@v3.2.1 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_ARCHS_LINUX: "aarch64" CIBW_BUILD: ${{ matrix.cibw_build }} @@ -130,7 +130,7 @@ jobs: CIBW_ENABLE: cpython-freethreading CIBW_BEFORE_BUILD: "python -m pip install -U pip && python -m pip install tox" - name: Save wheels - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v5 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: ./wheelhouse/*.whl @@ -141,7 +141,7 @@ jobs: runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') steps: - - uses: actions/download-artifact@v5 + - uses: actions/download-artifact@v6 with: pattern: cibw-* path: dist From 029f36597b8d182484f3c38c6be6a71ad07ddc96 Mon Sep 17 00:00:00 2001 From: Johannes Jordan Date: Mon, 12 Jan 2026 13:29:35 +0100 Subject: [PATCH 36/36] Strip debug symbols from lz4 build When a wheel is built using lz4 library source files bundled with the package, we advise the linker to strip debug symbols from the binary output. This leads to a significant size reduction of the wheel. Fixes #305 --- setup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.py b/setup.py index 350b551..eb9030f 100644 --- a/setup.py +++ b/setup.py @@ -121,6 +121,8 @@ def pkgconfig_installed_check(lib, required_version, default): '-Wall', '-Wundef' ] + # strip debug symbols from libraries to reduce wheel size + extension_kwargs['extra_link_args'] = ['-s'] else: print('Unrecognized compiler: {0}'.format(compiler)) sys.exit(1)