From 6e11368f5d54f7d4878dc209717495c37be03c68 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:35:16 +0900 Subject: [PATCH 01/55] update Cython to 3.0.11 (#617) --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1164a941..b677f068 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1 @@ -# Also declared in pyproject.toml, if updating here please also update there. -Cython~=3.0.10 +Cython~=3.0.11 From 9e26d80ab2a02221d3ca36cc0b5ca2268f391204 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 17:56:01 +0900 Subject: [PATCH 02/55] update cibuildwheel to 2.20.0 (#618) --- .github/workflows/wheel.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d57e0586..01d0bbd1 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.17.0 + uses: pypa/cibuildwheel@v2.20.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -40,13 +40,11 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: pp* - - name: Build pure Python wheel + - name: Build sdist if: runner.os == 'Linux' - env: - MSGPACK_PUREPYTHON: "1" run: | pip install build - python -m build -w -o wheelhouse + python -m build -s -o wheelhouse - name: Upload Wheels to artifact uses: actions/upload-artifact@v4 From 9d0c7f2f9cab21c3e39d23001cd9d2034824ee61 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 19 Aug 2024 20:36:26 +0900 Subject: [PATCH 03/55] Release v1.1.0rc2 (#619) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 01d0bbd1..50157223 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -38,7 +38,7 @@ jobs: CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_ARCHS_LINUX: auto aarch64 CIBW_ARCHS_MACOS: x86_64 universal2 arm64 - CIBW_SKIP: pp* + CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist if: runner.os == 'Linux' diff --git a/ChangeLog.rst b/ChangeLog.rst index 9b16e41f..47328ffb 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0rc2 +======== + +Release Date: 2024-08-19 + +* Update Cython to 3.0.11 for better Python 3.13 support. +* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + 1.1.0rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index c2525444..a72e974c 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc1") -__version__ = "1.1.0rc1" +version = (1, 1, 0, "rc2") +__version__ = "1.1.0rc2" if os.environ.get("MSGPACK_PUREPYTHON"): From 20a2b8eaa26d76169049ce150e10d1d2aa37d3ab Mon Sep 17 00:00:00 2001 From: Thomas A Caswell Date: Wed, 21 Aug 2024 01:56:00 -0400 Subject: [PATCH 04/55] use `PyLong_*` instead of `PyInt_*` (#620) 9af421163cb8081414be347038dee7a82b29e8dd in Cython removed back-compatibility `#define`. --- msgpack/unpack.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 23aa6220..58a2f4f5 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -47,7 +47,7 @@ static inline msgpack_unpack_object unpack_callback_root(unpack_user* u) static inline int unpack_callback_uint16(unpack_user* u, uint16_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong((long)d); + PyObject *p = PyLong_FromLong((long)d); if (!p) return -1; *o = p; @@ -61,7 +61,7 @@ static inline int unpack_callback_uint8(unpack_user* u, uint8_t d, msgpack_unpac static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromSize_t((size_t)d); + PyObject *p = PyLong_FromSize_t((size_t)d); if (!p) return -1; *o = p; @@ -74,7 +74,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp if (d > LONG_MAX) { p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } if (!p) return -1; @@ -84,7 +84,7 @@ static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unp static inline int unpack_callback_int32(unpack_user* u, int32_t d, msgpack_unpack_object* o) { - PyObject *p = PyInt_FromLong(d); + PyObject *p = PyLong_FromLong(d); if (!p) return -1; *o = p; @@ -107,7 +107,7 @@ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpac if (d > LONG_MAX || d < LONG_MIN) { p = PyLong_FromLongLong((PY_LONG_LONG)d); } else { - p = PyInt_FromLong((long)d); + p = PyLong_FromLong((long)d); } *o = p; return 0; From 4587393b1ae2c9ebbd5bc93005b8aea2c8050b27 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 10 Sep 2024 01:58:00 +0900 Subject: [PATCH 05/55] release v1.1.0 (#622) --- ChangeLog.rst | 8 ++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 47328ffb..863c6b2c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,11 @@ +1.1.0 +===== + +Release Date: 2024-09-10 + +* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with + future Cython. (#620) + 1.1.0rc2 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index a72e974c..b6151054 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0, "rc2") -__version__ = "1.1.0rc2" +version = (1, 1, 0) +__version__ = "1.1.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0eeabfb453844b441a4a77097b3d5aa0cb6645b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Ram=C3=ADrez=20Mondrag=C3=B3n?= <16805946+edgarrmondragon@users.noreply.github.com> Date: Tue, 8 Oct 2024 03:04:56 -0600 Subject: [PATCH 06/55] Add Python 3.13 trove classifier (#626) --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index d041d4c0..e24f2b88 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Intended Audience :: Developers", From 868aa2cd83f39237deb957c68ce7232422a5950b Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 31 May 2025 12:45:06 +0900 Subject: [PATCH 07/55] update Cython to 3.1.1 (#637) --- .github/workflows/test.yml | 9 +++++++-- pyproject.toml | 12 +++++++----- requirements.txt | 2 +- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 530238c9..23d221c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.13-dev", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -26,10 +26,15 @@ jobs: allow-prereleases: true cache: "pip" - - name: Build + - name: Prepare shell: bash run: | + pip install -U pip pip install -r requirements.txt pytest + + - name: Build + shell: bash + run: | make cython pip install . diff --git a/pyproject.toml b/pyproject.toml index e24f2b88..b1628322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,10 +1,13 @@ [build-system] -requires = ["setuptools >= 69.5.1"] +# 75.3.0 is the latest version supporting Python 3.8 +requires = ["setuptools >= 75.3.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] +# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. +# https://github.com/msgpack/msgpack-python/pull/637 license = {text="Apache 2.0"} authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" @@ -14,18 +17,17 @@ requires-python = ">=3.8" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 3", + "Topic :: File Formats", + "Intended Audience :: Developers", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", - "Intended Audience :: Developers", - "License :: OSI Approved :: Apache Software License", ] [project.urls] diff --git a/requirements.txt b/requirements.txt index b677f068..78a2f38f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.0.11 +Cython~=3.1.1 From cdc764450370ff80e7c83edbe8d015f08f6fb9b3 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sun, 1 Jun 2025 16:56:44 +0900 Subject: [PATCH 08/55] update cibuildwheel to v2.23.3 (#638) --- .github/workflows/wheel.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 50157223..d97de1dd 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -9,17 +9,12 @@ jobs: build_wheels: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + # macos-13 is for intel + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} steps: - - name: Set up QEMU - if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 - with: - platforms: all - - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: @@ -32,12 +27,10 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.20.0 + uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_ARCHS_LINUX: auto aarch64 - CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist From fe9e620a607702b31476f092ad01a387cff4cfbd Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 2 Jun 2025 14:46:53 +0900 Subject: [PATCH 09/55] upload to PyPI on create a release (#639) --- .github/workflows/wheel.yml | 48 ++++++++++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index d97de1dd..686d7dd0 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -1,8 +1,10 @@ -name: Build Wheels +name: Build sdist and Wheels on: push: branches: [main] - create: + release: + types: + - published workflow_dispatch: jobs: @@ -34,7 +36,7 @@ jobs: CIBW_SKIP: "pp* cp38-macosx_*" - name: Build sdist - if: runner.os == 'Linux' + if: runner.os == 'Linux' && runner.arch == 'X64' run: | pip install build python -m build -s -o wheelhouse @@ -44,3 +46,43 @@ jobs: with: name: wheels-${{ matrix.os }} path: wheelhouse + + # combine all wheels into one artifact + combine_wheels: + needs: [build_wheels] + runs-on: ubuntu-latest + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Upload Wheels to artifact + uses: actions/upload-artifact@v4 + with: + name: wheels-all + path: dist + + # https://github.com/pypa/cibuildwheel/blob/main/examples/github-deploy.yml + upload_pypi: + needs: [build_wheels] + runs-on: ubuntu-latest + environment: pypi + permissions: + id-token: write + if: github.event_name == 'release' && github.event.action == 'published' + # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) + # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') + steps: + - uses: actions/download-artifact@v4 + with: + # unpacks all CIBW artifacts into dist/ + pattern: wheels-* + path: dist + merge-multiple: true + + - uses: pypa/gh-action-pypi-publish@release/v1 + #with: + # To test: repository-url: https://test.pypi.org/legacy/ From e6445d3b922ca0b9bc82695dd9d1c1529763095a Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Jun 2025 09:38:38 +0900 Subject: [PATCH 10/55] v1.1.1rc1 --- ChangeLog.rst | 7 +++++++ docs/conf.py | 6 +++--- msgpack/__init__.py | 4 ++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 863c6b2c..0f67502c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1rc1 +======== + +Release Date: 2025-06-06 + +* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + 1.1.0 ===== diff --git a/docs/conf.py b/docs/conf.py index ab0ad3c8..28116cd6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -12,9 +12,9 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#import os -#import sys -#sys.path.insert(0, os.path.abspath('..')) +# import os +# import sys +# sys.path.insert(0, os.path.abspath('..')) # -- General configuration ----------------------------------------------------- diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b6151054..b8d83da8 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 0) -__version__ = "1.1.0" +version = (1, 1, 1, "rc1") +__version__ = "1.1.1rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From 42f056f3cfaf2e3ec220db2f864e7613d433ad48 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 13 Jun 2025 15:41:08 +0900 Subject: [PATCH 11/55] v1.1.1 --- ChangeLog.rst | 7 +++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/ChangeLog.rst b/ChangeLog.rst index 0f67502c..418c444c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,10 @@ +1.1.1 +===== + +Release Date: 2025-06-13 + +* No change from 1.1.1rc1. + 1.1.1rc1 ======== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index b8d83da8..ad68271d 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 1, "rc1") -__version__ = "1.1.1rc1" +version = (1, 1, 1) +__version__ = "1.1.1" if os.environ.get("MSGPACK_PUREPYTHON"): From d9873dab049c60d5fd553223b9be10f1dcb56929 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Sat, 26 Jul 2025 10:59:05 +0900 Subject: [PATCH 12/55] ci: update cibuildwheel and drop Python 3.8 (#642) --- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23d221c8..60894797 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 686d7dd0..12600363 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -29,11 +29,11 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v2.23.3 + uses: pypa/cibuildwheel@v3.1.1 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_SKIP: "pp* cp38-*" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/requirements.txt b/requirements.txt index 78a2f38f..5d2e20b1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.1.1 +Cython~=3.1.2 From c2a9f1fda5401de2193c43d7a4f93bfc0b2fc1e6 Mon Sep 17 00:00:00 2001 From: MS-GITS <137760120+Greenie0701@users.noreply.github.com> Date: Fri, 26 Sep 2025 09:47:17 +0530 Subject: [PATCH 13/55] ci: add support for building windows on arm wheels (#643) --- .github/workflows/test.yml | 12 ++++++++---- .github/workflows/wheel.yml | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 60894797..26b4ce68 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,9 +9,13 @@ jobs: test: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "macos-latest"] + os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] - + exclude: + - os: windows-11-arm + py: "3.9" + - os: windows-11-arm + py: "3.10" runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -29,8 +33,8 @@ jobs: - name: Prepare shell: bash run: | - pip install -U pip - pip install -r requirements.txt pytest + python -m pip install -U pip + python -m pip install -r requirements.txt pytest - name: Build shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 12600363..fb369a13 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: # macos-13 is for intel - os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "macos-13", "macos-latest"] + os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"] runs-on: ${{ matrix.os }} name: Build wheels on ${{ matrix.os }} @@ -33,7 +33,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-*" + CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' From 0f3c4be465b5f39ab2d6903797e57ef8a0fd097e Mon Sep 17 00:00:00 2001 From: TW Date: Wed, 8 Oct 2025 09:10:46 +0200 Subject: [PATCH 14/55] README: fix typos and grammar (#648) --- README.md | 64 +++++++++++++++++++++++++++---------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 61a03c60..1f063242 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![Build Status](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml/badge.svg)](https://github.com/msgpack/msgpack-python/actions/workflows/wheel.yml) [![Documentation Status](https://readthedocs.org/projects/msgpack-python/badge/?version=latest)](https://msgpack-python.readthedocs.io/en/latest/?badge=latest) -## What's this +## What is this? [MessagePack](https://msgpack.org/) is an efficient binary serialization format. It lets you exchange data among multiple languages like JSON. @@ -25,9 +25,9 @@ But msgpack provides a pure Python implementation (`msgpack.fallback`) for PyPy. ### Windows -When you can't use a binary distribution, you need to install Visual Studio -or Windows SDK on Windows. -Without extension, using pure Python implementation on CPython runs slowly. +If you can't use a binary distribution, you need to install Visual Studio +or the Windows SDK on Windows. +Without the extension, the pure Python implementation on CPython runs slowly. ## How to use @@ -35,11 +35,11 @@ Without extension, using pure Python implementation on CPython runs slowly. ### One-shot pack & unpack Use `packb` for packing and `unpackb` for unpacking. -msgpack provides `dumps` and `loads` as an alias for compatibility with +msgpack provides `dumps` and `loads` as aliases for compatibility with `json` and `pickle`. -`pack` and `dump` packs to a file-like object. -`unpack` and `load` unpacks from a file-like object. +`pack` and `dump` pack to a file-like object. +`unpack` and `load` unpack from a file-like object. ```pycon >>> import msgpack @@ -73,7 +73,7 @@ for unpacked in unpacker: ``` -### Packing/unpacking of custom data type +### Packing/unpacking of custom data types It is also possible to pack/unpack custom data types. Here is an example for `datetime.datetime`. @@ -140,8 +140,8 @@ True ### Advanced unpacking control As an alternative to iteration, `Unpacker` objects provide `unpack`, -`skip`, `read_array_header` and `read_map_header` methods. The former two -read an entire message from the stream, respectively de-serialising and returning +`skip`, `read_array_header`, and `read_map_header` methods. The former two +read an entire message from the stream, respectively deserializing and returning the result, or ignoring it. The latter two methods return the number of elements in the upcoming container, so that each element in an array, or key-value pair in a map, can be unpacked or skipped individually. @@ -149,7 +149,7 @@ in a map, can be unpacked or skipped individually. ## Notes -### string and binary type in old msgpack spec +### String and binary types in the old MessagePack spec Early versions of msgpack didn't distinguish string and binary types. The type for representing both string and binary types was named **raw**. @@ -167,7 +167,7 @@ and `raw=True` options. ### ext type -To use the **ext** type, pass `msgpack.ExtType` object to packer. +To use the **ext** type, pass a `msgpack.ExtType` object to the packer. ```pycon >>> import msgpack @@ -181,26 +181,26 @@ You can use it with `default` and `ext_hook`. See below. ### Security -To unpacking data received from unreliable source, msgpack provides +When unpacking data received from an unreliable source, msgpack provides two security options. `max_buffer_size` (default: `100*1024*1024`) limits the internal buffer size. -It is used to limit the preallocated list size too. +It is also used to limit preallocated list sizes. `strict_map_key` (default: `True`) limits the type of map keys to bytes and str. -While msgpack spec doesn't limit the types of the map keys, -there is a risk of the hashdos. +While the MessagePack spec doesn't limit map key types, +there is a risk of a hash DoS. If you need to support other types for map keys, use `strict_map_key=False`. ### Performance tips -CPython's GC starts when growing allocated object. -This means unpacking may cause useless GC. -You can use `gc.disable()` when unpacking large message. +CPython's GC starts when the number of allocated objects grows. +This means unpacking may trigger unnecessary GC. +You can use `gc.disable()` when unpacking a large message. -List is the default sequence type of Python. -But tuple is lighter than list. +A list is the default sequence type in Python. +However, a tuple is lighter than a list. You can use `use_list=False` while unpacking when performance is important. @@ -208,7 +208,7 @@ You can use `use_list=False` while unpacking when performance is important. ### msgpack 0.5 -Package name on PyPI was changed from `msgpack-python` to `msgpack` from 0.5. +The package name on PyPI was changed from `msgpack-python` to `msgpack` in 0.5. When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` before `pip install -U msgpack`. @@ -218,25 +218,25 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be * Python 2 support - * The extension module does not support Python 2 anymore. + * The extension module no longer supports Python 2. The pure Python implementation (`msgpack.fallback`) is used for Python 2. * msgpack 1.0.6 drops official support of Python 2.7, as pip and - GitHub Action (setup-python) no longer support Python 2.7. + GitHub Action "setup-python" no longer supports Python 2.7. * Packer * Packer uses `use_bin_type=True` by default. - Bytes are encoded in bin type in msgpack. - * The `encoding` option is removed. UTF-8 is used always. + Bytes are encoded in the bin type in MessagePack. + * The `encoding` option is removed. UTF-8 is always used. * Unpacker - * Unpacker uses `raw=False` by default. It assumes str types are valid UTF-8 string - and decode them to Python str (unicode) object. + * Unpacker uses `raw=False` by default. It assumes str values are valid UTF-8 strings + and decodes them to Python str (Unicode) objects. * `encoding` option is removed. You can use `raw=True` to support old format (e.g. unpack into bytes, not str). - * Default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attack. + * The default value of `max_buffer_size` is changed from 0 to 100 MiB to avoid DoS attacks. You need to pass `max_buffer_size=0` if you have large but safe data. - * Default value of `strict_map_key` is changed to True to avoid hashdos. - You need to pass `strict_map_key=False` if you have data which contain map keys - which type is not bytes or str. + * The default value of `strict_map_key` is changed to True to avoid hash DoS. + You need to pass `strict_map_key=False` if you have data that contain map keys + whose type is neither bytes nor str. From 19b5d33ded09d5a563a677c1a3d75b6246181259 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 8 Oct 2025 17:56:20 +0900 Subject: [PATCH 15/55] release v1.1.2 (#649) --- .github/workflows/wheel.yml | 2 +- ChangeLog.rst | 13 +++++++++++++ msgpack/__init__.py | 4 ++-- pyproject.toml | 18 ++++-------------- requirements.txt | 2 +- test/test_buffer.py | 4 ++-- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fb369a13..fca321af 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -29,7 +29,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v3.1.1 + uses: pypa/cibuildwheel@v3.2.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/ChangeLog.rst b/ChangeLog.rst index 418c444c..beeab15c 100644 --- a/ChangeLog.rst +++ b/ChangeLog.rst @@ -1,3 +1,16 @@ +1.1.2 +===== + +Release Date: 2025-10-08 + +This release does not change source code. It updates only building wheels: + +* Update Cython to v3.1.4 +* Update cibuildwheel to v3.2.0 +* Drop Python 3.8 +* Add Python 3.14 +* Add windows-arm + 1.1.1 ===== diff --git a/msgpack/__init__.py b/msgpack/__init__.py index ad68271d..f3266b70 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 1) -__version__ = "1.1.1" +version = (1, 1, 2) +__version__ = "1.1.2" if os.environ.get("MSGPACK_PUREPYTHON"): diff --git a/pyproject.toml b/pyproject.toml index b1628322..3976adc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,31 +1,21 @@ [build-system] -# 75.3.0 is the latest version supporting Python 3.8 -requires = ["setuptools >= 75.3.0"] +requires = ["setuptools >= 80.9.0"] build-backend = "setuptools.build_meta" [project] name = "msgpack" dynamic = ["version"] -# `license = "Apache-2.0"` is preferred. But keep old syntax for Python 3.8 compatibility. -# https://github.com/msgpack/msgpack-python/pull/637 -license = {text="Apache 2.0"} +license = "Apache-2.0" authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] -requires-python = ">=3.8" +requires-python = ">=3.9" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", "Topic :: File Formats", "Intended Audience :: Developers", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Programming Language :: Python :: 3.13", - "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] @@ -46,7 +36,7 @@ version = {attr = "msgpack.__version__"} [tool.ruff] line-length = 100 -target-version = "py38" +target-version = "py39" lint.select = [ "E", # pycodestyle "F", # Pyflakes diff --git a/requirements.txt b/requirements.txt index 5d2e20b1..b544ccac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1 @@ -Cython~=3.1.2 +Cython==3.1.4 diff --git a/test/test_buffer.py b/test/test_buffer.py index 2c5a14c5..ca097222 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_bytearray(): obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_unpack_memoryview(): @@ -26,7 +26,7 @@ def test_unpack_memoryview(): obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_packer_getbuffer(): From ef4f83df16e552e8c8fb12ceac55f5fda9f7f340 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 13:00:46 +0900 Subject: [PATCH 16/55] relax setuptools version (#652) --- .github/workflows/test.yml | 4 +--- MANIFEST.in | 2 +- pyproject.toml | 2 +- requirements.txt | 2 ++ 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 26b4ce68..74c311cb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,6 @@ jobs: - name: Prepare shell: bash run: | - python -m pip install -U pip python -m pip install -r requirements.txt pytest - name: Build @@ -55,8 +54,7 @@ jobs: - name: build packages shell: bash run: | - pip install build - python -m build + python -m build -nv - name: upload packages uses: actions/upload-artifact@v4 diff --git a/MANIFEST.in b/MANIFEST.in index 57d84a4c..6317706e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include setup.py include COPYING include README.md -recursive-include msgpack *.h *.c *.pyx *.cpp +recursive-include msgpack *.h *.c *.pyx recursive-include test *.py diff --git a/pyproject.toml b/pyproject.toml index 3976adc7..3633d6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 80.9.0"] +requires = ["setuptools >= 77.0.3"] build-backend = "setuptools.build_meta" [project] diff --git a/requirements.txt b/requirements.txt index b544ccac..f8f83709 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,3 @@ Cython==3.1.4 +setuptools==77.0.3 +build From c2546eabc414b6f232d41b7d9e51b9ff9fa93296 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 15:28:01 +0900 Subject: [PATCH 17/55] update setuptools requirements to >=78.1.1 (#653) https://github.com/advisories/GHSA-5rjg-fvgr-3xxf --- pyproject.toml | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3633d6b6..a4c2172a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools >= 77.0.3"] +requires = ["setuptools >= 78.1.1"] build-backend = "setuptools.build_meta" [project] diff --git a/requirements.txt b/requirements.txt index f8f83709..2f1c55b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ Cython==3.1.4 -setuptools==77.0.3 +setuptools==78.1.1 build From af456409709f58add777090bdbe6a4696879e49d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 15:53:08 +0900 Subject: [PATCH 18/55] cython: freethreading_compatible (#654) ``` $ v3/bin/python -VV Python 3.14.0 free-threading build (main, Oct 7 2025, 15:35:12) [Clang 20.1.4 ] $ v3/bin/python -c 'import sys,msgpack; print(sys._is_gil_enabled())' False ``` --- .github/workflows/test.yml | 2 +- msgpack/_cmsgpack.pyx | 3 ++- msgpack/_packer.pyx | 10 ++++++++-- msgpack/_unpacker.pyx | 11 +++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 74c311cb..20410fe9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9"] + py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"] exclude: - os: windows-11-arm py: "3.9" diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 1faaac3a..9680b31e 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,5 +1,6 @@ -# coding: utf-8 #cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +#cython: freethreading_compatible = True +import cython from cpython.datetime cimport import_datetime, datetime_new import_datetime() diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 402b6946..94d1462c 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from cpython.datetime cimport ( @@ -129,6 +127,7 @@ cdef class Packer: if self.exports > 0: raise BufferError("Existing exports of data: Packer cannot be changed") + @cython.critical_section def __init__(self, *, default=None, bint use_single_float=False, bint autoreset=True, bint use_bin_type=True, bint strict_types=False, bint datetime=False, unicode_errors=None, @@ -269,6 +268,7 @@ cdef class Packer: return ret return self._pack_inner(o, 0, nest_limit) + @cython.critical_section def pack(self, object obj): cdef int ret self._check_exports() @@ -284,6 +284,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_ext_type(self, typecode, data): self._check_exports() if len(data) > ITEM_LIMIT: @@ -291,6 +292,7 @@ cdef class Packer: msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) + @cython.critical_section def pack_array_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -301,6 +303,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_header(self, long long size): self._check_exports() if size > ITEM_LIMIT: @@ -311,6 +314,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def pack_map_pairs(self, object pairs): """ Pack *pairs* as msgpack map type. @@ -331,6 +335,7 @@ cdef class Packer: self.pk.length = 0 return buf + @cython.critical_section def reset(self): """Reset internal buffer. @@ -339,6 +344,7 @@ cdef class Packer: self._check_exports() self.pk.length = 0 + @cython.critical_section def bytes(self): """Return internal buffer contents as bytes object""" return PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 34ff3304..f0cf96d7 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,5 +1,3 @@ -# coding: utf-8 - from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject @@ -324,6 +322,7 @@ cdef class Unpacker: PyMem_Free(self.buf) self.buf = NULL + @cython.critical_section def __init__(self, file_like=None, *, Py_ssize_t read_size=0, bint use_list=True, bint raw=False, int timestamp=0, bint strict_map_key=True, object object_hook=None, object object_pairs_hook=None, object list_hook=None, @@ -384,6 +383,7 @@ cdef class Unpacker: max_str_len, max_bin_len, max_array_len, max_map_len, max_ext_len) + @cython.critical_section def feed(self, object next_bytes): """Append `next_bytes` to internal buffer.""" cdef Py_buffer pybuff @@ -484,6 +484,7 @@ cdef class Unpacker: else: raise ValueError("Unpack failed: error = %d" % (ret,)) + @cython.critical_section def read_bytes(self, Py_ssize_t nbytes): """Read a specified number of raw bytes from the stream""" cdef Py_ssize_t nread @@ -496,6 +497,7 @@ cdef class Unpacker: self.stream_offset += nread return ret + @cython.critical_section def unpack(self): """Unpack one object @@ -503,6 +505,7 @@ cdef class Unpacker: """ return self._unpack(unpack_construct) + @cython.critical_section def skip(self): """Read and ignore one object, returning None @@ -510,6 +513,7 @@ cdef class Unpacker: """ return self._unpack(unpack_skip) + @cython.critical_section def read_array_header(self): """assuming the next object is an array, return its size n, such that the next n unpack() calls will iterate over its contents. @@ -518,6 +522,7 @@ cdef class Unpacker: """ return self._unpack(read_array_header) + @cython.critical_section def read_map_header(self): """assuming the next object is a map, return its size n, such that the next n * 2 unpack() calls will iterate over its key-value pairs. @@ -526,6 +531,7 @@ cdef class Unpacker: """ return self._unpack(read_map_header) + @cython.critical_section def tell(self): """Returns the current position of the Unpacker in bytes, i.e., the number of bytes that were read from the input, also the starting @@ -536,6 +542,7 @@ cdef class Unpacker: def __iter__(self): return self + @cython.critical_section def __next__(self): return self._unpack(unpack_construct, 1) From c1ecd23dbfaa6c6f2a408e7e5c4727c6bdce7608 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 9 Oct 2025 18:33:23 +0900 Subject: [PATCH 19/55] drop Python 3.9 (#656) --- .github/workflows/test.yml | 8 +++----- .github/workflows/wheel.yml | 6 +++--- pyproject.toml | 4 ++-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 20410fe9..6b1664ae 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,10 +10,8 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10", "3.9"] + py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - - os: windows-11-arm - py: "3.9" - os: windows-11-arm py: "3.10" runs-on: ${{ matrix.os }} @@ -21,10 +19,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: ${{ matrix.py }} allow-prereleases: true diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fca321af..be54e5e1 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -17,8 +17,8 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-python@v6 with: python-version: "3.x" cache: "pip" @@ -33,7 +33,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-* cp39-win_arm64 cp310-win_arm64" + CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/pyproject.toml b/pyproject.toml index a4c2172a..c69d5a7c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [{name="Inada Naoki", email="songofacandy@gmail.com"}] description = "MessagePack serializer" readme = "README.md" keywords = ["msgpack", "messagepack", "serializer", "serialization", "binary"] -requires-python = ">=3.9" +requires-python = ">=3.10" classifiers = [ "Development Status :: 5 - Production/Stable", "Operating System :: OS Independent", @@ -36,7 +36,7 @@ version = {attr = "msgpack.__version__"} [tool.ruff] line-length = 100 -target-version = "py39" +target-version = "py310" lint.select = [ "E", # pycodestyle "F", # Pyflakes From f9806368ae302722f095ea884436b20ed1cddf33 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 1 Dec 2025 14:16:03 +0900 Subject: [PATCH 20/55] update cython and cibuildwheel (#658) --- .github/workflows/wheel.yml | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index be54e5e1..531abbc3 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -17,7 +17,7 @@ jobs: name: Build wheels on ${{ matrix.os }} steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v6 - uses: actions/setup-python@v6 with: python-version: "3.x" @@ -29,7 +29,7 @@ jobs: make cython - name: Build - uses: pypa/cibuildwheel@v3.2.0 + uses: pypa/cibuildwheel@v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" diff --git a/requirements.txt b/requirements.txt index 2f1c55b2..9e4643b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -Cython==3.1.4 +Cython==3.2.1 setuptools==78.1.1 build From 6357bc272c83fb450fbe7de7e7ca15d815e75174 Mon Sep 17 00:00:00 2001 From: Julien Stephan Date: Sat, 21 Feb 2026 09:09:34 +0100 Subject: [PATCH 21/55] ci: add riscv64 manylinux/musllinux wheels (#664) Now that cibuildwheel and PyPI support riscv64, we can start building riscv64 wheels. Because there is no native riscv64 runner available, this PR adds a QEMU-based riscv64 job to the cibuildwheel workflow. Signed-off-by: Julien Stephan --- .github/workflows/wheel.yml | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 531abbc3..f3fbe29e 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -12,9 +12,19 @@ jobs: strategy: matrix: # macos-13 is for intel - os: ["ubuntu-24.04", "ubuntu-24.04-arm", "windows-latest", "windows-11-arm", "macos-13", "macos-latest"] + include: + - os: ubuntu-24.04 + - os: ubuntu-24.04-arm + - os: windows-latest + - os: windows-11-arm + - os: macos-13 + - os: macos-latest + - os: ubuntu-24.04 + cibw_archs: riscv64 + name_suffix: "-riscv64" + runs-on: ${{ matrix.os }} - name: Build wheels on ${{ matrix.os }} + name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - uses: actions/checkout@v6 @@ -28,12 +38,19 @@ jobs: pip install -r requirements.txt make cython + - name: Set up QEMU for emulation + if: matrix.cibw_archs == 'riscv64' + uses: docker/setup-qemu-action@v3 + with: + platforms: ${{ matrix.cibw_archs }} + - name: Build uses: pypa/cibuildwheel@v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" + CIBW_ARCHS: ${{ matrix.cibw_archs || 'auto' }} - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' @@ -44,7 +61,7 @@ jobs: - name: Upload Wheels to artifact uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.os }} + name: wheels-${{ matrix.os }}${{ matrix.name_suffix || '' }} path: wheelhouse # combine all wheels into one artifact From 156bb05a15b9e58acf88b138185d5cb2068dad88 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 21 Apr 2026 09:22:06 +0200 Subject: [PATCH 22/55] fix: check `unpack_callback_uint32` result (#666) Similar to #665, just a return value check to propagate the error in case one happens (instead of silently ignoring it). Note that as opposed to the previous lines, we don't need to `PyErr_SetString` since `unpack_callback_uint32` calls `PyLong_FromSize_t` which itself should set whatever Python error is relevant; we just need to make it clear to the caller that an error occurred. --- msgpack/unpack_container_header.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack_container_header.h b/msgpack/unpack_container_header.h index c14a3c2b..e48d43ba 100644 --- a/msgpack/unpack_container_header.h +++ b/msgpack/unpack_container_header.h @@ -45,7 +45,10 @@ static inline int unpack_container_header(unpack_context* ctx, const char* data, PyErr_SetString(PyExc_ValueError, "Unexpected type header on stream"); return -1; } - unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj); + + if (unpack_callback_uint32(&ctx->user, size, &ctx->stack[0].obj) < 0) + return -1; + return 1; } From 5d465bdb772d977d0b825a63e8061bd2fab5588b Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 21 Apr 2026 11:06:10 +0200 Subject: [PATCH 23/55] fix: re-raise existing exception when available (#667) --- msgpack/_unpacker.pyx | 7 ++++++- test/test_except.py | 30 +++++++++++++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index f0cf96d7..e25986ee 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -205,7 +205,10 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, raise FormatError elif ret == -3: raise StackError - raise ValueError("Unpack failed: error = %d" % (ret,)) + elif PyErr_Occurred(): + raise + else: + raise ValueError("Unpack failed: error = %d" % (ret,)) cdef class Unpacker: @@ -481,6 +484,8 @@ cdef class Unpacker: raise FormatError elif ret == -3: raise StackError + elif PyErr_Occurred(): + raise else: raise ValueError("Unpack failed: error = %d" % (ret,)) diff --git a/test/test_except.py b/test/test_except.py index b77ac800..c56a6a30 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -4,7 +4,7 @@ from pytest import raises -from msgpack import FormatError, OutOfData, StackError, Unpacker, packb, unpackb +from msgpack import ExtType, FormatError, OutOfData, StackError, Unpacker, packb, unpackb class DummyException(Exception): @@ -32,6 +32,34 @@ def hook(obj): ) +def test_raise_from_list_hook(): + def hook(lst: list) -> list: + raise DummyException + + with raises(DummyException): + unpackb(packb([1, 2, 3]), list_hook=hook) + + with raises(DummyException): + unpacker = Unpacker(list_hook=hook) + unpacker.feed(packb([1, 2, 3])) + unpacker.unpack() + + +def test_raise_from_ext_hook(): + def hook(code: int, data: bytes) -> ExtType: + raise DummyException + + packed = packb(ExtType(42, b"hello")) + + with raises(DummyException): + unpackb(packed, ext_hook=hook) + + with raises(DummyException): + unpacker = Unpacker(ext_hook=hook) + unpacker.feed(packed) + unpacker.unpack() + + def test_invalidvalue(): incomplete = b"\xd9\x97#DL_" # raw8 - length=0x97 with raises(ValueError): From 95c8be516db9cb5d8c1e10aa973091dcf6c82ff8 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 21 Apr 2026 11:27:11 +0200 Subject: [PATCH 24/55] fix: check return code in `unpack_callback_int64` (#665) --- msgpack/unpack.h | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 58a2f4f5..0f9ffc05 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -70,12 +70,7 @@ static inline int unpack_callback_uint32(unpack_user* u, uint32_t d, msgpack_unp static inline int unpack_callback_uint64(unpack_user* u, uint64_t d, msgpack_unpack_object* o) { - PyObject *p; - if (d > LONG_MAX) { - p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); - } else { - p = PyLong_FromLong((long)d); - } + PyObject *p = PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)d); if (!p) return -1; *o = p; @@ -103,12 +98,9 @@ static inline int unpack_callback_int8(unpack_user* u, int8_t d, msgpack_unpack_ static inline int unpack_callback_int64(unpack_user* u, int64_t d, msgpack_unpack_object* o) { - PyObject *p; - if (d > LONG_MAX || d < LONG_MIN) { - p = PyLong_FromLongLong((PY_LONG_LONG)d); - } else { - p = PyLong_FromLong((long)d); - } + PyObject *p = PyLong_FromLongLong((PY_LONG_LONG)d); + if (!p) + return -1; *o = p; return 0; } From bcf89c7efd4083e013ac305030b96252251c8aa5 Mon Sep 17 00:00:00 2001 From: Valentin Valls Date: Sun, 26 Apr 2026 18:39:18 +0200 Subject: [PATCH 25/55] Fixed README example (#668) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1f063242..223742dd 100644 --- a/README.md +++ b/README.md @@ -119,15 +119,15 @@ It is also possible to pack/unpack custom data types using the **ext** type. >>> import array >>> def default(obj): ... if isinstance(obj, array.array) and obj.typecode == 'd': -... return msgpack.ExtType(42, obj.tostring()) +... return msgpack.ExtType(42, obj.tobytes()) ... raise TypeError("Unknown type: %r" % (obj,)) ... >>> def ext_hook(code, data): ... if code == 42: ... a = array.array('d') -... a.fromstring(data) +... a.frombytes(data) ... return a -... return ExtType(code, data) +... return msgpack.ExtType(code, data) ... >>> data = array.array('d', [1.2, 3.4]) >>> packed = msgpack.packb(data, default=default) From a54fad13c10d3a555d6bd71b06dccf980d516a18 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 15:06:20 +0900 Subject: [PATCH 26/55] pinact --- .github/workflows/docs.yaml | 4 ++-- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 6 +++--- .github/workflows/wheel.yml | 18 +++++++++--------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b696b926..917e3027 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Setup Python - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: '3.x' cache: "pip" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 198cf7b5..3ea2a87f 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6b1664ae..9d447843 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,10 +19,10 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v5 + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: ${{ matrix.py }} allow-prereleases: true @@ -55,7 +55,7 @@ jobs: python -m build -nv - name: upload packages - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: dist-${{ matrix.os }}-${{ matrix.py }} path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index f3fbe29e..e9304670 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -27,8 +27,8 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@v6 - - uses: actions/setup-python@v6 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" cache: "pip" @@ -40,12 +40,12 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 with: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@v3.3.0 + uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -59,7 +59,7 @@ jobs: python -m build -s -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: wheels-${{ matrix.os }}${{ matrix.name_suffix || '' }} path: wheelhouse @@ -69,7 +69,7 @@ jobs: needs: [build_wheels] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* @@ -77,7 +77,7 @@ jobs: merge-multiple: true - name: Upload Wheels to artifact - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: wheels-all path: dist @@ -93,13 +93,13 @@ jobs: # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* path: dist merge-multiple: true - - uses: pypa/gh-action-pypi-publish@release/v1 + - uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0 #with: # To test: repository-url: https://test.pypi.org/legacy/ From 94242eb0e9c2292b51b76400cb6523eee6d8d576 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 15:07:06 +0900 Subject: [PATCH 27/55] dependabot --- .github/dependabot.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..c2e9b9c9 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,15 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + all-dependencies: + patterns: + - "*" From b7ebb87b9d48969959c79644e225d90d08b3e47c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Apr 2026 15:16:05 +0900 Subject: [PATCH 28/55] action: Bump the all-dependencies group with 6 updates (#669) --- .github/workflows/docs.yaml | 4 ++-- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 4 ++-- .github/workflows/wheel.yml | 12 ++++++------ 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 917e3027..4467ca8b 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,10 +11,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.x' cache: "pip" diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 3ea2a87f..93b64947 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9d447843..729a45f1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -55,7 +55,7 @@ jobs: python -m build -nv - name: upload packages - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: dist-${{ matrix.os }}-${{ matrix.py }} path: dist diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index e9304670..eb457277 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -40,12 +40,12 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 + uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 with: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@63fd63b352a9a8bdcc24791c9dbee952ee9a8abc # v3.3.0 + uses: pypa/cibuildwheel@8d2b08b68458a16aeb24b64e68a09ab1c8e82084 # v3.4.1 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" @@ -59,7 +59,7 @@ jobs: python -m build -s -o wheelhouse - name: Upload Wheels to artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-${{ matrix.os }}${{ matrix.name_suffix || '' }} path: wheelhouse @@ -69,7 +69,7 @@ jobs: needs: [build_wheels] runs-on: ubuntu-latest steps: - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* @@ -77,7 +77,7 @@ jobs: merge-multiple: true - name: Upload Wheels to artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: wheels-all path: dist @@ -93,7 +93,7 @@ jobs: # or, alternatively, upload to PyPI on every tag starting with 'v' (remove on: release above to use this) # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') steps: - - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: # unpacks all CIBW artifacts into dist/ pattern: wheels-* From 0d600a33285c8ef968a4926840ccc6a096ae3e09 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Mon, 27 Apr 2026 17:02:52 +0900 Subject: [PATCH 29/55] ci: use ubuntu-slim for lint (#670) --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 93b64947..e2631ffa 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -8,7 +8,7 @@ jobs: # by the push to the branch. if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository - runs-on: ubuntu-latest + runs-on: ubuntu-slim steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 From cd8137093ac0abcfa9acc9d9c2ad6abeeee8ad30 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 12 May 2026 06:59:40 +0200 Subject: [PATCH 30/55] fix: enforce `strict_map_key` with `object_pairs_hook` (#673) This PR makes it so that `self._strict_map_key` is enforced even when using `_object_pairs_hook` which didn't use to be the case. --- msgpack/fallback.py | 12 +++++++++--- test/test_except.py | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/msgpack/fallback.py b/msgpack/fallback.py index b02e47cf..d71bbfc7 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -518,9 +518,15 @@ def _unpack(self, execute=EX_CONSTRUCT): self._unpack(EX_SKIP) return if self._object_pairs_hook is not None: - ret = self._object_pairs_hook( - (self._unpack(EX_CONSTRUCT), self._unpack(EX_CONSTRUCT)) for _ in range(n) - ) + + def _gen(): + for _ in range(n): + key = self._unpack(EX_CONSTRUCT) + if self._strict_map_key and type(key) not in (str, bytes): + raise ValueError("%s is not allowed for map key" % str(type(key))) + yield key, self._unpack(EX_CONSTRUCT) + + ret = self._object_pairs_hook(_gen()) else: ret = {} for _ in range(n): diff --git a/test/test_except.py b/test/test_except.py index c56a6a30..be8ca4cf 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -89,3 +89,17 @@ def test_strict_map_key(): packed = packb(invalid, use_bin_type=True) with raises(ValueError): unpackb(packed, raw=False, strict_map_key=True) + + +def test_strict_map_key_with_object_pairs_hook(): + # strict_map_key should be enforced even when object_pairs_hook is set + invalid = {42: "value"} + packed = packb(invalid, use_bin_type=True) + with raises(ValueError): + unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) + + # valid keys (str/bytes) should still work with object_pairs_hook + valid = {"key": "value"} + packed = packb(valid, use_bin_type=True) + result = unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) + assert result == [("key", "value")] From 4b2749cdff9326ec8bf2b7006530ac43d263b992 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 22 May 2026 16:08:23 +0900 Subject: [PATCH 31/55] Raise DEFAULT_RECURSE_LIMIT from 511 to 1024 (#676) --- msgpack/_packer.pyx | 2 +- msgpack/fallback.py | 2 +- test/test_limits.py | 24 ++++++++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 94d1462c..4d0d2d21 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -38,7 +38,7 @@ cdef extern from "pack.h": int msgpack_pack_timestamp(msgpack_packer* x, long long seconds, unsigned long nanoseconds) except -1 -cdef int DEFAULT_RECURSE_LIMIT=511 +cdef int DEFAULT_RECURSE_LIMIT=1024 cdef long long ITEM_LIMIT = (2**32)-1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index d71bbfc7..1f2daf7b 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -52,7 +52,7 @@ def newlist_hint(size): TYPE_BIN = 4 TYPE_EXT = 5 -DEFAULT_RECURSE_LIMIT = 511 +DEFAULT_RECURSE_LIMIT = 1024 def _check_type_strict(obj, t, type=type, tuple=tuple): diff --git a/test/test_limits.py b/test/test_limits.py index 9b92b4d9..468c1c36 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -153,6 +153,30 @@ def test_auto_max_array_len(): unpacker.unpack() +def test_nest_limit_1024(): + import sys + + # Build a list nested 1024 levels deep + d = None + for _ in range(1024): + d = [d] + + # Temporarily raise Python's recursion limit so packing 1024 levels succeeds + old_limit = sys.getrecursionlimit() + sys.setrecursionlimit(max(old_limit, 10000)) + try: + packed = packb(d) + result = unpackb(packed) + finally: + sys.setrecursionlimit(old_limit) + + # Verify structure iteratively to avoid hitting C-level recursion limit in == + for _ in range(1024): + assert isinstance(result, list) and len(result) == 1 + result = result[0] + assert result is None + + def test_auto_max_map_len(): # len(packed) == 6 -> max_map_len == 3 packed = b"\xde\x00\x04zzz" From 378edc60f15c425231d1db99b1886b93c0dcf784 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Sat, 23 May 2026 18:48:01 +0200 Subject: [PATCH 32/55] fix: properly handle return codes in `pack_timestamp` (#672) --- msgpack/pack_template.h | 32 ++++++++++++++++++++------------ 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/msgpack/pack_template.h b/msgpack/pack_template.h index b8959f02..e02ec324 100644 --- a/msgpack/pack_template.h +++ b/msgpack/pack_template.h @@ -551,6 +551,7 @@ static inline int msgpack_pack_ext(msgpack_packer* x, char typecode, size_t l) */ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uint32_t nanoseconds) { + int ret; if ((seconds >> 34) == 0) { /* seconds is unsigned and fits in 34 bits */ uint64_t data64 = ((uint64_t)nanoseconds << 34) | (uint64_t)seconds; @@ -558,26 +559,33 @@ static inline int msgpack_pack_timestamp(msgpack_packer* x, int64_t seconds, uin /* no nanoseconds and seconds is 32bits or smaller. timestamp32. */ unsigned char buf[4]; uint32_t data32 = (uint32_t)data64; - msgpack_pack_ext(x, -1, 4); + ret = msgpack_pack_ext(x, -1, 4); + if (ret != 0) + return ret; + _msgpack_store32(buf, data32); - msgpack_pack_raw_body(x, buf, 4); + return msgpack_pack_raw_body(x, buf, 4); } else { /* timestamp64 */ unsigned char buf[8]; - msgpack_pack_ext(x, -1, 8); - _msgpack_store64(buf, data64); - msgpack_pack_raw_body(x, buf, 8); + ret = msgpack_pack_ext(x, -1, 8); + if (ret != 0) + return ret; + _msgpack_store64(buf, data64); + return msgpack_pack_raw_body(x, buf, 8); } } else { - /* seconds is signed or >34bits */ - unsigned char buf[12]; - _msgpack_store32(&buf[0], nanoseconds); - _msgpack_store64(&buf[4], seconds); - msgpack_pack_ext(x, -1, 12); - msgpack_pack_raw_body(x, buf, 12); + /* seconds is signed or >34bits */ + unsigned char buf[12]; + _msgpack_store32(&buf[0], nanoseconds); + _msgpack_store64(&buf[4], seconds); + ret = msgpack_pack_ext(x, -1, 12); + if (ret != 0) + return ret; + + return msgpack_pack_raw_body(x, buf, 12); } - return 0; } From 4a077456754a060d2bb14578e03bcb1153d952cc Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Wed, 27 May 2026 11:04:24 +0200 Subject: [PATCH 33/55] fix: avoid memory leak when decoding invalid nested arrays (#671) --- msgpack/_unpacker.pyx | 1 + msgpack/unpack_template.h | 8 ++++++++ test/test_except.py | 35 +++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e25986ee..29cdec4a 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -322,6 +322,7 @@ cdef class Unpacker: self.buf = NULL def __dealloc__(self): + unpack_clear(&self.ctx) PyMem_Free(self.buf) self.buf = NULL diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index cce29e7a..42306618 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -72,6 +72,14 @@ static inline PyObject* unpack_data(unpack_context* ctx) static inline void unpack_clear(unpack_context *ctx) { + unsigned int i; + for (i = 1; i < ctx->top; i++) { + Py_CLEAR(ctx->stack[i].obj); + /* map_key holds a live reference only while waiting for the value */ + if (ctx->stack[i].ct == CT_MAP_VALUE) { + Py_CLEAR(ctx->stack[i].map_key); + } + } Py_CLEAR(ctx->stack[0].obj); } diff --git a/test/test_except.py b/test/test_except.py index be8ca4cf..c04b110d 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -1,6 +1,8 @@ #!/usr/bin/env python import datetime +import gc +import tracemalloc from pytest import raises @@ -80,6 +82,39 @@ def test_invalidvalue(): unpackb(b"\x91" * 3000) # nested fixarray(len=1) +def test_no_memory_leak_on_nested_invalid_tag() -> None: + """Regression test: unpacking nested arrays containing an invalid tag must not leak objects.""" + + kwargs: dict = { + "raw": False, + "strict_map_key": False, + "max_array_len": 1 << 20, + "max_map_len": 1 << 20, + } + n = 1000 + + for depth in range(1, 15): + data = bytes([0x91] * depth + [0xC1]) + + gc.collect() + tracemalloc.start() + s1 = tracemalloc.take_snapshot() + + for _ in range(n): + try: + unpackb(data, **kwargs) + except Exception: + pass + + gc.collect() + s2 = tracemalloc.take_snapshot() + tracemalloc.stop() + + leaked = sum(s.count_diff for s in s2.compare_to(s1, "lineno") if s.count_diff > 0) + per_call = leaked / n + assert per_call < 1.0, f"depth={depth}: {per_call:.2f} leaked objects/call (expected < 1)" + + def test_strict_map_key(): valid = {"unicode": 1, b"bytes": 2} packed = packb(valid, use_bin_type=True) From 284782d647815ed46b07c6bd2d5c620a8a46869e Mon Sep 17 00:00:00 2001 From: Kadir Can Ozden <101993364+bysiber@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:27:05 +0300 Subject: [PATCH 34/55] Add missing autoreset in Packer.pack_ext_type (#663) --- msgpack/_packer.pyx | 4 ++++ msgpack/fallback.py | 4 ++++ pyproject.toml | 7 +++++++ test/test_extension.py | 11 ++++++++++- 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 4d0d2d21..2d92e8a4 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -291,6 +291,10 @@ cdef class Packer: raise ValueError("ext data too large") msgpack_pack_ext(&self.pk, typecode, len(data)) msgpack_pack_raw_body(&self.pk, data, len(data)) + if self.autoreset: + buf = PyBytes_FromStringAndSize(self.pk.buf, self.pk.length) + self.pk.length = 0 + return buf @cython.critical_section def pack_array_header(self, long long size): diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 1f2daf7b..61d77db0 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -867,6 +867,10 @@ def pack_ext_type(self, typecode, data): self._buffer.write(b"\xc9" + struct.pack(">I", L)) self._buffer.write(struct.pack("B", typecode)) self._buffer.write(data) + if self._autoreset: + ret = self._buffer.getvalue() + self._buffer = BytesIO() + return ret def _pack_array_header(self, n): if n <= 0x0F: diff --git a/pyproject.toml b/pyproject.toml index c69d5a7c..4e4510b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,6 +19,7 @@ classifiers = [ "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] +dependencies = [] [project.urls] Homepage = "https://msgpack.org/" @@ -43,3 +44,9 @@ lint.select = [ "I", # isort #"UP", pyupgrade ] + +[dependency-groups] +dev = [ + "cython>=3.2.5", + "pytest>=9.0.3", +] diff --git a/test/test_extension.py b/test/test_extension.py index aaf0fd92..61852f15 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -6,7 +6,7 @@ def test_pack_ext_type(): def p(s): - packer = msgpack.Packer() + packer = msgpack.Packer(autoreset=False) packer.pack_ext_type(0x42, s) return packer.bytes() @@ -20,6 +20,15 @@ def p(s): assert p(b"A" * 0x00012345) == b"\xc9\x00\x01\x23\x45\x42" + b"A" * 0x00012345 # ext 32 +def test_pack_ext_type_autoreset(): + packer = msgpack.Packer() + + assert packer.pack_ext_type(0x42, b"A") == b"\xd4\x42A" + assert packer.bytes() == b"" + assert packer.pack_ext_type(0x42, b"ABC") == b"\xc7\x03\x42ABC" + assert packer.bytes() == b"" + + def test_unpack_ext_type(): def check(b, expected): assert msgpack.unpackb(b) == expected From 9de2fd9b24b2026ce5921fc45417893d98f534bf Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 03:21:05 -0400 Subject: [PATCH 35/55] Add no-GIL interpreter support (#641) Co-authored-by: Lysandros Nikolaou Co-authored-by: Frank Dana Co-authored-by: Inada Naoki --- .github/workflows/test.yml | 20 +++++++++++ test/uneeded_test_multithreading.py | 51 +++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 test/uneeded_test_multithreading.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 729a45f1..a8873d51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,6 +14,7 @@ jobs: exclude: - os: windows-11-arm py: "3.10" + runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -33,6 +34,11 @@ jobs: run: | python -m pip install -r requirements.txt pytest + - name: Install pytest-run-parallel under free-threading + if: contains(matrix.py, 't') + run: | + pip install pytest-run-parallel + - name: Build shell: bash run: | @@ -40,15 +46,29 @@ jobs: pip install . - name: Test (C extension) + if: ${{ ! contains(matrix.py, 't') }} shell: bash run: | pytest -v test - name: Test (pure Python fallback) + if: ${{ ! contains(matrix.py, 't') }} shell: bash run: | MSGPACK_PUREPYTHON=1 pytest -v test + - name: Test (C extension) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + pytest -v --parallel-threads=auto --iterations=20 test + + - name: Test (pure Python fallback) in parallel under free-threading + if: contains(matrix.py, 't') + shell: bash + run: | + MSGPACK_PUREPYTHON=1 pytest -v --parallel-threads=auto --iterations=20 test + - name: build packages shell: bash run: | diff --git a/test/uneeded_test_multithreading.py b/test/uneeded_test_multithreading.py new file mode 100644 index 00000000..6694fdc6 --- /dev/null +++ b/test/uneeded_test_multithreading.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +import threading +from concurrent.futures import ThreadPoolExecutor + +from msgpack import Packer + + +def run_threaded( + func, + num_threads=8, + pass_count=False, + pass_barrier=False, + outer_iterations=1, + prepare_args=None, +): + """Runs a function many times in parallel""" + for _ in range(outer_iterations): + with ThreadPoolExecutor(max_workers=num_threads) as tpe: + if prepare_args is None: + args = [] + else: + args = prepare_args() + if pass_barrier: + barrier = threading.Barrier(num_threads) + args.append(barrier) + if pass_count: + all_args = [(func, i, *args) for i in range(num_threads)] + else: + all_args = [(func, *args) for i in range(num_threads)] + try: + futures = [] + for arg in all_args: + futures.append(tpe.submit(*arg)) + finally: + if len(futures) < num_threads and pass_barrier: + barrier.abort() + for f in futures: + f.result() + + +def test_multithread_packing(): + output = [] + test_data = "abcd" * 10_000_000 + packer = Packer() + + def closure(b): + data = packer.pack(test_data) + output.append(data) + b.wait() + + run_threaded(closure, num_threads=10, pass_barrier=True, pass_count=False) From b83a0aa62b7288f50ae0181f03f573cb0ef83919 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Tue, 2 Jun 2026 20:56:05 +0900 Subject: [PATCH 36/55] skip recursion limit test on free-threaded CPython builds (#679) For some reason, it seems that the recursion limit cannot be avoided in free-threaded CPython on GitHub Actions, so I will skip the test. --- test/test_limits.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_limits.py b/test/test_limits.py index 468c1c36..e3bab146 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +import sysconfig + import pytest from msgpack import ( @@ -153,6 +155,11 @@ def test_auto_max_array_len(): unpacker.unpack() +# Skip on free-threaded CPython builds because this test depends on recursion behavior. +IS_FREE_THREADED_BUILD = bool(sysconfig.get_config_var("Py_GIL_DISABLED")) + + +@pytest.mark.skipif(IS_FREE_THREADED_BUILD, reason="Skipped on free-threaded build") def test_nest_limit_1024(): import sys From 98d2c7907ddf5a6a810aaece9a7394e20bc490bf Mon Sep 17 00:00:00 2001 From: Kadir Can Ozden <101993364+bysiber@users.noreply.github.com> Date: Tue, 2 Jun 2026 15:41:52 +0300 Subject: [PATCH 37/55] Fix Timestamp.from_datetime returning wrong value for pre-epoch datetimes (#662) --- msgpack/ext.py | 2 +- test/test_timestamp.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/msgpack/ext.py b/msgpack/ext.py index 9694819a..92ea4530 100644 --- a/msgpack/ext.py +++ b/msgpack/ext.py @@ -167,4 +167,4 @@ def from_datetime(dt): :rtype: Timestamp """ - return Timestamp(seconds=int(dt.timestamp()), nanoseconds=dt.microsecond * 1000) + return Timestamp(seconds=int(dt.timestamp() // 1), nanoseconds=dt.microsecond * 1000) diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 831141a1..6dfe3689 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -103,6 +103,13 @@ def test_timestamp_datetime(): assert Timestamp.from_datetime(ts).to_datetime() == ts + # Regression test: pre-epoch fractional seconds must floor toward -inf. + pre_epoch = datetime.datetime(1969, 12, 31, 23, 59, 59, 500000, tzinfo=utc) + ts_pre_epoch = Timestamp.from_datetime(pre_epoch) + assert ts_pre_epoch.seconds == -1 + assert ts_pre_epoch.nanoseconds == 500000000 + assert ts_pre_epoch.to_datetime() == pre_epoch + def test_unpack_datetime(): t = Timestamp(42, 14) From 975a2a4c84095c3a16b32bfaf5d6dea0a9593d79 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 08:42:57 -0400 Subject: [PATCH 38/55] Add 3.15 to CI (#678) --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8873d51..8b36670b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] - py: ["3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] + py: ["3.15", "3.15t", "3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - os: windows-11-arm py: "3.10" From e861f75374b023c9a6b35451757d23e713ec4068 Mon Sep 17 00:00:00 2001 From: Thomas Kowalski Date: Tue, 2 Jun 2026 17:21:59 +0200 Subject: [PATCH 39/55] fix: use-after-free in `get_data_from_buffer` (#677) --- msgpack/_unpacker.pyx | 7 +++---- msgpack/fallback.py | 2 +- test/test_memoryview.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 29cdec4a..40d12291 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -129,10 +129,9 @@ cdef inline int get_data_from_buffer(object obj, PyBuffer_Release(view) # create a contiguous copy and get buffer contiguous = PyMemoryView_GetContiguous(obj, PyBUF_READ, b'C') - PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) - # view must hold the only reference to contiguous, - # so memory is freed when view is released - Py_DECREF(contiguous) + if PyObject_GetBuffer(contiguous, view, PyBUF_SIMPLE) == -1: + raise + buffer_len[0] = view.len buf[0] = view.buf return 1 diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 61d77db0..860d94ac 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -328,7 +328,7 @@ def feed(self, next_bytes): self._buf_checkpoint = 0 # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython - self._buffer.extend(view) + self._buffer.extend(view if view.contiguous else view.tobytes()) view.release() def _consume(self): diff --git a/test/test_memoryview.py b/test/test_memoryview.py index 0a2a6f53..3f6a39d4 100644 --- a/test/test_memoryview.py +++ b/test/test_memoryview.py @@ -97,3 +97,15 @@ def test_multidim_memoryview(): data = view.cast(view.format, (3, 2)) packed = packb(data) assert packed == b"\xc4\x06\x00\x00\x00\x00\x00\x00" + + +def test_unpack_noncontiguous_memoryview(): + # Use a multi-byte value so the padded stride-2 view is non-contiguous. + packed = packb(2**32) + padded = bytearray() + for byte in packed: + padded.append(byte) + padded.append(0) + noncont = memoryview(bytes(padded))[::2] + assert not noncont.c_contiguous + assert unpackb(noncont) == 2**32 From 6cd75741833c38a482c5007c9113e3a263cdcc3d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 00:44:43 +0900 Subject: [PATCH 40/55] change changelog format to markdown (#680) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 555 +++++++++++++++++++++++++++++++++++++++++++ ChangeLog.rst | 625 ------------------------------------------------- pyproject.toml | 2 +- 3 files changed, 556 insertions(+), 626 deletions(-) create mode 100644 CHANGELOG.md delete mode 100644 ChangeLog.rst diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000..74e6eb47 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,555 @@ +# 1.1.2 + +Release Date: 2025-10-08 + +This release does not change source code. It updates only building +wheels: + +- Update Cython to v3.1.4 +- Update cibuildwheel to v3.2.0 +- Drop Python 3.8 +- Add Python 3.14 +- Add windows-arm + +# 1.1.1 + +Release Date: 2025-06-13 + +- No change from 1.1.1rc1. + +# 1.1.1rc1 + +Release Date: 2025-06-06 + +- Update Cython to 3.1.1 and cibuildwheel to 2.23.3. + +# 1.1.0 + +Release Date: 2024-09-10 + +- use `PyLong_*` instead of `PyInt_*` for compatibility with future + Cython. (#620) + +# 1.1.0rc2 + +Release Date: 2024-08-19 + +- Update Cython to 3.0.11 for better Python 3.13 support. +- Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. + +# 1.1.0rc1 + +Release Date: 2024-05-07 + +- Update Cython to 3.0.10 to reduce C warnings and future support for + Python 3.13. +- Stop using C++ mode in Cython to reduce compile error on some + compilers. +- `Packer()` has `buf_size` option to specify initial size of internal + buffer to reduce reallocation. +- The default internal buffer size of `Packer()` is reduced from 1MiB to + 256KiB to optimize for common use cases. Use `buf_size` if you are + packing large data. +- `Timestamp.to_datetime()` and `Timestamp.from_datetime()` become more + accurate by avoiding floating point calculations. (#591) +- The Cython code for `Unpacker` has been slightly rewritten for + maintainability. +- The fallback implementation of `Packer()` and `Unpacker()` now uses + keyword-only arguments to improve compatibility with the Cython + implementation. + +# 1.0.8 + +Release Date: 2024-03-01 + +- Update Cython to 3.0.8. This fixes memory leak when iterating + `Unpacker` object on Python 3.12. +- Do not include C/Cython files in binary wheels. + +# 1.0.7 + +Release Date: 2023-09-28 + +- Fix build error of extension module on Windows. (#567) +- `setup.py` doesn't skip build error of extension module. (#568) + +# 1.0.6 + +Release Date: 2023-09-21 + +> [!NOTE] +> v1.0.6 Wheels for Windows don't contain extension module. Please +> upgrade to v1.0.7 or newer. + +- Add Python 3.12 wheels (#517) +- Remove Python 2.7, 3.6, and 3.7 support + +# 1.0.5 + +Release Date: 2023-03-08 + +- Use `__BYTE_ORDER__` instead of `__BYTE_ORDER` for portability. (#513, + \#514) +- Add Python 3.11 wheels (#517) +- fallback: Fix packing multidimensional memoryview (#527) + +# 1.0.4 + +Release Date: 2022-06-03 + +- Support Python 3.11 (beta). +- Don't define `__*_ENDIAN__` macro + on Unix. by @methane in + +- Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in + +- Fix Unpacker max_buffer_length handling by @methane in + + +# 1.0.3 + +Release Date: 2021-11-24 JST + +- Fix Docstring (#459) +- Fix error formatting (#463) +- Improve error message about strict_map_key (#485) + +# 1.0.2 + +- Fix year 2038 problem regression in 1.0.1. (#451) + +# 1.0.1 + +- Add Python 3.9 and linux/arm64 wheels. (#439) +- Fixed Unpacker.tell() after read_bytes() (#426) +- Fixed unpacking datetime before epoch on Windows (#433) +- Fixed fallback Packer didn't check DateTime.tzinfo (#434) + +# 1.0.0 + +Release Date: 2020-02-17 + +- Remove Python 2 support from the `msgpack/_cmsgpack`. + `msgpack/fallback` still supports Python 2. +- Remove `encoding` option from the Packer and Unpacker. +- Unpacker: The default value of `max_buffer_size` is changed to 100MiB. +- Unpacker: `strict_map_key` is True by default now. +- Unpacker: String map keys are interned. +- Drop old buffer protocol support. +- Support Timestamp type. +- Support serializing and decerializing `datetime` object with tzinfo. +- Unpacker: `Fix Unpacker.read_bytes()` in fallback implementation. + (#352) + +# 0.6.2 + +Release Date: 2019-09-20 + +- Support Python 3.8. +- Update Cython to 0.29.13 for support Python 3.8. +- Some small optimizations. + +# 0.6.1 + +Release Date: 2019-01-25 + +This release is for mitigating pain caused by v0.6.0 reduced max input +limits for security reason. + +- `unpackb(data)` configures `max_*_len` options from `len(data)`, + instead of static default sizes. +- `Unpacker(max_buffer_len=N)` configures `max_*_len` options from `N`, + instead of static default sizes. +- `max_bin_len`, `max_str_len`, and `max_ext_len` are deprecated. Since + this is minor release, it's document only deprecation. + +# 0.6.0 + +Release Date: 2018-11-30 + +This release contains some backward incompatible changes for security +reason (DoS). + +## Important changes + +- unpacker: Default value of input limits are smaller than before to + avoid DoS attack. If you need to handle large data, you need to + specify limits manually. (#319) +- Unpacker doesn't wrap underlying `ValueError` (including + `UnicodeError`) into `UnpackValueError`. If you want to catch all + exception during unpack, you need to use `try ... except Exception` + with minimum try code block. (#323, \#233) +- `PackValueError` and `PackOverflowError` are also removed. You need to + catch normal `ValueError` and `OverflowError`. (#323, \#233) +- Unpacker has `strict_map_key` option now. When it is true, only bytes + and str (unicode in Python 2) are allowed for map keys. It is + recommended to avoid hashdos. Default value of this option is False + for backward compatibility reason. But it will be changed True in 1.0. + (#296, \#334) + +## Other changes + +- Extension modules are merged. There is `msgpack._cmsgpack` instead of + `msgpack._packer` and `msgpack._unpacker`. (#314, \#328) +- Add `Unpacker.getbuffer()` method. (#320) +- unpacker: `msgpack.StackError` is raised when input data contains too + nested data. (#331) +- unpacker: `msgpack.FormatError` is raised when input data is not valid + msgpack format. (#331) + +# 0.5.6 + +- Fix fallback.Unpacker.feed() dropped unused data from buffer (#287) +- Resurrect fallback.unpack() and `unpacker.unpack()`. They were removed + at 0.5.5 but it breaks backward compatibility. (#288, #290) + +# 0.5.5 + +- Fix memory leak in pure Python Unpacker.feed() (#283) +- Fix unpack() didn't support `raw` option + (#285) + +# 0.5.4 + +- Undeprecate `unicode_errors` option. (#278) + +# 0.5.3 + +- Fixed regression when passing `unicode_errors` to Packer but not + `encoding`. (#277) + +# 0.5.2 + +- Add `raw` option to Unpacker. It is preferred way than `encoding` + option. +- Packer.pack() reset buffer on exception (#274) + +# 0.5.1 + +- Remove FutureWarning about use_bin_type option (#271) + +# 0.5.0 + +There are some deprecations. Please read changes carefully. + +## Changes + +- Drop Python 2.6 and ~3.4 support. Python 2.7 and 3.5+ are supported. +- Deprecate useless custom exceptions. Use ValueError instead of + PackValueError, Exception instead of PackException and + UnpackException, etc... See msgpack/exceptions.py +- Add *strict_types* option to packer. It can be used to serialize + subclass of builtin types. For example, when packing object which type + is subclass of dict, `default()` is called. `default()` is called for + tuple too. +- Pure Python implementation supports packing memoryview object. +- Support packing bytearray. +- Add `Unpacker.tell()`. And `write_bytes` option is deprecated. + +## Bugs fixed + +- Fixed zero length raw can't be decoded when encoding is specified. + (#236) + +# 0.4.8 + +Release Date: 2016-07-29 + +## Bugs fixed + +- Calling ext_hook with wrong length. (Only on Windows, maybe. \#203) + +# 0.4.7 + +Release Date: 2016-01-25 + +## Bugs fixed + +- Memory leak when unpack is failed + +## Changes + +- Reduce compiler warnings while building extension module +- unpack() now accepts ext_hook argument like Unpacker and unpackb() +- Update Cython version to 0.23.4 +- default function is called when integer overflow + +# 0.4.6 + +Release Date: 2015-03-13 + +## Bugs fixed + +- fallback.Unpacker: Fix Data corruption when OutOfData. This bug only + affects "Streaming unpacking." + +# 0.4.5 + +Release Date: 2015-01-25 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Fix test failure on pytest 2.3. (by @ktdreyer) +- Fix typos in ChangeLog. (Thanks to @dmick) +- Improve README.rst (by @msabramo) + +# 0.4.4 + +Release Date: 2015-01-09 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Fix compile error. + +# 0.4.3 + +Release Date: 2015-01-07 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. + (#101) +- Build failed on Windows Python 2.7. + +# 0.4.2 + +Release Date: 2014-03-26 + +## Incompatible Changes + +## Changes + +## Bugs fixed + +- Unpacker doesn't increment refcount of ExtType hook. +- Packer raises no exception for inputs doesn't fit to msgpack format. + +# 0.4.1 + +Release Date: 2014-02-17 + +## Incompatible Changes + +## Changes + +- fallback.Unpacker.feed() supports bytearray. + +## Bugs fixed + +- Unpacker doesn't increment refcount of hooks. Hooks may be GCed while + unpacking. +- Unpacker may read unfilled internal buffer. + +# 0.4.0 + +Release Date: 2013-10-21 + +## Incompatible Changes + +- Raises TypeError instead of ValueError when packer receives + unsupported type. + +## Changes + +- Support New msgpack spec. + +# 0.3.0 + +## Incompatible Changes + +- Default value of `use_list` is `True` for now. (It was `False` for + 0.2.x) You should pass it explicitly for compatibility to 0.2.x. +- `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of + `StopIteration`. `StopIteration` is used for iterator protocol only. + +## Changes + +- Pure Python fallback module is added. (thanks to bwesterb) +- Add `.skip()` method to `Unpacker` (thanks to jnothman) +- Add capturing feature. You can pass the writable object to + `Unpacker.unpack()` as a second parameter. +- Add `Packer.pack_array_header` and `Packer.pack_map_header`. These + methods only pack header of each type. +- Add `autoreset` option to `Packer` (default: True). Packer doesn't + return packed bytes and clear internal buffer. +- Add `Packer.pack_map_pairs`. It packs sequence of pair to map type. + +# 0.2.4 + +Release Date: 2012-12-22 + +## Bugs fixed + +- Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) + +# 0.2.3 + +Release Date: 2012-12-11 + +## Changes + +- Warn when use_list is not specified. It's default value will be + changed in 0.3. + +## Bugs fixed + +- Can't pack subclass of dict. + +# 0.2.2 + +Release Date: 2012-09-21 + +## Changes + +- Add `use_single_float` option to `Packer`. When it is true, packs + float object in single precision format. + +## Bugs fixed + +- `unpack()` didn't restores gc state when it called with gc disabled. + `unpack()` doesn't control gc now instead of restoring gc state + collectly. User can control gc state when gc cause performance issue. +- `Unpacker`'s `read_size` option didn't used. + +# 0.2.1 + +Release Date: 2012-08-20 + +## Changes + +- Add `max_buffer_size` parameter to Unpacker. It limits internal buffer + size and allows unpack data from untrusted source safely. +- Unpacker's buffer reallocation algorithm is less greedy now. It cause + performance decrease in rare case but memory efficient and don't + allocate than `max_buffer_size`. + +## Bugs fixed + +- Fix msgpack didn't work on SPARC Solaris. It was because choosing + wrong byteorder on compilation time. Use `sys.byteorder` to get + correct byte order. Very thanks to Chris Casey for giving test + environment to me. + +# 0.2.0 + +Release Date: 2012-06-27 + +## Changes + +- Drop supporting Python 2.5 and unify tests for Py2 and Py3. +- Use new version of msgpack-c. It packs correctly on big endian + platforms. +- Remove deprecated packs and unpacks API. + +## Bugs fixed + +- \#8 Packing subclass of dict raises TypeError. (Thanks to Steeve + Morin.) + +# 0.1.13 + +Release Date: 2012-04-21 + +## New + +- Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) + It allows customize how it serialized with `default` argument. + +## Bugs fixed + +- Fix wrong error message. (David Wolever) +- Fix memory leak while unpacking when `object_hook` or `list_hook` is + used. (Steeve Morin) + +## Other changes + +- setup.py works on Python 2.5 (Steffen Siering) +- Optimization for serializing dict. + +# 0.1.12 + +Release Date: 2011-12-27 + +## Bugs fixed + +- Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 + is released. + +# 0.1.11 + +Release Date: 2011-12-26 + +## Bugs fixed + +- Include test code for Python3 to sdist. (Johan Bergström) +- Fix compilation error on MSVC. (davidgaleano) + +# 0.1.10 + +Release Date: 2011-08-22 + +## New feature + +- Add `encoding` and `unicode_errors` option to packer and unpacker. + When this option is specified, (un)packs unicode object instead of + bytes. This enables using msgpack as a replacement of json. (tailhook) + +# 0.1.9 + +Release Date: 2011-01-29 + +## New feature + +- `use_list` option is added to unpack(b) like Unpacker. (Use keyword + argument because order of parameters are different) + +## Bugs fixed + +- Fix typo. +- Add MemoryError check. + +# 0.1.8 + +Release Date: 2011-01-10 + +## New feature + +- Support `loads` and `dumps` aliases for API compatibility with + simplejson and pickle. +- Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacking mapping type and array type. +- Add *default* option to packer. It allows you to pack unsupported + types. +- unpacker accepts (old) buffer types. + +## Bugs fixed + +- Fix segv around `Unpacker.feed` or `Unpacker(file)`. + +# 0.1.7 + +Release Date: 2010-11-02 + +## New feature + +- Add *object_hook* and *list_hook* option to unpacker. It allows you to + hook unpacking mapping type and array type. +- Add *default* option to packer. It allows you to pack unsupported + types. +- unpacker accepts (old) buffer types. + +## Bugs fixed + +- Compilation error on win32. diff --git a/ChangeLog.rst b/ChangeLog.rst deleted file mode 100644 index beeab15c..00000000 --- a/ChangeLog.rst +++ /dev/null @@ -1,625 +0,0 @@ -1.1.2 -===== - -Release Date: 2025-10-08 - -This release does not change source code. It updates only building wheels: - -* Update Cython to v3.1.4 -* Update cibuildwheel to v3.2.0 -* Drop Python 3.8 -* Add Python 3.14 -* Add windows-arm - -1.1.1 -===== - -Release Date: 2025-06-13 - -* No change from 1.1.1rc1. - -1.1.1rc1 -======== - -Release Date: 2025-06-06 - -* Update Cython to 3.1.1 and cibuildwheel to 2.23.3. - -1.1.0 -===== - -Release Date: 2024-09-10 - -* use ``PyLong_*`` instead of ``PyInt_*`` for compatibility with - future Cython. (#620) - -1.1.0rc2 -======== - -Release Date: 2024-08-19 - -* Update Cython to 3.0.11 for better Python 3.13 support. -* Update cibuildwheel to 2.20.0 to build Python 3.13 wheels. - -1.1.0rc1 -======== - -Release Date: 2024-05-07 - -* Update Cython to 3.0.10 to reduce C warnings and future support for Python 3.13. -* Stop using C++ mode in Cython to reduce compile error on some compilers. -* ``Packer()`` has ``buf_size`` option to specify initial size of - internal buffer to reduce reallocation. -* The default internal buffer size of ``Packer()`` is reduced from - 1MiB to 256KiB to optimize for common use cases. Use ``buf_size`` - if you are packing large data. -* ``Timestamp.to_datetime()`` and ``Timestamp.from_datetime()`` become - more accurate by avoiding floating point calculations. (#591) -* The Cython code for ``Unpacker`` has been slightly rewritten for maintainability. -* The fallback implementation of ``Packer()`` and ``Unpacker()`` now uses keyword-only - arguments to improve compatibility with the Cython implementation. - -1.0.8 -===== - -Release Date: 2024-03-01 - -* Update Cython to 3.0.8. This fixes memory leak when iterating - ``Unpacker`` object on Python 3.12. -* Do not include C/Cython files in binary wheels. - - -1.0.7 -===== - -Release Date: 2023-09-28 - -* Fix build error of extension module on Windows. (#567) -* ``setup.py`` doesn't skip build error of extension module. (#568) - - -1.0.6 -===== - -Release Date: 2023-09-21 - -.. note:: - v1.0.6 Wheels for Windows don't contain extension module. - Please upgrade to v1.0.7 or newer. - -* Add Python 3.12 wheels (#517) -* Remove Python 2.7, 3.6, and 3.7 support - - -1.0.5 -===== - -Release Date: 2023-03-08 - -* Use ``__BYTE_ORDER__`` instead of ``__BYTE_ORDER`` for portability. (#513, #514) -* Add Python 3.11 wheels (#517) -* fallback: Fix packing multidimensional memoryview (#527) - -1.0.4 -===== - -Release Date: 2022-06-03 - -* Support Python 3.11 (beta). -* Don't define `__*_ENDIAN__` macro on Unix. by @methane in https://github.com/msgpack/msgpack-python/pull/495 -* Use PyFloat_Pack8() on Python 3.11a7 by @vstinner in https://github.com/msgpack/msgpack-python/pull/499 -* Fix Unpacker max_buffer_length handling by @methane in https://github.com/msgpack/msgpack-python/pull/506 - -1.0.3 -===== - -Release Date: 2021-11-24 JST - -* Fix Docstring (#459) -* Fix error formatting (#463) -* Improve error message about strict_map_key (#485) - -1.0.2 -===== - -* Fix year 2038 problem regression in 1.0.1. (#451) - -1.0.1 -===== - -* Add Python 3.9 and linux/arm64 wheels. (#439) -* Fixed Unpacker.tell() after read_bytes() (#426) -* Fixed unpacking datetime before epoch on Windows (#433) -* Fixed fallback Packer didn't check DateTime.tzinfo (#434) - -1.0.0 -===== - -Release Date: 2020-02-17 - -* Remove Python 2 support from the ``msgpack/_cmsgpack``. - ``msgpack/fallback`` still supports Python 2. -* Remove ``encoding`` option from the Packer and Unpacker. -* Unpacker: The default value of ``max_buffer_size`` is changed to 100MiB. -* Unpacker: ``strict_map_key`` is True by default now. -* Unpacker: String map keys are interned. -* Drop old buffer protocol support. -* Support Timestamp type. -* Support serializing and decerializing ``datetime`` object - with tzinfo. -* Unpacker: ``Fix Unpacker.read_bytes()`` in fallback implementation. (#352) - - -0.6.2 -===== - -Release Date: 2019-09-20 - -* Support Python 3.8. -* Update Cython to 0.29.13 for support Python 3.8. -* Some small optimizations. - - -0.6.1 -====== - -Release Date: 2019-01-25 - -This release is for mitigating pain caused by v0.6.0 reduced max input limits -for security reason. - -* ``unpackb(data)`` configures ``max_*_len`` options from ``len(data)``, - instead of static default sizes. - -* ``Unpacker(max_buffer_len=N)`` configures ``max_*_len`` options from ``N``, - instead of static default sizes. - -* ``max_bin_len``, ``max_str_len``, and ``max_ext_len`` are deprecated. - Since this is minor release, it's document only deprecation. - - -0.6.0 -====== - -Release Date: 2018-11-30 - -This release contains some backward incompatible changes for security reason (DoS). - -Important changes ------------------ - -* unpacker: Default value of input limits are smaller than before to avoid DoS attack. - If you need to handle large data, you need to specify limits manually. (#319) - -* Unpacker doesn't wrap underlying ``ValueError`` (including ``UnicodeError``) into - ``UnpackValueError``. If you want to catch all exception during unpack, you need - to use ``try ... except Exception`` with minimum try code block. (#323, #233) - -* ``PackValueError`` and ``PackOverflowError`` are also removed. You need to catch - normal ``ValueError`` and ``OverflowError``. (#323, #233) - -* Unpacker has ``strict_map_key`` option now. When it is true, only bytes and str - (unicode in Python 2) are allowed for map keys. It is recommended to avoid - hashdos. Default value of this option is False for backward compatibility reason. - But it will be changed True in 1.0. (#296, #334) - -Other changes -------------- - -* Extension modules are merged. There is ``msgpack._cmsgpack`` instead of - ``msgpack._packer`` and ``msgpack._unpacker``. (#314, #328) - -* Add ``Unpacker.getbuffer()`` method. (#320) - -* unpacker: ``msgpack.StackError`` is raised when input data contains too - nested data. (#331) - -* unpacker: ``msgpack.FormatError`` is raised when input data is not valid - msgpack format. (#331) - - -0.5.6 -====== - -* Fix fallback.Unpacker.feed() dropped unused data from buffer (#287) -* Resurrect fallback.unpack() and _unpacker.unpack(). - They were removed at 0.5.5 but it breaks backward compatibility. (#288, #290) - -0.5.5 -====== - -* Fix memory leak in pure Python Unpacker.feed() (#283) -* Fix unpack() didn't support `raw` option (#285) - -0.5.4 -====== - -* Undeprecate ``unicode_errors`` option. (#278) - -0.5.3 -====== - -* Fixed regression when passing ``unicode_errors`` to Packer but not ``encoding``. (#277) - -0.5.2 -====== - -* Add ``raw`` option to Unpacker. It is preferred way than ``encoding`` option. - -* Packer.pack() reset buffer on exception (#274) - - -0.5.1 -====== - -* Remove FutureWarning about use_bin_type option (#271) - -0.5.0 -====== - -There are some deprecations. Please read changes carefully. - -Changes -------- - -* Drop Python 2.6 and ~3.4 support. Python 2.7 and 3.5+ are supported. - -* Deprecate useless custom exceptions. Use ValueError instead of PackValueError, - Exception instead of PackException and UnpackException, etc... - See msgpack/exceptions.py - -* Add *strict_types* option to packer. It can be used to serialize subclass of - builtin types. For example, when packing object which type is subclass of dict, - ``default()`` is called. ``default()`` is called for tuple too. - -* Pure Python implementation supports packing memoryview object. - -* Support packing bytearray. - -* Add ``Unpacker.tell()``. And ``write_bytes`` option is deprecated. - - -Bugs fixed ----------- - -* Fixed zero length raw can't be decoded when encoding is specified. (#236) - - -0.4.8 -===== -:release date: 2016-07-29 - -Bugs fixed ----------- - -* Calling ext_hook with wrong length. (Only on Windows, maybe. #203) - - -0.4.7 -===== -:release date: 2016-01-25 - -Bugs fixed ----------- - -* Memory leak when unpack is failed - -Changes -------- - -* Reduce compiler warnings while building extension module -* unpack() now accepts ext_hook argument like Unpacker and unpackb() -* Update Cython version to 0.23.4 -* default function is called when integer overflow - - -0.4.6 -===== -:release date: 2015-03-13 - -Bugs fixed ----------- - -* fallback.Unpacker: Fix Data corruption when OutOfData. - This bug only affects "Streaming unpacking." - - -0.4.5 -===== -:release date: 2015-01-25 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Fix test failure on pytest 2.3. (by @ktdreyer) -* Fix typos in ChangeLog. (Thanks to @dmick) -* Improve README.rst (by @msabramo) - - -0.4.4 -===== -:release date: 2015-01-09 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Fix compile error. - -0.4.3 -===== -:release date: 2015-01-07 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Unpacker may unpack wrong uint32 value on 32bit or LLP64 environment. (#101) -* Build failed on Windows Python 2.7. - -0.4.2 -===== -:release date: 2014-03-26 - -Incompatible Changes --------------------- - -Changes -------- - -Bugs fixed ----------- - -* Unpacker doesn't increment refcount of ExtType hook. -* Packer raises no exception for inputs doesn't fit to msgpack format. - -0.4.1 -===== -:release date: 2014-02-17 - -Incompatible Changes --------------------- - -Changes -------- - -* fallback.Unpacker.feed() supports bytearray. - -Bugs fixed ----------- - -* Unpacker doesn't increment refcount of hooks. Hooks may be GCed while unpacking. -* Unpacker may read unfilled internal buffer. - -0.4.0 -===== -:release date: 2013-10-21 - -Incompatible Changes --------------------- - -* Raises TypeError instead of ValueError when packer receives unsupported type. - -Changes -------- - -* Support New msgpack spec. - - -0.3.0 -===== - -Incompatible Changes --------------------- - -* Default value of ``use_list`` is ``True`` for now. (It was ``False`` for 0.2.x) - You should pass it explicitly for compatibility to 0.2.x. -* `Unpacker.unpack()` and some unpack methods now raise `OutOfData` instead of - `StopIteration`. `StopIteration` is used for iterator protocol only. - -Changes -------- -* Pure Python fallback module is added. (thanks to bwesterb) -* Add ``.skip()`` method to ``Unpacker`` (thanks to jnothman) -* Add capturing feature. You can pass the writable object to - ``Unpacker.unpack()`` as a second parameter. -* Add ``Packer.pack_array_header`` and ``Packer.pack_map_header``. - These methods only pack header of each type. -* Add ``autoreset`` option to ``Packer`` (default: True). - Packer doesn't return packed bytes and clear internal buffer. -* Add ``Packer.pack_map_pairs``. It packs sequence of pair to map type. - - - -0.2.4 -===== -:release date: 2012-12-22 - -Bugs fixed ----------- - -* Fix SEGV when object_hook or object_pairs_hook raise Exception. (#39) - -0.2.3 -===== -:release date: 2012-12-11 - -Changes -------- -* Warn when use_list is not specified. It's default value will be changed in 0.3. - -Bugs fixed ----------- -* Can't pack subclass of dict. - -0.2.2 -===== -:release date: 2012-09-21 - -Changes -------- -* Add ``use_single_float`` option to ``Packer``. When it is true, packs float - object in single precision format. - -Bugs fixed ----------- -* ``unpack()`` didn't restores gc state when it called with gc disabled. - ``unpack()`` doesn't control gc now instead of restoring gc state collectly. - User can control gc state when gc cause performance issue. - -* ``Unpacker``'s ``read_size`` option didn't used. - -0.2.1 -===== -:release date: 2012-08-20 - -Changes -------- -* Add ``max_buffer_size`` parameter to Unpacker. It limits internal buffer size - and allows unpack data from untrusted source safely. - -* Unpacker's buffer reallocation algorithm is less greedy now. It cause performance - decrease in rare case but memory efficient and don't allocate than ``max_buffer_size``. - -Bugs fixed ----------- -* Fix msgpack didn't work on SPARC Solaris. It was because choosing wrong byteorder - on compilation time. Use ``sys.byteorder`` to get correct byte order. - Very thanks to Chris Casey for giving test environment to me. - - -0.2.0 -===== -:release date: 2012-06-27 - -Changes -------- -* Drop supporting Python 2.5 and unify tests for Py2 and Py3. -* Use new version of msgpack-c. It packs correctly on big endian platforms. -* Remove deprecated packs and unpacks API. - -Bugs fixed ----------- -* #8 Packing subclass of dict raises TypeError. (Thanks to Steeve Morin.) - - -0.1.13 -====== -:release date: 2012-04-21 - -New ---- -* Don't accept subtype of list and tuple as msgpack list. (Steeve Morin) - It allows customize how it serialized with ``default`` argument. - -Bugs fixed ----------- -* Fix wrong error message. (David Wolever) -* Fix memory leak while unpacking when ``object_hook`` or ``list_hook`` is used. - (Steeve Morin) - -Other changes -------------- -* setup.py works on Python 2.5 (Steffen Siering) -* Optimization for serializing dict. - - -0.1.12 -====== -:release date: 2011-12-27 - -Bugs fixed ----------- - -* Re-enable packs/unpacks removed at 0.1.11. It will be removed when 0.2 is released. - - -0.1.11 -====== -:release date: 2011-12-26 - -Bugs fixed ----------- - -* Include test code for Python3 to sdist. (Johan Bergström) -* Fix compilation error on MSVC. (davidgaleano) - - -0.1.10 -====== -:release date: 2011-08-22 - -New feature ------------ -* Add ``encoding`` and ``unicode_errors`` option to packer and unpacker. - When this option is specified, (un)packs unicode object instead of bytes. - This enables using msgpack as a replacement of json. (tailhook) - - -0.1.9 -===== -:release date: 2011-01-29 - -New feature ------------ -* ``use_list`` option is added to unpack(b) like Unpacker. - (Use keyword argument because order of parameters are different) - -Bugs fixed ----------- -* Fix typo. -* Add MemoryError check. - -0.1.8 -===== -:release date: 2011-01-10 - -New feature ------------ -* Support ``loads`` and ``dumps`` aliases for API compatibility with - simplejson and pickle. - -* Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacking mapping type and array type. - -* Add *default* option to packer. It allows you to pack unsupported types. - -* unpacker accepts (old) buffer types. - -Bugs fixed ----------- -* Fix segv around ``Unpacker.feed`` or ``Unpacker(file)``. - - -0.1.7 -===== -:release date: 2010-11-02 - -New feature ------------ -* Add *object_hook* and *list_hook* option to unpacker. It allows you to - hook unpacking mapping type and array type. - -* Add *default* option to packer. It allows you to pack unsupported types. - -* unpacker accepts (old) buffer types. - -Bugs fixed ----------- -* Compilation error on win32. diff --git a/pyproject.toml b/pyproject.toml index 4e4510b4..ebc6b50d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ Homepage = "https://msgpack.org/" Documentation = "https://msgpack-python.readthedocs.io/" Repository = "https://github.com/msgpack/msgpack-python/" Tracker = "https://github.com/msgpack/msgpack-python/issues" -Changelog = "https://github.com/msgpack/msgpack-python/blob/main/ChangeLog.rst" +Changelog = "https://github.com/msgpack/msgpack-python/blob/main/CHANGELOG.md" [tool.setuptools] # Do not install C/C++/Cython source files From e327a0e93bc742b19f6a407c1dbbc732a822ce17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Jun 2026 10:36:10 +0900 Subject: [PATCH 41/55] Bump the all-dependencies group with 2 updates (#684) --- .github/workflows/docs.yaml | 2 +- .github/workflows/lint.yaml | 2 +- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 4467ca8b..221f5d42 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e2631ffa..6ed5a1c1 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-slim steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: ruff check run: | diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b36670b..169f1fda 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - name: Set up Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index eb457277..a9c4b09e 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -27,7 +27,7 @@ jobs: name: Build wheels on ${{ matrix.os }}${{ matrix.name_suffix || '' }} steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" @@ -40,7 +40,7 @@ jobs: - name: Set up QEMU for emulation if: matrix.cibw_archs == 'riscv64' - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 + uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0 with: platforms: ${{ matrix.cibw_archs }} From d627d308ef6308dd86911e5f6de1a4bc56183aa3 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Tue, 2 Jun 2026 23:30:00 -0400 Subject: [PATCH 42/55] Bump sys.setrecursionlimit within test_nest_limit_1024 (#682) Fixes test failures with no-GIL interpreters --- test/test_limits.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_limits.py b/test/test_limits.py index e3bab146..bb554d68 100644 --- a/test/test_limits.py +++ b/test/test_limits.py @@ -170,7 +170,7 @@ def test_nest_limit_1024(): # Temporarily raise Python's recursion limit so packing 1024 levels succeeds old_limit = sys.getrecursionlimit() - sys.setrecursionlimit(max(old_limit, 10000)) + sys.setrecursionlimit(max(old_limit, 30000)) try: packed = packb(d) result = unpackb(packed) From 7df7136e20933fd3a498bd58bff7975a24790516 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:22:40 +0900 Subject: [PATCH 43/55] Guard `Packer` buffer protocol hooks with Cython critical sections (#686) --- msgpack/_packer.pyx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 2d92e8a4..277239d8 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -360,9 +360,11 @@ cdef class Packer: """ return memoryview(self) + @cython.critical_section def __getbuffer__(self, Py_buffer *buffer, int flags): PyBuffer_FillInfo(buffer, self, self.pk.buf, self.pk.length, 1, flags) self.exports += 1 + @cython.critical_section def __releasebuffer__(self, Py_buffer *buffer): self.exports -= 1 From 77395c19a50061238306df5ea10e67c5312ceff0 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Wed, 3 Jun 2026 16:40:16 +0900 Subject: [PATCH 44/55] Harden `Unpacker.__init__` re-entry cleanup to prevent buffer/context leaks (#687) --- msgpack/_unpacker.pyx | 9 ++++++--- test/test_unpack.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 40d12291..4bfbe064 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -317,9 +317,6 @@ cdef class Unpacker: cdef Py_ssize_t max_buffer_size cdef uint64_t stream_offset - def __cinit__(self): - self.buf = NULL - def __dealloc__(self): unpack_clear(&self.ctx) PyMem_Free(self.buf) @@ -338,6 +335,12 @@ cdef class Unpacker: Py_ssize_t max_ext_len=-1): cdef const char *cerr=NULL + unpack_clear(&self.ctx) + unpack_init(&self.ctx) + if self.buf != NULL: + PyMem_Free(self.buf) + self.buf = NULL + self.object_hook = object_hook self.object_pairs_hook = object_pairs_hook self.list_hook = list_hook diff --git a/test/test_unpack.py b/test/test_unpack.py index b17c3c53..705c16a6 100644 --- a/test/test_unpack.py +++ b/test/test_unpack.py @@ -1,4 +1,6 @@ +import gc import sys +import weakref from io import BytesIO from pytest import mark, raises @@ -87,3 +89,37 @@ def test_unpacker_tell_read_bytes(): assert obj == unp assert pos == unpacker.tell() assert unpacker.read_bytes(n) == raw + + +@mark.skipif( + Unpacker.__module__ == "msgpack.fallback", + reason="specific to C extension reinit leak", +) +def test_unpacker_reinit_clears_partial_state(): + refs = [] + + class Marker: + pass + + def hook(code, data): + obj = Marker() + refs.append(weakref.ref(obj)) + return obj + + unpacker = Unpacker(ext_hook=hook, strict_map_key=False) + # Keep parser state mid-map with a live key object from ext_hook. + # Encodes: [ {ExtType(1, b"a"): } ]. + unpacker.feed(b"\x91\x81\xd4\x01a") + with raises(OutOfData): + unpacker.unpack() + assert len(refs) == 1 + assert refs[0]() is not None + + unpacker.__init__() + gc.collect() + assert refs[0]() is None + with raises(OutOfData): + unpacker.unpack() + + unpacker.feed(packb({"a": 1})) + assert unpacker.unpack() == {"a": 1} From 5eb57e15455f5dd25001e97a7ef1170911dee104 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 17:50:51 +0900 Subject: [PATCH 45/55] release v1.2.0rc1 (#681) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 16 ++++++++++++++++ msgpack/__init__.py | 4 ++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e6eb47..63cfcce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +# 1.2.0 + +Release Date: TBD + +- Support free threaded Python. #654, #686 +- Dropped support for Python 3.9. #656 +- Fix missing error checks in C code. #665, #666, #667, #672 +- Fix `strict_map_key` option didn't work for `object_pairs_hook`. #673 +- Increase DEFAULT_RECURSE_LIMIT of Unpacker to 1024. #676 +- Fix memory leak when Unpacker returns error for invalid input. #671 +- Fix `Packer.pack_ext_type()` ignored `autoreset` option. #663 +- Fix `Timestamp.from_datetime()` returning wrong value for pre-epoch datetimes. #662 +- Fix use-after-free in `unpackb()` and `Unpacker.unpack()` for non-contiguous input. #677 +- Fix possible memory leak when calling `Unpacker.__init__()` several times. #687 + + # 1.1.2 Release Date: 2025-10-08 diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f3266b70..f7346d0f 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 1, 2) -__version__ = "1.1.2" +version = (1, 2, 0) +__version__ = "1.2.0rc1" if os.environ.get("MSGPACK_PUREPYTHON"): From cdde1b0d6336caff4e82310c3c7d2b4c73f22d59 Mon Sep 17 00:00:00 2001 From: Guido Imperiale Date: Wed, 3 Jun 2026 13:20:15 +0100 Subject: [PATCH 46/55] Wheels CI hangs for MacOS Intel (#689) - Fix wheels generation for MacOS Intel, which is hanging due to gha VM brownout: https://github.com/msgpack/msgpack-python/actions/runs/26874183762/job/79271522595 - Run tests on MacOS Intel and on Ubuntu ARM --- .github/workflows/test.yml | 8 +++++++- .github/workflows/wheel.yml | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 169f1fda..bc101aa7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,13 @@ jobs: test: strategy: matrix: - os: ["ubuntu-latest", "windows-latest", "windows-11-arm", "macos-latest"] + os: + - ubuntu-latest + - ubuntu-24.04-arm + - windows-latest + - windows-11-arm + - macos-15-intel + - macos-latest py: ["3.15", "3.15t", "3.14", "3.14t", "3.13", "3.12", "3.11", "3.10"] exclude: - os: windows-11-arm diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index a9c4b09e..fb786883 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -11,13 +11,12 @@ jobs: build_wheels: strategy: matrix: - # macos-13 is for intel include: - os: ubuntu-24.04 - os: ubuntu-24.04-arm - os: windows-latest - os: windows-11-arm - - os: macos-13 + - os: macos-15-intel - os: macos-latest - os: ubuntu-24.04 cibw_archs: riscv64 From 97ba6ca0d29c1e8a5b30112f13f37a4d6b3c4492 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Wed, 3 Jun 2026 22:17:26 +0900 Subject: [PATCH 47/55] skip ci: remove unneeded CIBW_SKIP option --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index fb786883..56b31f03 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -48,7 +48,7 @@ jobs: env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" - CIBW_SKIP: "pp* cp38-* cp39-* cp310-win_arm64" + CIBW_SKIP: "cp38-* cp39-* cp310-win_arm64" CIBW_ARCHS: ${{ matrix.cibw_archs || 'auto' }} - name: Build sdist From c410a388c58d55091736e510cad064977a12cb8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 11 Jun 2026 11:54:05 +0900 Subject: [PATCH 48/55] Bump pypa/cibuildwheel from 3.4.1 to 4.0.0 (#691) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 56b31f03..a3d80b5c 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -44,7 +44,7 @@ jobs: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@8d2b08b68458a16aeb24b64e68a09ab1c8e82084 # v3.4.1 + uses: pypa/cibuildwheel@f03ac7617d6cff873ccf24cc0d567ef5ba5a9e6d # v4.0.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 11ed0a5110c1920b85b7d24d9414f480983e0f16 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Thu, 11 Jun 2026 12:29:41 +0900 Subject: [PATCH 49/55] release v1.2.0 (#692) --- CHANGELOG.md | 2 +- msgpack/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63cfcce1..a29b4b2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # 1.2.0 -Release Date: TBD +Release Date: 2026-06-11 - Support free threaded Python. #654, #686 - Dropped support for Python 3.9. #656 diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f7346d0f..f053abd0 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -5,7 +5,7 @@ from .ext import ExtType, Timestamp version = (1, 2, 0) -__version__ = "1.2.0rc1" +__version__ = "1.2.0" if os.environ.get("MSGPACK_PUREPYTHON"): From 0f4f350b6f3e80f24750422673424cce5d96e15d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jun 2026 13:22:58 +0900 Subject: [PATCH 50/55] Bump pypa/cibuildwheel from 4.0.0 to 4.1.0 in the all-dependencies group (#694) --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index a3d80b5c..92fce016 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -44,7 +44,7 @@ jobs: platforms: ${{ matrix.cibw_archs }} - name: Build - uses: pypa/cibuildwheel@f03ac7617d6cff873ccf24cc0d567ef5ba5a9e6d # v4.0.0 + uses: pypa/cibuildwheel@294735312765b09d24a2fbec22660ce817587d55 # v4.1.0 env: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" From 2c56ddb5d0025ed481d962c0f5d62d19dec7476d Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 00:13:13 +0900 Subject: [PATCH 51/55] Merge commit from fork * fix Unpacker crash after unpack failure. * fixup --- msgpack/_unpacker.pyx | 8 ++++++-- msgpack/unpack_template.h | 22 +++++++++++++++------- requirements.txt | 3 ++- test/test_except.py | 26 ++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 10 deletions(-) diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 4bfbe064..e0cdde43 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -51,6 +51,7 @@ cdef extern from "unpack.h": execute_fn unpack_skip execute_fn read_array_header execute_fn read_map_header + void unpack_init(unpack_context* ctx) object unpack_data(unpack_context* ctx) void unpack_clear(unpack_context* ctx) @@ -197,6 +198,7 @@ def unpackb(object packed, *, object object_hook=None, object list_hook=None, if off < buf_len: raise ExtraData(obj, PyBytes_FromStringAndSize(buf+off, buf_len-off)) return obj + unpack_clear(&ctx) if ret == 0: raise ValueError("Unpack failed: incomplete input") @@ -475,7 +477,7 @@ cdef class Unpacker: obj = unpack_data(&self.ctx) unpack_init(&self.ctx) return obj - elif ret == 0: + if ret == 0: if self.file_like is not None: self.read_from_file() continue @@ -483,7 +485,9 @@ cdef class Unpacker: raise StopIteration("No more data to unpack.") else: raise OutOfData("No more data to unpack.") - elif ret == -2: + + unpack_clear(&self.ctx) + if ret == -2: raise FormatError elif ret == -3: raise StackError diff --git a/msgpack/unpack_template.h b/msgpack/unpack_template.h index 42306618..ab5887ae 100644 --- a/msgpack/unpack_template.h +++ b/msgpack/unpack_template.h @@ -72,15 +72,14 @@ static inline PyObject* unpack_data(unpack_context* ctx) static inline void unpack_clear(unpack_context *ctx) { - unsigned int i; - for (i = 1; i < ctx->top; i++) { - Py_CLEAR(ctx->stack[i].obj); + for (unsigned int i = 0; i < ctx->top; i++) { /* map_key holds a live reference only while waiting for the value */ if (ctx->stack[i].ct == CT_MAP_VALUE) { Py_CLEAR(ctx->stack[i].map_key); } + Py_CLEAR(ctx->stack[i].obj); } - Py_CLEAR(ctx->stack[0].obj); + unpack_init(ctx); } static inline int unpack_execute(bool construct, unpack_context* ctx, const char* data, Py_ssize_t len, Py_ssize_t* off) @@ -200,7 +199,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char case 0xd5: // fixext 2 case 0xd6: // fixext 4 case 0xd7: // fixext 8 - again_fixed_trail_if_zero(ACS_EXT_VALUE, + again_fixed_trail_if_zero(ACS_EXT_VALUE, (1 << (((unsigned int)*p) & 0x03))+1, _ext_zero); case 0xd8: // fixext 16 @@ -344,6 +343,7 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char goto _header_again; case CT_MAP_VALUE: if(construct_cb(_map_item)(user, c->count, &c->obj, c->map_key, obj) < 0) { goto _failed; } + c->map_key = NULL; if(++c->count == c->size) { obj = c->obj; if (construct_cb(_map_end)(user, &obj) < 0) { goto _failed; } @@ -406,10 +406,18 @@ static inline int unpack_execute(bool construct, unpack_context* ctx, const char #undef start_container static int unpack_construct(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(1, ctx, data, len, off); + int ret = unpack_execute(1, ctx, data, len, off); + if (ret == -1) { + unpack_clear(ctx); + } + return ret; } static int unpack_skip(unpack_context *ctx, const char *data, Py_ssize_t len, Py_ssize_t *off) { - return unpack_execute(0, ctx, data, len, off); + int ret = unpack_execute(0, ctx, data, len, off); + if (ret == -1) { + unpack_clear(ctx); + } + return ret; } #define unpack_container_header read_array_header diff --git a/requirements.txt b/requirements.txt index 9e4643b6..7991e3f0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ -Cython==3.2.1 +cython==3.2.5 setuptools==78.1.1 +pytest build diff --git a/test/test_except.py b/test/test_except.py index c04b110d..a3bf4675 100644 --- a/test/test_except.py +++ b/test/test_except.py @@ -33,6 +33,22 @@ def hook(obj): object_pairs_hook=hook, ) + up = Unpacker(object_hook=hook) + + def up_unpack(x): + up.feed(x) + return up.unpack() + + raises(DummyException, up_unpack, packb({})) + raises(DummyException, up_unpack, packb({"fizz": "buzz"})) + raises(DummyException, up_unpack, packb({"fizz": "buzz"})) + raises(DummyException, up_unpack, packb({"fizz": {"buzz": "spam"}})) + raises( + DummyException, + up_unpack, + packb({"fizz": {"buzz": "spam"}}), + ) + def test_raise_from_list_hook(): def hook(lst: list) -> list: @@ -138,3 +154,13 @@ def test_strict_map_key_with_object_pairs_hook(): packed = packb(valid, use_bin_type=True) result = unpackb(packed, raw=False, strict_map_key=True, object_pairs_hook=list) assert result == [("key", "value")] + + +def test_unpacker_should_not_crash_after_exception(): + up = Unpacker() # default: strict_map_key=True + up.feed(b"\x83\x73\xc4\x00") # fixmap(3): int key (rejected) + empty bin8 + try: + up.unpack() # ValueError: int is not allowed for map key ... + except Exception: + pass + up.skip() # SIGSEGV (resumes from a corrupt parser context) From 448d43f5dcca7b3a2b0810e161af0a6431d90071 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 00:33:29 +0900 Subject: [PATCH 52/55] release v1.2.1 (#698) Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- CHANGELOG.md | 9 +++++++++ README.md | 6 +++++- msgpack/__init__.py | 4 ++-- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a29b4b2f..b8fbd8ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# 1.2.1 + +Release Date: 2026-06-19 + +Fix a segfault when calling `Unpacker.unpack()` or `Unpacker.skip()` after an unpacking failure. +But note that reusing the same `Unpacker` instance after an unpacking failure is not supported. +Please create a new `Unpacker` instance instead. GHSA-6v7p-g79w-8964 + + # 1.2.0 Release Date: 2026-06-11 diff --git a/README.md b/README.md index 223742dd..e66d4549 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,10 @@ for unpacked in unpacker: print(unpacked) ``` +> [!IMPORTANT] +> If `Unpacker.unpack()` stops with an exception other than `OutOfData`, that `Unpacker` cannot be reused. +> Create a new `Unpacker` when reading another stream. + ### Packing/unpacking of custom data types @@ -220,7 +224,7 @@ When upgrading from msgpack-0.4 or earlier, do `pip uninstall msgpack-python` be * The extension module no longer supports Python 2. The pure Python implementation (`msgpack.fallback`) is used for Python 2. - + * msgpack 1.0.6 drops official support of Python 2.7, as pip and GitHub Action "setup-python" no longer supports Python 2.7. diff --git a/msgpack/__init__.py b/msgpack/__init__.py index f053abd0..eeb85afa 100644 --- a/msgpack/__init__.py +++ b/msgpack/__init__.py @@ -4,8 +4,8 @@ from .exceptions import * # noqa: F403 from .ext import ExtType, Timestamp -version = (1, 2, 0) -__version__ = "1.2.0" +version = (1, 2, 1) +__version__ = "1.2.1" if os.environ.get("MSGPACK_PUREPYTHON"): From f1a170234a95dd64d4c0dee398d16d4a011e90ea Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 18:49:02 +0900 Subject: [PATCH 53/55] fix docstring for read_size (#700) fix #697 --- .gitignore | 1 + msgpack/_unpacker.pyx | 2 +- msgpack/fallback.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 341be631..31d7d6e1 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ msgpack/*.cpp /tags /docs/_build .cache +uv.lock diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index e0cdde43..26fb377e 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -222,7 +222,7 @@ cdef class Unpacker: If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) + Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*. :param bool use_list: If true, unpack msgpack array to Python list. diff --git a/msgpack/fallback.py b/msgpack/fallback.py index 860d94ac..824f59d5 100644 --- a/msgpack/fallback.py +++ b/msgpack/fallback.py @@ -137,7 +137,7 @@ class Unpacker: If specified, unpacker reads serialized data from it and `.feed()` is not usable. :param int read_size: - Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`) + Used as `file_like.read(read_size)`. Must be equal to or smaller than *max_buffer_size*. :param bool use_list: If true, unpack msgpack array to Python list. From d13418061d00e10964394bda555247727f85ce28 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 20:48:27 +0900 Subject: [PATCH 54/55] unpacker: fix silent datetime truncation with datetime=3 (#701) --- msgpack/unpack.h | 9 ++++++++- test/test_timestamp.py | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/msgpack/unpack.h b/msgpack/unpack.h index 0f9ffc05..55cfdab9 100644 --- a/msgpack/unpack.h +++ b/msgpack/unpack.h @@ -283,6 +283,7 @@ static int unpack_timestamp(const char* buf, unsigned int buflen, msgpack_timest ts->tv_sec = _msgpack_load64(int64_t, buf + 4); return 0; default: + PyErr_Format(PyExc_ValueError, "invalid timestamp data (length %u)", buflen); return -1; } } @@ -336,12 +337,18 @@ static int unpack_callback_ext(unpack_user* u, const char* base, const char* pos else if (u->timestamp == 3) { // datetime // Calculate datetime using epoch + delta // due to limitations PyDateTime_FromTimestamp on Windows with negative timestamps + int64_t days = ts.tv_sec / (24*3600); + if (days < INT_MIN || days > INT_MAX) { + PyErr_Format(PyExc_OverflowError, + "days=%lld; too large to convert to C int", days); + return -1; + } PyObject *epoch = PyDateTimeAPI->DateTime_FromDateAndTime(1970, 1, 1, 0, 0, 0, 0, u->utc, PyDateTimeAPI->DateTimeType); if (epoch == NULL) { return -1; } - PyObject* d = PyDelta_FromDSU(ts.tv_sec/(24*3600), ts.tv_sec%(24*3600), ts.tv_nsec / 1000); + PyObject* d = PyDelta_FromDSU((int)days, ts.tv_sec%(24*3600), ts.tv_nsec / 1000); if (d == NULL) { Py_DECREF(epoch); return -1; diff --git a/test/test_timestamp.py b/test/test_timestamp.py index 6dfe3689..7c8e3e83 100644 --- a/test/test_timestamp.py +++ b/test/test_timestamp.py @@ -176,3 +176,9 @@ def test_pack_datetime_without_tzinfo(): packed = msgpack.packb(dt, datetime=True) unpacked = msgpack.unpackb(packed, timestamp=3) assert unpacked == dt + + +def test_too_large_timestamp(): + # When timestamp64 is too large, conversion to datetime fails due to int64 -> int32 conversion. + # https://github.com/msgpack/msgpack-python/issues/696 + print(msgpack.unpackb(b"\xd7\xff" + b"\x00" * 8, timestamp=3)) From 7082130739d640134e961a384d33ca9939e1f969 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 19 Jun 2026 21:45:06 +0900 Subject: [PATCH 55/55] run test with debug build (#702) --- .github/workflows/test_debug.yml | 62 ++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/test_debug.yml diff --git a/.github/workflows/test_debug.yml b/.github/workflows/test_debug.yml new file mode 100644 index 00000000..8aeec640 --- /dev/null +++ b/.github/workflows/test_debug.yml @@ -0,0 +1,62 @@ +name: Run tests with debug Python +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + env: + PYTHON_VERSION: 3.14.6 + steps: + - name: Checkout + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + + - uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: /opt/python-debug + key: python-debug-${{ runner.os }}-${{ env.PYTHON_VERSION }} + + - name: Install build dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: | + sudo apt-get update + sudo apt-get install -y build-essential libssl-dev zlib1g-dev \ + libbz2-dev libreadline-dev libsqlite3-dev curl libncursesw5-dev \ + xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev + + - name: Build Python with Py_DEBUG + if: steps.cache.outputs.cache-hit != 'true' + run: | + wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz + tar -xzf Python-${PYTHON_VERSION}.tgz + cd Python-${PYTHON_VERSION} + + ./configure --with-pydebug --prefix=/opt/python-debug + make -j4 + sudo make install + + - name: Create venv from debug Python + run: | + PYDBG_BIN="/opt/python-debug/bin/python3" + "$PYDBG_BIN" -m venv .venv + echo "$PWD/.venv/bin" >> "$GITHUB_PATH" + echo "VIRTUAL_ENV=$PWD/.venv" >> "$GITHUB_ENV" + + - name: Prepare + run: | + python -m pip install -r requirements.txt pytest + + - name: Build + run: | + make cython + pip install . + + - name: Test (C extension) + run: | + pytest -v test + + - name: Test (pure Python fallback) + run: | + MSGPACK_PUREPYTHON=1 pytest -v test