From 7f8e45d3c2ef33999eacb1b16e93cc154ef09d08 Mon Sep 17 00:00:00 2001 From: Ryu Juheon Date: Sat, 11 Jan 2025 13:11:24 +0900 Subject: [PATCH 01/30] chore: addition of marker file according to pep561 --- pyproject.toml | 2 ++ python/pythonmonkey/py.typed | 1 + 2 files changed, 3 insertions(+) create mode 100644 python/pythonmonkey/py.typed diff --git a/pyproject.toml b/pyproject.toml index d69bc902..066ff5d4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,8 @@ include = [ { path = "CMakeLists.txt", format = "sdist" }, { path = "*.sh", format = "sdist" }, { path = "mozcentral.version", format = "sdist" }, + # Add marker file for pep561 + "python/pythonmonkey/py.typed", ] diff --git a/python/pythonmonkey/py.typed b/python/pythonmonkey/py.typed new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/python/pythonmonkey/py.typed @@ -0,0 +1 @@ + From 805996191732e270bd8a1153c2f594d0cbb4659e Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 4 Jun 2025 19:10:01 +0000 Subject: [PATCH 02/30] feat: clone SpiderMonkey source code from the official Firefox github repo --- setup.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.sh b/setup.sh index a3b23f0b..2001a374 100755 --- a/setup.sh +++ b/setup.sh @@ -44,8 +44,8 @@ echo "Done installing dependencies" echo "Downloading spidermonkey source code" # Read the commit hash for mozilla-central from the `mozcentral.version` file MOZCENTRAL_VERSION=$(cat mozcentral.version) -wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla/gecko-dev/archive/${MOZCENTRAL_VERSION}.zip -unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv gecko-dev-${MOZCENTRAL_VERSION} firefox-source +wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip +unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source echo "Done downloading spidermonkey source code" echo "Building spidermonkey" From 8a54b8433c0c710df61617719648f8830d593d08 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Wed, 4 Jun 2025 19:13:33 +0000 Subject: [PATCH 03/30] fix Firefox source git version https://github.com/mozilla-firefox/firefox/commit/6bca861985ba51920c1cacc21986af01c51bd690 is equivalent to https://github.com/mozilla/gecko-dev/commit/cdfe9f2e144a04618cb6e9ffd9e6202d6d85ed56 --- mozcentral.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mozcentral.version b/mozcentral.version index f72b9ef3..55aeecbf 100644 --- a/mozcentral.version +++ b/mozcentral.version @@ -1 +1 @@ -cdfe9f2e144a04618cb6e9ffd9e6202d6d85ed56 +6bca861985ba51920c1cacc21986af01c51bd690 From 42911524a6f27be220ddb4e8e656eaf2577fe80e Mon Sep 17 00:00:00 2001 From: Tom Wenzheng Tang Date: Tue, 17 Jun 2025 14:36:22 -0400 Subject: [PATCH 04/30] chore: fix mixed types in an array in pyproject.toml --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 066ff5d4..0cb6bdd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,8 +28,9 @@ include = [ { path = "CMakeLists.txt", format = "sdist" }, { path = "*.sh", format = "sdist" }, { path = "mozcentral.version", format = "sdist" }, + # Add marker file for pep561 - "python/pythonmonkey/py.typed", + { path = "python/pythonmonkey/py.typed", format = ["sdist", "wheel"] }, ] From 1021684b944cba1770a073b1c3760c13f47d72c8 Mon Sep 17 00:00:00 2001 From: Tom Wenzheng Tang Date: Tue, 17 Jun 2025 15:18:49 -0400 Subject: [PATCH 05/30] Revert "Write our own minimum copy of npm in Python to remove dependency on Node.js during installation" --- README.md | 1 + python/pminit/pmpm.py | 67 ------------------------------ python/pminit/post-install-hook.py | 42 +++++++++++++++---- python/pminit/pyproject.toml | 1 - 4 files changed, 35 insertions(+), 76 deletions(-) delete mode 100644 python/pminit/pmpm.py diff --git a/README.md b/README.md index 343a73fe..1532ae6f 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Read this if you want to build a local version. - rust - python3.8 or later with header files (python3-dev) - spidermonkey latest from mozilla-central + - npm (nodejs) - [Poetry](https://python-poetry.org/docs/#installation) - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning) diff --git a/python/pminit/pmpm.py b/python/pminit/pmpm.py deleted file mode 100644 index 59a78a14..00000000 --- a/python/pminit/pmpm.py +++ /dev/null @@ -1,67 +0,0 @@ -# @file pmpm.py -# A minimum copy of npm written in pure Python. -# Currently, this can only install dependencies specified by package-lock.json into node_modules. -# @author Tom Tang -# @date July 2023 - -import json -import io -import os, shutil -import tempfile -import tarfile -from dataclasses import dataclass -import urllib.request -from typing import List, Union - -@dataclass -class PackageItem: - installation_path: str - tarball_url: str - has_install_script: bool - -def parse_package_lock_json(json_data: Union[str, bytes]) -> List[PackageItem]: - # See https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages - packages: dict = json.loads(json_data)["packages"] - items: List[PackageItem] = [] - for key, entry in packages.items(): - if key == "": - # Skip the root project (listed with a key of "") - continue - items.append( - PackageItem( - installation_path=key, # relative path from the root project folder - # The path is flattened for nested node_modules, e.g., "node_modules/create-ecdh/node_modules/bn.js" - tarball_url=entry["resolved"], # TODO: handle git dependencies - has_install_script=entry.get("hasInstallScript", False) # the package has a preinstall, install, or postinstall script - ) - ) - return items - -def download_package(tarball_url: str) -> bytes: - with urllib.request.urlopen(tarball_url) as response: - tarball_data: bytes = response.read() - return tarball_data - -def unpack_package(work_dir:str, installation_path: str, tarball_data: bytes): - installation_path = os.path.join(work_dir, installation_path) - shutil.rmtree(installation_path, ignore_errors=True) - - with tempfile.TemporaryDirectory(prefix="pmpm_cache-") as tmpdir: - with io.BytesIO(tarball_data) as tar_file: - with tarfile.open(fileobj=tar_file) as tar: - tar.extractall(tmpdir) - shutil.move( - os.path.join(tmpdir, "package"), # Strip the root folder - installation_path - ) - -def main(work_dir: str): - with open(os.path.join(work_dir, "package-lock.json"), encoding="utf-8") as f: - items = parse_package_lock_json(f.read()) - for i in items: - print("Installing " + i.installation_path) - tarball_data = download_package(i.tarball_url) - unpack_package(work_dir, i.installation_path, tarball_data) - -if __name__ == "__main__": - main(os.getcwd()) diff --git a/python/pminit/post-install-hook.py b/python/pminit/post-install-hook.py index 061492c1..1b2d26dd 100644 --- a/python/pminit/post-install-hook.py +++ b/python/pminit/post-install-hook.py @@ -1,13 +1,39 @@ -import os -import pmpm +import subprocess +import sys +import shutil -WORK_DIR = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "pythonmonkey" -) +def execute(cmd: str): + popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, + shell = True, text = True ) + for stdout_line in iter(popen.stdout.readline, ""): + sys.stdout.write(stdout_line) + sys.stdout.flush() + + popen.stdout.close() + return_code = popen.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd) def main(): - pmpm.main(WORK_DIR) # cd pythonmonkey && npm i + node_package_manager = 'npm' + # check if npm is installed on the system + if (shutil.which(node_package_manager) is None): + print(""" + +PythonMonkey Build Error: + + + * It appears npm is not installed on this system. + * npm is required for PythonMonkey to build. + * Please install NPM and Node.js before installing PythonMonkey. + * Refer to the documentation for installing NPM and Node.js here: https://nodejs.org/en/download + + + """) + raise Exception("PythonMonkey build error: Unable to find npm on the system.") + else: + execute(f"cd pythonmonkey && {node_package_manager} i --no-package-lock") # do not update package-lock.json if __name__ == "__main__": - main() + main() + diff --git a/python/pminit/pyproject.toml b/python/pminit/pyproject.toml index 7c7b84b6..09d28174 100644 --- a/python/pminit/pyproject.toml +++ b/python/pminit/pyproject.toml @@ -11,7 +11,6 @@ documentation = "https://docs.pythonmonkey.io/" repository = "https://github.com/Distributive-Network/PythonMonkey" include = [ - "pmpm.py", # Install extra files into the pythonmonkey package "pythonmonkey/package*.json", { path = "pythonmonkey/node_modules/**/*", format = "wheel" }, From 29e7b8559bb152abe45381a149bfb361cdc90a29 Mon Sep 17 00:00:00 2001 From: Tom Wenzheng Tang Date: Sat, 12 Jul 2025 09:13:01 -0400 Subject: [PATCH 06/30] feat(CI): nodejs is required to build pminit --- .github/workflows/test-and-publish.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 337e85f6..ad81d8bd 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -154,6 +154,7 @@ jobs: apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev apt-get install -y git # required for `actions/checkout` + apt-get install -y nodejs npm # required for pminit to build apt-get install -y build-essential apt-get install -y strace # required to run JS tests DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata # tzdata may ask for user interaction if not explicitly installed here From c5b0ddaf17f055dcc4e860c43ddb98d621ef19f1 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Sat, 12 Jul 2025 13:57:10 +0000 Subject: [PATCH 07/30] feat(CI): upload spidermonkey build as CI artifacts --- .github/workflows/test-and-publish.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index ad81d8bd..305d7a34 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -83,7 +83,6 @@ jobs: path: | ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - lookup-only: true # skip download - name: Setup container if: ${{ matrix.os == 'ubuntu-22.04' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | @@ -102,6 +101,11 @@ jobs: - name: Build spidermonkey if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: ./setup.sh + - name: Upload spidermonkey build as CI artifacts + uses: actions/upload-artifact@v4 + with: + name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + path: ./_spidermonkey_install/ build-spidermonkey-win: runs-on: windows-2022 # SpiderMonkey requires Visual Studio 2022 or newer. @@ -117,7 +121,6 @@ jobs: path: | ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - lookup-only: true # skip download - name: Install dependencies if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} shell: powershell @@ -138,6 +141,11 @@ jobs: # see https://groups.google.com/u/1/a/mozilla.org/g/dev-platform/c/hF51Q3j6ca8 USE_MINTTY: 0 run: /c/mozilla-build/start-shell.bat -use-full-path -here ./setup.sh + - name: Upload spidermonkey build as CI artifacts + uses: actions/upload-artifact@v4 + with: + name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + path: ./_spidermonkey_install/ build-and-test: needs: [build-spidermonkey-unix, build-spidermonkey-win] strategy: From 6849db8606e964610587e51346507a54ee5a480e Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 09:26:01 -0400 Subject: [PATCH 08/30] fix the embedded null character issue SpiderMonkey doesn't store the extra null character while some Python APIs assume the string buffer is null-terminated. The issue hasn't been a problem before because it somehow didn't allocate the buffer that follows, making the string buffer null-terminated effectively. --- src/StrType.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/StrType.cc b/src/StrType.cc index 7df301bd..1c9ac772 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -135,6 +135,12 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { if (JS::LinearStringHasLatin1Chars(lstr)) { // latin1 spidermonkey, latin1 python const JS::Latin1Char *chars = JS::GetLatin1LinearStringChars(nogc, lstr); + if (chars[length] != 0) { // not a null-terminated string + // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy + PyObject *copied = PyUnicode_FromObject(pyString); // create a copy when it's not a true Unicode object + Py_DECREF(pyString); + return copied; + } PY_UNICODE_OBJECT_DATA_ANY(pyString) = (void *)chars; PY_UNICODE_OBJECT_KIND(pyString) = PyUnicode_1BYTE_KIND; @@ -157,6 +163,11 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { } else { // utf16 spidermonkey, ucs2 python const char16_t *chars = JS::GetTwoByteLinearStringChars(nogc, lstr); + if (chars[length] != 0) { // not a null-terminated string + PyObject *copied = PyUnicode_FromObject(pyString); + Py_DECREF(pyString); + return copied; + } PY_UNICODE_OBJECT_DATA_ANY(pyString) = (void *)chars; PY_UNICODE_OBJECT_KIND(pyString) = PyUnicode_2BYTE_KIND; From 79cd32b45538a17ef4320ab65f531a0723961dcc Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 09:43:27 -0400 Subject: [PATCH 09/30] chore(CI): setup Python using pyenv --- .github/workflows/test-and-publish.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 305d7a34..dd83c857 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -179,7 +179,20 @@ jobs: submodules: recursive fetch-depth: 0 # fetch all history for all branches and tags # poetry-dynamic-versioning needs git tags to produce the correct version number + - name: Setup Python + if: ${{ runner.os == 'Linux' }} + run: | + # Use pyenv to install Python version that is not available via `actions/setup-python` + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc + echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc + echo 'eval "$(pyenv init - bash)"' >> ~/.bashrc + pyenv install $PYTHON_VERSION + pyenv global $PYTHON_VERSION + env: + PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 + if: ${{ runner.os != 'Linux' }} with: python-version: ${{ matrix.python_version }} - name: Setup Poetry From f8efe5e824b9cda8b3ce330374152ffbd208e565 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 09:50:40 -0400 Subject: [PATCH 10/30] fix ci --- .github/workflows/test-and-publish.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index dd83c857..2d3c8823 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -187,6 +187,7 @@ jobs: echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc echo 'eval "$(pyenv init - bash)"' >> ~/.bashrc + source ~/.bashrc pyenv install $PYTHON_VERSION pyenv global $PYTHON_VERSION env: From 1449faef9e6b29ef7be5c2fe9a8d48e9c98bbc22 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 09:54:44 -0400 Subject: [PATCH 11/30] fix ci --- .github/workflows/test-and-publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 2d3c8823..42ed784f 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -183,11 +183,12 @@ jobs: if: ${{ runner.os == 'Linux' }} run: | # Use pyenv to install Python version that is not available via `actions/setup-python` + unset PYENV_ROOT curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc echo 'eval "$(pyenv init - bash)"' >> ~/.bashrc - source ~/.bashrc + export PATH="$HOME/.pyenv/bin:$PATH" pyenv install $PYTHON_VERSION pyenv global $PYTHON_VERSION env: From 36daa908d800804ed50580115c365f18d55e7923 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 10:12:53 -0400 Subject: [PATCH 12/30] fix CI Setup container --- .github/workflows/test-and-publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 42ed784f..2780ea1e 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -160,7 +160,8 @@ jobs: if: ${{ matrix.os == 'ubuntu-22.04' }} run: | apt-get update -y - apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y sudo curl libnss3-dev libssl-dev + apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv apt-get install -y git # required for `actions/checkout` apt-get install -y nodejs npm # required for pminit to build apt-get install -y build-essential From 25ea0eac367729b404a0ca748eab4ad1585b8658 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 10:28:10 -0400 Subject: [PATCH 13/30] fix ci --- .github/workflows/test-and-publish.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 2780ea1e..d2cda65e 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -160,8 +160,8 @@ jobs: if: ${{ matrix.os == 'ubuntu-22.04' }} run: | apt-get update -y - apt-get install -y sudo curl libnss3-dev libssl-dev - apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv + apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y curl python3 zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv apt-get install -y git # required for `actions/checkout` apt-get install -y nodejs npm # required for pminit to build apt-get install -y build-essential From ed90f47975c35b8c334df77abc36904f598c48c7 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 10:40:34 -0400 Subject: [PATCH 14/30] fix ci --- .github/workflows/test-and-publish.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index d2cda65e..8bb0a197 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -198,10 +198,15 @@ jobs: if: ${{ runner.os != 'Linux' }} with: python-version: ${{ matrix.python_version }} + - run: | + source ~/.bashrc || true # load pyenv into the current shell + python --version + python3 --version - name: Setup Poetry uses: snok/install-poetry@v1 with: version: 1.5.1 + virtualenvs-create: false - name: Install Dependencies run: | echo "Installing Dependencies" From 4858659582b1db8a33b9abd5c2af4ebacbe1ac62 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 11:05:55 -0400 Subject: [PATCH 15/30] bashrc file is not read, so we need to add to GITHUB_PATH manually --- .github/workflows/test-and-publish.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 8bb0a197..1ec1896d 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -161,7 +161,7 @@ jobs: run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev - apt-get install -y curl python3 zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv + apt-get install -y curl zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv apt-get install -y git # required for `actions/checkout` apt-get install -y nodejs npm # required for pminit to build apt-get install -y build-essential @@ -186,9 +186,9 @@ jobs: # Use pyenv to install Python version that is not available via `actions/setup-python` unset PYENV_ROOT curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash - echo 'export PYENV_ROOT="$HOME/.pyenv"' >> ~/.bashrc - echo '[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"' >> ~/.bashrc - echo 'eval "$(pyenv init - bash)"' >> ~/.bashrc + echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually + echo "$HOME/.pyenv/shims" >> $GITHUB_PATH + echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV export PATH="$HOME/.pyenv/bin:$PATH" pyenv install $PYTHON_VERSION pyenv global $PYTHON_VERSION @@ -199,14 +199,12 @@ jobs: with: python-version: ${{ matrix.python_version }} - run: | - source ~/.bashrc || true # load pyenv into the current shell python --version python3 --version - name: Setup Poetry uses: snok/install-poetry@v1 with: version: 1.5.1 - virtualenvs-create: false - name: Install Dependencies run: | echo "Installing Dependencies" From 79e00fcbde4c41f4d5874f0a2503ce7005bf1b4b Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 11:38:10 -0400 Subject: [PATCH 16/30] try running a Ubuntu 20.04 container on arm64 Linux --- .github/workflows/test-and-publish.yaml | 28 ++++++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 1ec1896d..b97496c6 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -71,7 +71,7 @@ jobs: # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available python_version: [ '3.10' ] runs-on: ${{ matrix.os }} - container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build + container: ${{ ( runner.os == 'Linux' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build steps: - uses: actions/checkout@v4 - name: Read the mozilla-central commit hash to be used @@ -84,14 +84,29 @@ jobs: ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - name: Setup container - if: ${{ matrix.os == 'ubuntu-22.04' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ runner.os == 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y curl zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 + - name: Setup Python + if: ${{ runner.os == 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + run: | + # Use pyenv to install Python version that is not available via `actions/setup-python` + unset PYENV_ROOT + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually + echo "$HOME/.pyenv/shims" >> $GITHUB_PATH + echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV + export PATH="$HOME/.pyenv/bin:$PATH" + pyenv install $PYTHON_VERSION + pyenv global $PYTHON_VERSION + env: + PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 - if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ runner.os != 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} with: python-version: ${{ matrix.python_version }} - name: Setup XCode @@ -154,10 +169,10 @@ jobs: os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ] python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} - container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} + container: ${{ (runner.os == 'Linux' && 'ubuntu:20.04') || null }} steps: - name: Setup container - if: ${{ matrix.os == 'ubuntu-22.04' }} + if: ${{ runner.os == 'Linux' }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev @@ -198,9 +213,6 @@ jobs: if: ${{ runner.os != 'Linux' }} with: python-version: ${{ matrix.python_version }} - - run: | - python --version - python3 --version - name: Setup Poetry uses: snok/install-poetry@v1 with: From f2c84429f721e96f9b4f76e9f95f90ec0038ee73 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 11:42:39 -0400 Subject: [PATCH 17/30] fix ci container --- .github/workflows/test-and-publish.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index b97496c6..6705480a 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -71,7 +71,7 @@ jobs: # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available python_version: [ '3.10' ] runs-on: ${{ matrix.os }} - container: ${{ ( runner.os == 'Linux' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build + container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build steps: - uses: actions/checkout@v4 - name: Read the mozilla-central commit hash to be used @@ -84,7 +84,7 @@ jobs: ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - name: Setup container - if: ${{ runner.os == 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev @@ -92,7 +92,7 @@ jobs: DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 - name: Setup Python - if: ${{ runner.os == 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | # Use pyenv to install Python version that is not available via `actions/setup-python` unset PYENV_ROOT @@ -106,7 +106,7 @@ jobs: env: PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 - if: ${{ runner.os != 'Linux' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ !startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} with: python-version: ${{ matrix.python_version }} - name: Setup XCode @@ -169,10 +169,10 @@ jobs: os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ] python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} - container: ${{ (runner.os == 'Linux' && 'ubuntu:20.04') || null }} + container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }} steps: - name: Setup container - if: ${{ runner.os == 'Linux' }} + if: ${{ startsWith(matrix.os, 'ubuntu') }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev @@ -196,7 +196,7 @@ jobs: fetch-depth: 0 # fetch all history for all branches and tags # poetry-dynamic-versioning needs git tags to produce the correct version number - name: Setup Python - if: ${{ runner.os == 'Linux' }} + if: ${{ startsWith(matrix.os, 'ubuntu') }} run: | # Use pyenv to install Python version that is not available via `actions/setup-python` unset PYENV_ROOT @@ -210,7 +210,7 @@ jobs: env: PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 - if: ${{ runner.os != 'Linux' }} + if: ${{ !startsWith(matrix.os, 'ubuntu') }} with: python-version: ${{ matrix.python_version }} - name: Setup Poetry From 150eea21bd31ed194e54116c620575b4bc9f42b1 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 11:46:50 -0400 Subject: [PATCH 18/30] fix ci --- .github/workflows/test-and-publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 6705480a..c7a8d27e 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -88,7 +88,8 @@ jobs: run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev - apt-get install -y curl zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv + apt-get install -y curl make git build-essential + apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 - name: Setup Python From 33b8765ccff6999dc34cbf7383553f85a6686959 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 13:21:19 -0400 Subject: [PATCH 19/30] chore(CI): install LLVM 18 --- .github/workflows/test-and-publish.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index c7a8d27e..d9d726c5 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -92,6 +92,12 @@ jobs: apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 + - name: Setup LLVM + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + run: | + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh 18 # install LLVM version 18 - name: Setup Python if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | From 120288848facc53bd72be8d57b9286bce8719643 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 13:27:21 -0400 Subject: [PATCH 20/30] fix ci --- .github/workflows/test-and-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index d9d726c5..5669b64a 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -88,7 +88,7 @@ jobs: run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev - apt-get install -y curl make git build-essential + apt-get install -y curl wget make git build-essential apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 From ed68856615fabf204d9c1e2e0528a443841750cf Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 13:40:15 -0400 Subject: [PATCH 21/30] fix ci --- .github/workflows/test-and-publish.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 5669b64a..7fe65f59 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -88,13 +88,14 @@ jobs: run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev - apt-get install -y curl wget make git build-essential + apt-get install -y curl make git build-essential apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 - name: Setup LLVM if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | + apt-get install -y lsb-release wget wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh ./llvm.sh 18 # install LLVM version 18 From a030d747d0b5da6b105dcb4aee403251b2c76741 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 13:43:26 -0400 Subject: [PATCH 22/30] fix ci --- .github/workflows/test-and-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 7fe65f59..e173f71a 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -95,7 +95,7 @@ jobs: - name: Setup LLVM if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | - apt-get install -y lsb-release wget + apt-get install -y lsb-release wget software-properties-common gnupg wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh ./llvm.sh 18 # install LLVM version 18 From d20db24a9dee8a1fca7bbf83aded07cb5fb9b29d Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 14:06:00 -0400 Subject: [PATCH 23/30] fix ci --- .github/workflows/test-and-publish.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index e173f71a..30997e80 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -96,9 +96,10 @@ jobs: if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | apt-get install -y lsb-release wget software-properties-common gnupg - wget https://apt.llvm.org/llvm.sh - chmod +x llvm.sh - ./llvm.sh 18 # install LLVM version 18 + echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | tee -a /etc/apt/sources.list.d/llvm.list + echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-defaults main" | tee -a /etc/apt/sources.list.d/llvm.list + apt-get update -y + apt-get install -y llvm-defaults clang - name: Setup Python if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | From a6b7696657b1add720bca981701dcd9553e3490d Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 14:43:42 -0400 Subject: [PATCH 24/30] fix ci --- .github/workflows/test-and-publish.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 30997e80..89354836 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -96,10 +96,12 @@ jobs: if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | apt-get install -y lsb-release wget software-properties-common gnupg - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-toolchain-$(lsb_release -cs) main" | tee -a /etc/apt/sources.list.d/llvm.list - echo "deb http://apt.llvm.org/$(lsb_release -cs)/ llvm-defaults main" | tee -a /etc/apt/sources.list.d/llvm.list - apt-get update -y - apt-get install -y llvm-defaults clang + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh 18 # install LLVM version 18 + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18 + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 - name: Setup Python if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | From acade44b8c6adb3931cae7017593b9fcb944f93e Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 16:36:20 -0400 Subject: [PATCH 25/30] fix ci --- .github/workflows/test-and-publish.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 89354836..9ba24ba2 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -95,6 +95,7 @@ jobs: - name: Setup LLVM if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | + apt-get install -y clang clang++ apt-get install -y lsb-release wget software-properties-common gnupg wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh @@ -102,6 +103,8 @@ jobs: update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18 update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 + clang --version + clang++ --version - name: Setup Python if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | From a4987171706eedccf1020f51dd2ea57bbbb3e6e4 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Mon, 14 Jul 2025 17:04:22 -0400 Subject: [PATCH 26/30] fix ci --- .github/workflows/test-and-publish.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 9ba24ba2..109561f8 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -95,7 +95,7 @@ jobs: - name: Setup LLVM if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | - apt-get install -y clang clang++ + apt-get install -y llvm clang apt-get install -y lsb-release wget software-properties-common gnupg wget https://apt.llvm.org/llvm.sh chmod +x llvm.sh From d3ea2ecbbbfbd4b1c8171e734afb45332f89cdec Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 18 Jul 2025 11:29:39 -0400 Subject: [PATCH 27/30] fix(string): short path to fix the `embedded null character` issue with Python 3.13+ --- src/StrType.cc | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/StrType.cc b/src/StrType.cc index 1c9ac772..32323fba 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -135,9 +135,11 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { if (JS::LinearStringHasLatin1Chars(lstr)) { // latin1 spidermonkey, latin1 python const JS::Latin1Char *chars = JS::GetLatin1LinearStringChars(nogc, lstr); - if (chars[length] != 0) { // not a null-terminated string - // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy - PyObject *copied = PyUnicode_FromObject(pyString); // create a copy when it's not a true Unicode object + if (Py_Version >= 0x030d0000) { // Python version is greater than 3.13 + // Short path to temporarily fix the issue with Python 3.13+ compact unicode representation. + // It would error with `ValueError: embedded null character`, which is caused by the fact that + // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy. + PyObject *copied = PyUnicode_FromObject((PyObject *)pyString); // create a copy when it's not a true Unicode object Py_DECREF(pyString); return copied; } @@ -163,8 +165,8 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { } else { // utf16 spidermonkey, ucs2 python const char16_t *chars = JS::GetTwoByteLinearStringChars(nogc, lstr); - if (chars[length] != 0) { // not a null-terminated string - PyObject *copied = PyUnicode_FromObject(pyString); + if (Py_Version >= 0x030d0000) { // Python 3.13+, see above + PyObject *copied = PyUnicode_FromObject((PyObject *)pyString); // create a copy when it's not a true Unicode object Py_DECREF(pyString); return copied; } From bf049a7fed7c462b623738bb18907e3264af0d22 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Fri, 18 Jul 2025 11:40:53 -0400 Subject: [PATCH 28/30] fix(string): `Py_Version` is only available on Python 3.11+ --- src/StrType.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/StrType.cc b/src/StrType.cc index 32323fba..0c384d51 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -135,7 +135,7 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { if (JS::LinearStringHasLatin1Chars(lstr)) { // latin1 spidermonkey, latin1 python const JS::Latin1Char *chars = JS::GetLatin1LinearStringChars(nogc, lstr); - if (Py_Version >= 0x030d0000) { // Python version is greater than 3.13 + if ((PY_VERSION_HEX) >= 0x030d0000) { // Python version is greater than 3.13 // Short path to temporarily fix the issue with Python 3.13+ compact unicode representation. // It would error with `ValueError: embedded null character`, which is caused by the fact that // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy. @@ -165,7 +165,7 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { } else { // utf16 spidermonkey, ucs2 python const char16_t *chars = JS::GetTwoByteLinearStringChars(nogc, lstr); - if (Py_Version >= 0x030d0000) { // Python 3.13+, see above + if ((PY_VERSION_HEX) >= 0x030d0000) { // Python 3.13+, see above PyObject *copied = PyUnicode_FromObject((PyObject *)pyString); // create a copy when it's not a true Unicode object Py_DECREF(pyString); return copied; From adb18fbce3dda1f54f70b84082cbd2be811ee029 Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Sat, 19 Jul 2025 07:38:11 -0400 Subject: [PATCH 29/30] fix(string): short path to convert strings --- src/StrType.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/StrType.cc b/src/StrType.cc index 0c384d51..18ecce09 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -139,7 +139,7 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { // Short path to temporarily fix the issue with Python 3.13+ compact unicode representation. // It would error with `ValueError: embedded null character`, which is caused by the fact that // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy. - PyObject *copied = PyUnicode_FromObject((PyObject *)pyString); // create a copy when it's not a true Unicode object + PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, chars, length); Py_DECREF(pyString); return copied; } @@ -166,7 +166,7 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { else { // utf16 spidermonkey, ucs2 python const char16_t *chars = JS::GetTwoByteLinearStringChars(nogc, lstr); if ((PY_VERSION_HEX) >= 0x030d0000) { // Python 3.13+, see above - PyObject *copied = PyUnicode_FromObject((PyObject *)pyString); // create a copy when it's not a true Unicode object + PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, chars, length); Py_DECREF(pyString); return copied; } From e3410eddd0e92a13ec0408d20e5a18c77851405b Mon Sep 17 00:00:00 2001 From: Tom Tang Date: Sat, 19 Jul 2025 09:34:05 -0400 Subject: [PATCH 30/30] fix(string): fix utf16 strings that contain surrogate pairs --- src/StrType.cc | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/StrType.cc b/src/StrType.cc index 18ecce09..a5c65788 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -165,11 +165,6 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { } else { // utf16 spidermonkey, ucs2 python const char16_t *chars = JS::GetTwoByteLinearStringChars(nogc, lstr); - if ((PY_VERSION_HEX) >= 0x030d0000) { // Python 3.13+, see above - PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, chars, length); - Py_DECREF(pyString); - return copied; - } PY_UNICODE_OBJECT_DATA_ANY(pyString) = (void *)chars; PY_UNICODE_OBJECT_KIND(pyString) = PyUnicode_2BYTE_KIND; @@ -202,6 +197,11 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { Py_DECREF(pyString); return ucs4Obj; } + if ((PY_VERSION_HEX) >= 0x030d0000) { // Python 3.13+, fix `ValueError: embedded null character` + PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, chars, length); // create a copy of the string buffer + Py_DECREF(pyString); + return copied; + } } return (PyObject *)pyString;