diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 337e85f6..109561f8 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -71,7 +71,7 @@ jobs: # see https://github.blog/changelog/2024-01-30-github-actions-macos-14-sonoma-is-now-available python_version: [ '3.10' ] runs-on: ${{ matrix.os }} - container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build + container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }} # Use the Ubuntu 20.04 container inside Ubuntu 22.04 runner to build steps: - uses: actions/checkout@v4 - name: Read the mozilla-central commit hash to be used @@ -83,16 +83,44 @@ jobs: path: | ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - lookup-only: true # skip download - name: Setup container - if: ${{ matrix.os == 'ubuntu-22.04' && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y curl make git build-essential + apt-get install -y zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata echo "AGENT_TOOLSDIRECTORY=/" >> $GITHUB_ENV # do not use the Python installation cached for Ubuntu 22.04 + - name: Setup LLVM + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + run: | + apt-get install -y llvm clang + apt-get install -y lsb-release wget software-properties-common gnupg + wget https://apt.llvm.org/llvm.sh + chmod +x llvm.sh + ./llvm.sh 18 # install LLVM version 18 + update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18 + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18 + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18 + clang --version + clang++ --version + - name: Setup Python + if: ${{ startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + run: | + # Use pyenv to install Python version that is not available via `actions/setup-python` + unset PYENV_ROOT + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually + echo "$HOME/.pyenv/shims" >> $GITHUB_PATH + echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV + export PATH="$HOME/.pyenv/bin:$PATH" + pyenv install $PYTHON_VERSION + pyenv global $PYTHON_VERSION + env: + PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 - if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} + if: ${{ !startsWith(matrix.os, 'ubuntu') && steps.cache-spidermonkey.outputs.cache-hit != 'true' }} with: python-version: ${{ matrix.python_version }} - name: Setup XCode @@ -102,6 +130,11 @@ jobs: - name: Build spidermonkey if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} run: ./setup.sh + - name: Upload spidermonkey build as CI artifacts + uses: actions/upload-artifact@v4 + with: + name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + path: ./_spidermonkey_install/ build-spidermonkey-win: runs-on: windows-2022 # SpiderMonkey requires Visual Studio 2022 or newer. @@ -117,7 +150,6 @@ jobs: path: | ./_spidermonkey_install/* key: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} - lookup-only: true # skip download - name: Install dependencies if: ${{ steps.cache-spidermonkey.outputs.cache-hit != 'true' }} shell: powershell @@ -138,6 +170,11 @@ jobs: # see https://groups.google.com/u/1/a/mozilla.org/g/dev-platform/c/hF51Q3j6ca8 USE_MINTTY: 0 run: /c/mozilla-build/start-shell.bat -use-full-path -here ./setup.sh + - name: Upload spidermonkey build as CI artifacts + uses: actions/upload-artifact@v4 + with: + name: spidermonkey-${{ env.MOZCENTRAL_VERSION }}-${{ runner.os }}-${{ runner.arch }} + path: ./_spidermonkey_install/ build-and-test: needs: [build-spidermonkey-unix, build-spidermonkey-win] strategy: @@ -146,14 +183,16 @@ jobs: os: [ 'ubuntu-22.04', 'macos-13', 'macos-14', 'windows-2022', 'ubuntu-22.04-arm' ] python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12', '3.13' ] runs-on: ${{ matrix.os }} - container: ${{ (matrix.os == 'ubuntu-22.04' && 'ubuntu:20.04') || null }} + container: ${{ (startsWith(matrix.os, 'ubuntu') && 'ubuntu:20.04') || null }} steps: - name: Setup container - if: ${{ matrix.os == 'ubuntu-22.04' }} + if: ${{ startsWith(matrix.os, 'ubuntu') }} run: | apt-get update -y apt-get install -y sudo libnss3-dev libssl-dev + apt-get install -y curl zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev libffi-dev liblzma-dev # required for pyenv apt-get install -y git # required for `actions/checkout` + apt-get install -y nodejs npm # required for pminit to build apt-get install -y build-essential apt-get install -y strace # required to run JS tests DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata # tzdata may ask for user interaction if not explicitly installed here @@ -170,7 +209,22 @@ jobs: submodules: recursive fetch-depth: 0 # fetch all history for all branches and tags # poetry-dynamic-versioning needs git tags to produce the correct version number + - name: Setup Python + if: ${{ startsWith(matrix.os, 'ubuntu') }} + run: | + # Use pyenv to install Python version that is not available via `actions/setup-python` + unset PYENV_ROOT + curl -L https://github.com/pyenv/pyenv-installer/raw/master/bin/pyenv-installer | bash + echo "$HOME/.pyenv/bin" >> $GITHUB_PATH # ~/.bashrc file is not read, so we need to add to GITHUB_PATH manually + echo "$HOME/.pyenv/shims" >> $GITHUB_PATH + echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV + export PATH="$HOME/.pyenv/bin:$PATH" + pyenv install $PYTHON_VERSION + pyenv global $PYTHON_VERSION + env: + PYTHON_VERSION: ${{ matrix.python_version }} - uses: actions/setup-python@v5 + if: ${{ !startsWith(matrix.os, 'ubuntu') }} with: python-version: ${{ matrix.python_version }} - name: Setup Poetry diff --git a/README.md b/README.md index 343a73fe..1532ae6f 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,7 @@ Read this if you want to build a local version. - rust - python3.8 or later with header files (python3-dev) - spidermonkey latest from mozilla-central + - npm (nodejs) - [Poetry](https://python-poetry.org/docs/#installation) - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning) diff --git a/mozcentral.version b/mozcentral.version index f72b9ef3..55aeecbf 100644 --- a/mozcentral.version +++ b/mozcentral.version @@ -1 +1 @@ -cdfe9f2e144a04618cb6e9ffd9e6202d6d85ed56 +6bca861985ba51920c1cacc21986af01c51bd690 diff --git a/pyproject.toml b/pyproject.toml index d69bc902..0cb6bdd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,9 @@ include = [ { path = "CMakeLists.txt", format = "sdist" }, { path = "*.sh", format = "sdist" }, { path = "mozcentral.version", format = "sdist" }, + + # Add marker file for pep561 + { path = "python/pythonmonkey/py.typed", format = ["sdist", "wheel"] }, ] diff --git a/python/pminit/pmpm.py b/python/pminit/pmpm.py deleted file mode 100644 index 59a78a14..00000000 --- a/python/pminit/pmpm.py +++ /dev/null @@ -1,67 +0,0 @@ -# @file pmpm.py -# A minimum copy of npm written in pure Python. -# Currently, this can only install dependencies specified by package-lock.json into node_modules. -# @author Tom Tang -# @date July 2023 - -import json -import io -import os, shutil -import tempfile -import tarfile -from dataclasses import dataclass -import urllib.request -from typing import List, Union - -@dataclass -class PackageItem: - installation_path: str - tarball_url: str - has_install_script: bool - -def parse_package_lock_json(json_data: Union[str, bytes]) -> List[PackageItem]: - # See https://docs.npmjs.com/cli/v9/configuring-npm/package-lock-json#packages - packages: dict = json.loads(json_data)["packages"] - items: List[PackageItem] = [] - for key, entry in packages.items(): - if key == "": - # Skip the root project (listed with a key of "") - continue - items.append( - PackageItem( - installation_path=key, # relative path from the root project folder - # The path is flattened for nested node_modules, e.g., "node_modules/create-ecdh/node_modules/bn.js" - tarball_url=entry["resolved"], # TODO: handle git dependencies - has_install_script=entry.get("hasInstallScript", False) # the package has a preinstall, install, or postinstall script - ) - ) - return items - -def download_package(tarball_url: str) -> bytes: - with urllib.request.urlopen(tarball_url) as response: - tarball_data: bytes = response.read() - return tarball_data - -def unpack_package(work_dir:str, installation_path: str, tarball_data: bytes): - installation_path = os.path.join(work_dir, installation_path) - shutil.rmtree(installation_path, ignore_errors=True) - - with tempfile.TemporaryDirectory(prefix="pmpm_cache-") as tmpdir: - with io.BytesIO(tarball_data) as tar_file: - with tarfile.open(fileobj=tar_file) as tar: - tar.extractall(tmpdir) - shutil.move( - os.path.join(tmpdir, "package"), # Strip the root folder - installation_path - ) - -def main(work_dir: str): - with open(os.path.join(work_dir, "package-lock.json"), encoding="utf-8") as f: - items = parse_package_lock_json(f.read()) - for i in items: - print("Installing " + i.installation_path) - tarball_data = download_package(i.tarball_url) - unpack_package(work_dir, i.installation_path, tarball_data) - -if __name__ == "__main__": - main(os.getcwd()) diff --git a/python/pminit/post-install-hook.py b/python/pminit/post-install-hook.py index 061492c1..1b2d26dd 100644 --- a/python/pminit/post-install-hook.py +++ b/python/pminit/post-install-hook.py @@ -1,13 +1,39 @@ -import os -import pmpm +import subprocess +import sys +import shutil -WORK_DIR = os.path.join( - os.path.realpath(os.path.dirname(__file__)), - "pythonmonkey" -) +def execute(cmd: str): + popen = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, + shell = True, text = True ) + for stdout_line in iter(popen.stdout.readline, ""): + sys.stdout.write(stdout_line) + sys.stdout.flush() + + popen.stdout.close() + return_code = popen.wait() + if return_code: + raise subprocess.CalledProcessError(return_code, cmd) def main(): - pmpm.main(WORK_DIR) # cd pythonmonkey && npm i + node_package_manager = 'npm' + # check if npm is installed on the system + if (shutil.which(node_package_manager) is None): + print(""" + +PythonMonkey Build Error: + + + * It appears npm is not installed on this system. + * npm is required for PythonMonkey to build. + * Please install NPM and Node.js before installing PythonMonkey. + * Refer to the documentation for installing NPM and Node.js here: https://nodejs.org/en/download + + + """) + raise Exception("PythonMonkey build error: Unable to find npm on the system.") + else: + execute(f"cd pythonmonkey && {node_package_manager} i --no-package-lock") # do not update package-lock.json if __name__ == "__main__": - main() + main() + diff --git a/python/pminit/pyproject.toml b/python/pminit/pyproject.toml index 7c7b84b6..09d28174 100644 --- a/python/pminit/pyproject.toml +++ b/python/pminit/pyproject.toml @@ -11,7 +11,6 @@ documentation = "https://docs.pythonmonkey.io/" repository = "https://github.com/Distributive-Network/PythonMonkey" include = [ - "pmpm.py", # Install extra files into the pythonmonkey package "pythonmonkey/package*.json", { path = "pythonmonkey/node_modules/**/*", format = "wheel" }, diff --git a/python/pythonmonkey/py.typed b/python/pythonmonkey/py.typed new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/python/pythonmonkey/py.typed @@ -0,0 +1 @@ + diff --git a/setup.sh b/setup.sh index a3b23f0b..2001a374 100755 --- a/setup.sh +++ b/setup.sh @@ -44,8 +44,8 @@ echo "Done installing dependencies" echo "Downloading spidermonkey source code" # Read the commit hash for mozilla-central from the `mozcentral.version` file MOZCENTRAL_VERSION=$(cat mozcentral.version) -wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla/gecko-dev/archive/${MOZCENTRAL_VERSION}.zip -unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv gecko-dev-${MOZCENTRAL_VERSION} firefox-source +wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip +unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source echo "Done downloading spidermonkey source code" echo "Building spidermonkey" diff --git a/src/StrType.cc b/src/StrType.cc index 7df301bd..a5c65788 100644 --- a/src/StrType.cc +++ b/src/StrType.cc @@ -135,6 +135,14 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { if (JS::LinearStringHasLatin1Chars(lstr)) { // latin1 spidermonkey, latin1 python const JS::Latin1Char *chars = JS::GetLatin1LinearStringChars(nogc, lstr); + if ((PY_VERSION_HEX) >= 0x030d0000) { // Python version is greater than 3.13 + // Short path to temporarily fix the issue with Python 3.13+ compact unicode representation. + // It would error with `ValueError: embedded null character`, which is caused by the fact that + // most Python C APIs assume the string buffer is null-terminated, so we need to create a copy. + PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_1BYTE_KIND, chars, length); + Py_DECREF(pyString); + return copied; + } PY_UNICODE_OBJECT_DATA_ANY(pyString) = (void *)chars; PY_UNICODE_OBJECT_KIND(pyString) = PyUnicode_1BYTE_KIND; @@ -189,6 +197,11 @@ PyObject *StrType::proxifyString(JSContext *cx, JS::HandleValue strVal) { Py_DECREF(pyString); return ucs4Obj; } + if ((PY_VERSION_HEX) >= 0x030d0000) { // Python 3.13+, fix `ValueError: embedded null character` + PyObject *copied = PyUnicode_FromKindAndData(PyUnicode_2BYTE_KIND, chars, length); // create a copy of the string buffer + Py_DECREF(pyString); + return copied; + } } return (PyObject *)pyString;