diff --git a/.github/workflows/test-and-publish.yaml b/.github/workflows/test-and-publish.yaml index 9456782e..c0261c67 100644 --- a/.github/workflows/test-and-publish.yaml +++ b/.github/workflows/test-and-publish.yaml @@ -5,7 +5,7 @@ on: branches: - main tags: - - '*' + - 'v*' workflow_call: workflow_dispatch: inputs: @@ -54,6 +54,10 @@ defaults: # run with Git Bash on Windows shell: bash +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: build-spidermonkey-unix: strategy: @@ -201,6 +205,17 @@ jobs: WORKFLOW_BUILD_TYPE=${{ inputs.build_type }} BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry build --format=wheel ls -lah ./dist/ + - name: Make the wheels we build also support lower versions of macOS + if: ${{ matrix.os == 'macos-12' || matrix.os == 'macos-14' }} + # Change the platform tag part of the wheel filename to `macosx_11_0_xxx` (means to support macOS 11.0 and above) + # See https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-format + # A wheel package file will only be selected by pip to install if the platform tag satisfies, regardless of whether the binary compatibility actually is. + # Otherwise, pip would fallback to compile from the source distribution. + run: | + cd ./dist/ + for file in *.whl; do + mv "$file" "$(echo "$file" | sed -E 's/macosx_[0-9]+_[0-9]+/macosx_11_0/')"; + done - name: Upload wheel as CI artifacts uses: actions/upload-artifact@v3 with: @@ -286,7 +301,7 @@ jobs: publish: needs: [build-and-test, sdist] runs-on: ubuntu-20.04 - if: ${{ success() && github.event_name == 'push' && contains(github.ref, 'refs/tags/') }} + if: ${{ success() && github.event_name == 'push' && github.ref_type == 'tag' }} steps: # no need to checkout - uses: actions/setup-python@v5 @@ -309,7 +324,7 @@ jobs: # and deploy the static files to GitHub Pages needs: [build-and-test, sdist] runs-on: ubuntu-20.04 - if: ${{ (success() || failure()) && github.ref_name == 'main' }} # publish nightly builds regardless of tests failure + if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }} # publish nightly builds regardless of tests failure permissions: # grant GITHUB_TOKEN the permissions required to make a Pages deployment pages: write id-token: write @@ -364,3 +379,45 @@ jobs: - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v2 + publish-archive: + # Publish to ⊇istributive's archive server (https://archive.distributed.computer/releases/pythonmonkey/) + needs: [build-and-test, sdist] + runs-on: ubuntu-20.04 + if: ${{ (success() || failure()) && (github.ref_name == 'main' || github.ref_type == 'tag') }} + environment: + name: archive + url: https://archive.distributed.computer/releases/pythonmonkey/${{ steps.get_path.outputs.ARCHIVE_PATH }} + steps: + # no need to checkout + - name: Download wheels built + uses: actions/download-artifact@v3 + with: + name: wheel-${{ github.run_id }}-${{ github.sha }} + path: ./ + - name: Download docs html generated by Doxygen + uses: actions/download-artifact@v3 + with: + name: docs-${{ github.run_id }}-${{ github.sha }} + path: ./docs/ + - name: Get the pythonmonkey/pminit version number + run: | + file=$(ls ./pminit*.tar.gz | head -1) + pm_version=$(basename "${file%.tar.gz}" | cut -d- -f2) # match /pminit-([^-]+).tar.gz/ + echo "PM_VERSION=$pm_version" >> $GITHUB_ENV + - name: Get the archive type (nightly or releases) and path + id: get_path + run: | + path="$ARCHIVE_TYPE/$PM_VERSION/" + echo "$path" + echo "ARCHIVE_PATH=$path" >> $GITHUB_OUTPUT + env: + ARCHIVE_TYPE: ${{ (github.ref_type == 'tag' && 'releases') || 'nightly' }} + - name: SCP to the archive server + uses: appleboy/scp-action@v0.1.7 + with: + host: ${{ secrets.ARCHIVE_HOST }} + username: pythonmonkey + key: ${{ secrets.ARCHIVE_KEY }} + source: ./* + target: archive/${{ steps.get_path.outputs.ARCHIVE_PATH }} + overwrite: true diff --git a/README.md b/README.md index 899a0090..1532ae6f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # PythonMonkey -![Testing Suite](https://github.com/Kings-Distributed-Systems/PythonMonkey/actions/workflows/tests.yaml/badge.svg) +[![Test and Publish Suite](https://github.com/Distributive-Network/PythonMonkey/actions/workflows/test-and-publish.yaml/badge.svg)](https://github.com/Distributive-Network/PythonMonkey/actions/workflows/test-and-publish.yaml) ## About [PythonMonkey](https://pythonmonkey.io) is a Mozilla [SpiderMonkey](https://firefox-source-docs.mozilla.org/js/index.html) JavaScript engine embedded into the Python Runtime, @@ -8,7 +8,7 @@ using the Python engine to provide the Javascript host environment. We feature JavaScript Array and Object methods implemented on Python List and Dictionaries using the cPython C API, and the inverse using the Mozilla Firefox Spidermonkey JavaScript C++ API. -This product is in an advanced stage, approximately 98% to MVP as of August 2024. It is under active development by [Distributive](https://distributive.network/). +This project has reached MVP as of September 2024. It is under maintenance by [Distributive](https://distributive.network/). External contributions and feedback are welcome and encouraged. @@ -73,7 +73,7 @@ Read this if you want to build a local version. - llvm - rust - python3.8 or later with header files (python3-dev) - - spidermonkey 115.1.0 or later + - spidermonkey latest from mozilla-central - npm (nodejs) - [Poetry](https://python-poetry.org/docs/#installation) - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning) @@ -136,6 +136,15 @@ $ poetry build --format=wheel ``` and install them by `pip install ./dist/*`. +## Uninstallation + +Installing `pythonmonkey` will also install the `pminit` package as a dependency. However, `pip uninstall`ing a package won't automatically remove its dependencies. +If you want to cleanly remove `pythonmonkey` from your system, do the following: + +```bash +$ pip uninstall pythonmonkey pminit +``` + ## Debugging Steps 1. [build the project locally](#build-instructions) @@ -462,7 +471,7 @@ List of commands: ```console $ pmjs -Welcome to PythonMonkey v0.4.0. +Welcome to PythonMonkey v1.0.0. Type ".help" for more information. > .python import sys > .python sys.path diff --git a/mozcentral.version b/mozcentral.version index 0efd2b63..9f4b8194 100644 --- a/mozcentral.version +++ b/mozcentral.version @@ -1 +1 @@ -8a28ad54f9f516c41ceddfa7ea32368fccf4a0eb \ No newline at end of file +a283127a5d0aa005c54d339e8ca27414b55f079b \ No newline at end of file diff --git a/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.d.ts b/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.d.ts index bdb159b0..2d8c4495 100644 --- a/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.d.ts +++ b/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.d.ts @@ -46,6 +46,9 @@ export declare function request( // callbacks for known exceptions onTimeoutError: (err: Error) => void, onNetworkError: (err: Error) => void, + // the debug logging function + /** See `pm.bootstrap.require("debug")` */ + debug: (selector: string) => ((...args: string[]) => void), ): Promise; /** diff --git a/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py b/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py index e3ea1824..ca716d88 100644 --- a/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py +++ b/python/pythonmonkey/builtin_modules/XMLHttpRequest-internal.py @@ -44,9 +44,10 @@ async def request( # callbacks for known exceptions onTimeoutError: Callable[[asyncio.TimeoutError], None], onNetworkError: Callable[[aiohttp.ClientError], None], + # the debug logging function, see `pm.bootstrap.require("debug")` + debug: Callable[[str], Callable[..., None]], / ): - debug = pm.bootstrap.require("debug") # to support HTTP-Keep-Alive global keepAliveConnector diff --git a/python/pythonmonkey/builtin_modules/XMLHttpRequest.js b/python/pythonmonkey/builtin_modules/XMLHttpRequest.js index 6db9bc04..3cbacfc1 100644 --- a/python/pythonmonkey/builtin_modules/XMLHttpRequest.js +++ b/python/pythonmonkey/builtin_modules/XMLHttpRequest.js @@ -20,7 +20,7 @@ const debug = globalThis.python.eval('__import__("pythonmonkey").bootstrap.requi * @param {any} what The thing to truncate; must have a slice method and index property. * Works with string, array, typedarray, etc. * @param {number} maxlen The maximum length for truncation - * @param {boolean} coerce Not false = coerce to printable character codes + * @param {boolean=} coerce Not false = coerce to printable character codes * @returns {string} */ function trunc(what, maxlen, coerce) @@ -104,6 +104,34 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget /** @type {EventListenerFn} */ onreadystatechange = null; + // + // debugging + // + /** The unique connection id to identify each XHR connection when debugging */ + #connectionId = Math.random().toString(16).slice(2, 9); // random 7-character hex string + + /** + * Wrapper to print debug logs with connection id information + * @param {string} selector + */ + #debug(selector) + { + return (...args) => debug(selector)(`Conn<${this.#connectionId}>:`, ...args); + } + + /** + * Allowing others to inspect the internal properties + */ + get _requestMetadata() + { + return { + method: this.#requestMethod, + url: this.#requestURL.toString(), + headers: this.#requestHeaders, + body: this.#requestBody, + }; + } + // // states // @@ -142,7 +170,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget */ open(method, url, async = true, username = null, password = null) { - debug('xhr:open')('open start, method=' + method); + this.#debug('xhr:open')('open start, method=' + method); // Normalize the method. // @ts-expect-error method = method.toString().toUpperCase(); @@ -156,7 +184,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget parsedURL.username = username; if (password) parsedURL.password = password; - debug('xhr:open')('url is ' + parsedURL.href); + this.#debug('xhr:open')('url is ' + parsedURL.href); // step 11 this.#sendFlag = false; @@ -176,7 +204,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget this.#state = XMLHttpRequest.OPENED; this.dispatchEvent(new Event('readystatechange')); } - debug('xhr:open')('finished open, state is ' + this.#state); + this.#debug('xhr:open')('finished open, state is ' + this.#state); } /** @@ -186,7 +214,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget */ setRequestHeader(name, value) { - debug('xhr:headers')(`set header ${name}=${value}`); + this.#debug('xhr:headers')(`set header ${name}=${value}`); if (this.#state !== XMLHttpRequest.OPENED) throw new DOMException('setRequestHeader can only be called when state is OPEN', 'InvalidStateError'); if (this.#sendFlag) @@ -252,7 +280,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget */ send(body = null) { - debug('xhr:send')(`sending; body length=${body?.length}`); + this.#debug('xhr:send')(`sending; body length=${body?.length} «${body ? trunc(body, 100) : ''}»`); if (this.#state !== XMLHttpRequest.OPENED) // step 1 throw new DOMException('connection must be opened before send() is called', 'InvalidStateError'); if (this.#sendFlag) // step 2 @@ -285,7 +313,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget if (!originalAuthorContentType && extractedContentType) this.#requestHeaders['content-type'] = extractedContentType; } - debug('xhr:send')(`content-type=${this.#requestHeaders['content-type']}`); + this.#debug('xhr:send')(`content-type=${this.#requestHeaders['content-type']}`); // step 5 if (this.#uploadObject._hasAnyListeners()) @@ -310,7 +338,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget */ #sendAsync() { - debug('xhr:send')('sending in async mode'); + this.#debug('xhr:send')('sending in async mode'); this.dispatchEvent(new ProgressEvent('loadstart', { loaded:0, total:0 })); // step 11.1 let requestBodyTransmitted = 0; // step 11.2 @@ -343,7 +371,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget let responseLength = 0; const processResponse = (response) => { - debug('xhr:response')(`response headers ----\n${response.getAllResponseHeaders()}`); + this.#debug('xhr:response')(`response headers ----\n${response.getAllResponseHeaders()}`); this.#response = response; // step 11.9.1 this.#state = XMLHttpRequest.HEADERS_RECEIVED; // step 11.9.4 this.dispatchEvent(new Event('readystatechange')); // step 11.9.5 @@ -354,7 +382,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget const processBodyChunk = (/** @type {Uint8Array} */ bytes) => { - debug('xhr:response')(`recv chunk, ${bytes.length} bytes (${trunc(bytes, 100)})`); + this.#debug('xhr:response')(`recv chunk, ${bytes.length} bytes «${trunc(bytes, 100)}»`); this.#receivedBytes.push(bytes); if (this.#state === XMLHttpRequest.HEADERS_RECEIVED) this.#state = XMLHttpRequest.LOADING; @@ -367,7 +395,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget */ const processEndOfBody = () => { - debug('xhr:response')(`end of body, received ${this.#receivedLength} bytes`); + this.#debug('xhr:response')(`end of body, received ${this.#receivedLength} bytes`); const transmitted = this.#receivedLength; // step 3 const length = responseLength || 0; // step 4 @@ -380,8 +408,8 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget this.dispatchEvent(new ProgressEvent(eventType, { loaded:transmitted, total:length })); }; - debug('xhr:send')(`${this.#requestMethod} ${this.#requestURL.href}`); - debug('xhr:headers')('headers=' + Object.entries(this.#requestHeaders)); + this.#debug('xhr:send')(`${this.#requestMethod} ${this.#requestURL.href}`); + this.#debug('xhr:headers')('headers=' + Object.entries(this.#requestHeaders)); // send() step 6 request( @@ -397,6 +425,7 @@ class XMLHttpRequest extends XMLHttpRequestEventTarget processEndOfBody, () => (this.#timedOutFlag = true), // onTimeoutError () => (this.#response = null /* network error */), // onNetworkError + this.#debug.bind(this), ).catch((e) => this.#handleErrors(e)); } diff --git a/src/PyObjectProxyHandler.cc b/src/PyObjectProxyHandler.cc index 565e6122..63e77dd2 100644 --- a/src/PyObjectProxyHandler.cc +++ b/src/PyObjectProxyHandler.cc @@ -44,8 +44,9 @@ bool PyObjectProxyHandler::handleGetOwnPropertyDescriptor(JSContext *cx, JS::Han JS::MutableHandle> desc, PyObject *item) { // see if we're calling a function if (id.isString()) { - JS::RootedString idString(cx, id.toString()); - const char *methodName = JS_EncodeStringToUTF8(cx, idString).get(); + JS::UniqueChars idString = JS_EncodeStringToUTF8(cx, JS::RootedString(cx, id.toString())); + const char *methodName = idString.get(); + if (!strcmp(methodName, "toString") || !strcmp(methodName, "toLocaleString") || !strcmp(methodName, "valueOf")) { JS::RootedObject objectPrototype(cx); if (!JS_GetClassPrototype(cx, JSProto_Object, &objectPrototype)) { diff --git a/src/modules/pythonmonkey/pythonmonkey.cc b/src/modules/pythonmonkey/pythonmonkey.cc index e1610a27..36a15d20 100644 --- a/src/modules/pythonmonkey/pythonmonkey.cc +++ b/src/modules/pythonmonkey/pythonmonkey.cc @@ -48,25 +48,39 @@ JS::PersistentRootedObject jsFunctionRegistry; +/** + * @brief During a GC, string buffers may have moved, so we need to re-point our JSStringProxies + * The char buffer pointer obtained by previous `JS::Get{Latin1,TwoByte}LinearStringChars` calls remains valid only as long as no GC occurs. + */ +void updateCharBufferPointers() { + if (_Py_IsFinalizing()) { + return; // do not move char pointers around if python is finalizing + } + + JS::AutoCheckCannotGC nogc; + for (const JSStringProxy *jsStringProxy: jsStringProxies) { + JSLinearString *str = JS_ASSERT_STRING_IS_LINEAR(jsStringProxy->jsString->toString()); + void *updatedCharBufPtr; // pointer to the moved char buffer after a GC + if (JS::LinearStringHasLatin1Chars(str)) { + updatedCharBufPtr = (void *)JS::GetLatin1LinearStringChars(nogc, str); + } else { // utf16 / ucs2 string + updatedCharBufPtr = (void *)JS::GetTwoByteLinearStringChars(nogc, str); + } + ((PyUnicodeObject *)(jsStringProxy))->data.any = updatedCharBufPtr; + } +} + void pythonmonkeyGCCallback(JSContext *cx, JSGCStatus status, JS::GCReason reason, void *data) { if (status == JSGCStatus::JSGC_END) { JS::ClearKeptObjects(GLOBAL_CX); while (JOB_QUEUE->runFinalizationRegistryCallbacks(GLOBAL_CX)); + updateCharBufferPointers(); + } +} - if (_Py_IsFinalizing()) { - return; // do not move char pointers around if python is finalizing - } - - JS::AutoCheckCannotGC nogc; - for (const JSStringProxy *jsStringProxy: jsStringProxies) { // char buffers may have moved, so we need to re-point our JSStringProxies - JSLinearString *str = (JSLinearString *)(jsStringProxy->jsString->toString()); // jsString is guaranteed to be linear - if (JS::LinearStringHasLatin1Chars(str)) { - (((PyUnicodeObject *)(jsStringProxy))->data.any) = (void *)JS::GetLatin1LinearStringChars(nogc, str); - } - else { // utf16 / ucs2 string - (((PyUnicodeObject *)(jsStringProxy))->data.any) = (void *)JS::GetTwoByteLinearStringChars(nogc, str); - } - } +void nurseryCollectionCallback(JSContext *cx, JS::GCNurseryProgress progress, JS::GCReason reason, void *data) { + if (progress == JS::GCNurseryProgress::GC_NURSERY_COLLECTION_END) { + updateCharBufferPointers(); } } @@ -565,6 +579,7 @@ PyMODINIT_FUNC PyInit_pythonmonkey(void) JS_SetGCParameter(GLOBAL_CX, JSGC_MAX_BYTES, (uint32_t)-1); JS_SetGCCallback(GLOBAL_CX, pythonmonkeyGCCallback, NULL); + JS::AddGCNurseryCollectionCallback(GLOBAL_CX, nurseryCollectionCallback, NULL); JS::RealmCreationOptions creationOptions = JS::RealmCreationOptions(); JS::RealmBehaviors behaviours = JS::RealmBehaviors();