Rebuild against current mozilla-central SpiderMonkey (157a1), enable SharedArrayBuffer/Atomics

.github/workflows/test-and-publish.yaml

@@ -100,10 +100,10 @@ jobs:
           apt-get install -y lsb-release wget software-properties-common gnupg
           wget https://apt.llvm.org/llvm.sh
           chmod +x llvm.sh
-          ./llvm.sh 18 # install LLVM version 18
-          update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-18 18
-          update-alternatives --install /usr/bin/clang clang /usr/bin/clang-18 18
-          update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-18 18
+          ./llvm.sh 19 # SpiderMonkey at the current mozcentral.version pin requires clang/llvm >= 19 (confirmed via its own configure error)
+          update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-19 19
+          update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 19
+          update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 19
           clang --version
           clang++ --version
       - name: Setup Python
@@ -238,9 +238,30 @@ jobs:
           if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
             sudo apt-get update -y
             sudo apt-get install -y cmake llvm
+            # SpiderMonkey's headers now require clang/llvm >= 19 and
+            # libstdc++ >= 10 to compile against (same requirement as the
+            # build-spidermonkey job) -- this job's default toolchain
+            # (ubuntu:20.04's stock gcc-9) doesn't meet it.
+            sudo apt-get install -y lsb-release wget software-properties-common gnupg
+            wget https://apt.llvm.org/llvm.sh
+            chmod +x llvm.sh
+            sudo ./llvm.sh 19
+            sudo update-alternatives --install /usr/bin/llvm-config llvm-config /usr/bin/llvm-config-19 19
+            sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 19
+            sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-19 19
+            sudo apt-get install -y libstdc++-10-dev
+            echo "CC=clang" >> $GITHUB_ENV
+            echo "CXX=clang++" >> $GITHUB_ENV
           elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
             brew update || true # allow failure
             brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
+            # Xcode's bundled clang is older than SpiderMonkey's own
+            # minimum (>=19) -- same fix as setup.sh's macOS branch. Pinned
+            # to llvm@19: the unversioned `llvm` formula (currently 23.x)
+            # has no bottle for Intel macOS or macOS 14, so it silently
+            # falls back to a multi-hour from-source build here.
+            brew install llvm@19
+            echo "PATH=$(brew --prefix llvm@19)/bin:$PATH" >> $GITHUB_ENV
           fi
           echo "Installing python deps"
           poetry self add "poetry-dynamic-versioning[plugin]"
@@ -255,10 +276,10 @@ jobs:
         run: |
             sudo apt-get install -y graphviz 
             # the newest version in Ubuntu 20.04 repository is 1.8.17, but we need Doxygen 1.9 series
-            wget -c -q https://www.doxygen.nl/files/doxygen-1.9.7.linux.bin.tar.gz
-            tar xf doxygen-1.9.7.linux.bin.tar.gz
-            cd doxygen-1.9.7 && sudo make install && cd -
-            rm -rf doxygen-1.9.7 doxygen-1.9.7.linux.bin.tar.gz
+            wget -c -q https://www.doxygen.nl/files/doxygen-1.15.0.linux.bin.tar.gz
+            tar xf doxygen-1.15.0.linux.bin.tar.gz
+            cd doxygen-1.15.0 && sudo make install && cd -
+            rm -rf doxygen-1.15.0 doxygen-1.15.0.linux.bin.tar.gz
             BUILD_DOCS=1 BUILD_TYPE=None poetry install
       - name: Upload Doxygen-generated docs as CI artifacts
         if: ${{ matrix.os == 'ubuntu-22.04' && matrix.python_version == '3.11' }}

CMakeLists.txt

@@ -30,7 +30,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   include(FetchContent)
 
   if (WIN32)
-    SET(COMPILE_FLAGS "/GR- /W0")
+    # This build bypasses moz.build, which normally defines XP_WIN; without
+    # it SpiderMonkey headers (e.g. PlatformMutex.h) take a POSIX path.
+    SET(COMPILE_FLAGS "/GR- /W0 /DXP_WIN")
 
     SET(OPTIMIZED "/O2")
     SET(UNOPTIMIZED "/Od")
@@ -39,7 +41,9 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     SET(PROFILE "/PROFILE")
     SET(ADDRESS_SANITIZE "/fsanitize=address /Oy-")
   else()
-    SET(COMPILE_FLAGS "-fno-rtti -Wno-invalid-offsetof")
+    # Same as -DXP_WIN above: without XP_UNIX, headers such as
+    # mfbt/UniquePtrExtensions.h hit their "Unsupported OS" #error.
+    SET(COMPILE_FLAGS "-fno-rtti -Wno-invalid-offsetof -DXP_UNIX")
 
     SET(OPTIMIZED "-Ofast -DNDEBUG")
     SET(UNOPTIMIZED "-O0")

include/JobQueue.hh

@@ -49,45 +49,37 @@ bool init(JSContext *cx);
  * If any error happens while generating the host defined data, this method
  * should set a pending exception to `cx` and return `false`.
  */
-bool getHostDefinedData(JSContext *cx, JS::MutableHandle<JSObject *> data) const override;
+bool getHostDefinedData(JSContext *cx, JS::MutableHandle<JSObject *> incumbentGlobal, JS::MutableHandle<JSObject *> data) const override;
 
 /**
- * @brief Enqueue a reaction job `job` for `promise`, which was allocated at
- * `allocationSite`. Provide `incumbentGlobal` as the incumbent global for
- * the reaction job's execution.
+ * @brief Ask the embedding for the host defined global to use when running
+ * a JS microtask.
  *
- * `promise` can be null if the promise is optimized out.
- * `promise` is guaranteed not to be optimized out if the promise has
- * non-default user-interaction flag.
+ * Not tracked by this embedding; null falls back to SpiderMonkey's default,
+ * as its own InternalJobQueue does.
  */
-bool enqueuePromiseJob(JSContext *cx, JS::HandleObject promise,
-  JS::HandleObject job, JS::HandleObject allocationSite,
-  JS::HandleObject incumbentGlobal) override;
+bool getHostDefinedGlobal(JSContext *cx, JS::MutableHandle<JSObject *> out) const override;
 
 /**
- * @brief Run all jobs in the queue. Running one job may enqueue others; continue to
- * run jobs until the queue is empty.
+ * @brief Pull every job SpiderMonkey has queued internally and forward each
+ * one to the Python event-loop.
+ *
+ * SpiderMonkey no longer pushes promise jobs to the embedding as they are
+ * created; it queues them and expects the embedder to drain them at microtask
+ * checkpoints via js::RunJobs(cx) (jsfriendapi.h), which calls this method.
+ * Every place PythonMonkey enters JS from Python must checkpoint afterwards,
+ * or promises settled there never run their reactions.
  *
  * Calling this method at the wrong time can break the web. The HTML spec
  * indicates exactly when the job queue should be drained (in HTML jargon,
  * when it should "perform a microtask checkpoint"), and doing so at other
  * times can incompatibly change the semantics of programs that use promises
  * or other microtask-based features.
- *
- * This method is called only via AutoDebuggerJobQueueInterruption, used by
- * the Debugger API implementation to ensure that the debuggee's job queue is
- * protected from the debugger's own activity. See the comments on
- * AutoDebuggerJobQueueInterruption.
  */
 void runJobs(JSContext *cx) override;
 
 /**
- * @return true if the job queue is empty, false otherwise.
- */
-bool empty() const override;
-
-/**
- * @return true if the job queue stopped draining, which results in `empty()` being false after `runJobs()`.
+ * @return true if the job queue stopped draining before it was empty.
  */
 bool isDrainingStopped() const override;
 
@@ -127,11 +119,31 @@ js::UniquePtr<JS::JobQueue::SavedJobQueue> saveJobQueue(JSContext *) override;
  * @brief The callback for dispatching an off-thread promise to the event loop
  *          see https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/public/Promise.h#l580
  *              https://hg.mozilla.org/releases/mozilla-esr102/file/tip/js/src/vm/OffThreadPromiseRuntimeState.cpp#l160
+ *
+ * Takes ownership of the Dispatchable (run via the public static
+ * Dispatchable::Run, since Dispatchable::run() is protected).
+ *
  * @param closure - closure, currently the javascript context
- * @param dispatchable - Pointer to the Dispatchable to be called
+ * @param dispatchable - the Dispatchable to be called; ownership transferred to this callback
  * @return not shutting down
  */
-static bool dispatchToEventLoop(void *closure, JS::Dispatchable *dispatchable);
+static bool dispatchToEventLoop(void *closure, js::UniquePtr<JS::Dispatchable> &&dispatchable);
+
+/**
+ * @brief The callback for dispatching an off-thread promise to the event
+ * loop after a delay.
+ *
+ * Always declines (returns false), which js/public/Promise.h permits when the
+ * embedding has no timeout manager. Only engine features needing a delayed
+ * off-thread callback (e.g. Atomics.waitAsync timeouts) are affected;
+ * setTimeout/setInterval use PyEventLoop::enqueueWithDelay instead.
+ *
+ * @param closure - closure, currently the javascript context
+ * @param dispatchable - the Dispatchable that would be called; ownership transferred to this callback
+ * @param delay - requested delay in milliseconds
+ * @return false (no timeout manager available)
+ */
+static bool delayedDispatchToEventLoop(void *closure, js::UniquePtr<JS::Dispatchable> &&dispatchable, uint32_t delay);
 
 /**
  * @brief The callback that gets invoked whenever a Promise is rejected without a rejection handler (uncaught/unhandled exception)

mozcentral.version

@@ -1 +1 @@
-6bca861985ba51920c1cacc21986af01c51bd690
+1704651e7d6c706fcb753adab577e0954d61cee0

setup.sh

@@ -15,41 +15,71 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then # Linux
   echo "Installing apt packages"
   $SUDO apt-get install --yes cmake llvm clang pkg-config m4 unzip \
     wget curl python3-dev
+  # SpiderMonkey's configure requires libstdc++ >= 10 (toolchain.configure,
+  # minimum_gcc_version). Ubuntu 20.04, which CI builds in for wheel
+  # compatibility, defaults to gcc-9's; the -10 headers are in its own repos
+  # and don't change which libstdc++.so the wheel links against.
+  $SUDO apt-get install --yes libstdc++-10-dev
 elif [[ "$OSTYPE" == "darwin"* ]]; then # macOS
   brew update || true # allow failure
   brew install cmake pkg-config wget unzip coreutils # `coreutils` installs the `realpath` command
-  brew install lld
-elif [[ "$OSTYPE" == "msys"* ]]; then # Windows
+  # SpiderMonkey requires clang >= 19 and lld as the host linker; Xcode's clang
+  # is older and ships no lld. Versioned formulas because the unversioned
+  # `llvm`/`lld` have no bottles for Intel macOS or macOS 14 and build from
+  # source for hours (and lld pulls in the unversioned llvm). Both kegs are
+  # keg-only, so put them on PATH explicitly.
+  brew install llvm@19 lld@19
+  export PATH="$(brew --prefix llvm@19)/bin:$(brew --prefix lld@19)/bin:$PATH"
+elif [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows
   echo "Dependencies are not going to be installed automatically on Windows."
 else
   echo "Unsupported OS"
   exit 1
 fi
-# Install rust compiler
-echo "Installing rust compiler"
-unset HOST_ABI_FLAGS
-if [[ "$OSTYPE" == "msys"* ]]; then # Windows
-  HOST_ABI_FLAGS=("--default-host" "$(clang --print-target-triple)")
+# Install rust compiler, skipping if the pinned toolchain is already the
+# default so this script can be re-run without re-downloading the installer.
+if command -v rustup >/dev/null && rustup default 2>/dev/null | grep -q '^1\.90'; then
+  echo "Rust 1.90 toolchain already installed, skipping rustup-init"
+else
+  echo "Installing rust compiler"
+  unset HOST_ABI_FLAGS
+  if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows
+    HOST_ABI_FLAGS=("--default-host" "$(clang --print-target-triple)")
+  fi
+  # SpiderMonkey's configure requires rustc >= 1.90.0
+  curl --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/rust-lang/rustup/refs/tags/1.28.2/rustup-init.sh -sSf | sh -s -- -y ${HOST_ABI_FLAGS+"${HOST_ABI_FLAGS[@]}"} --default-toolchain 1.90.0
 fi
-curl --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/rust-lang/rustup/refs/tags/1.28.2/rustup-init.sh -sSf | sh -s -- -y ${HOST_ABI_FLAGS+"${HOST_ABI_FLAGS[@]}"} --default-toolchain 1.85
 CARGO_BIN="$HOME/.cargo/bin/cargo" # also works for Windows. On Windows this equals to %USERPROFILE%\.cargo\bin\cargo
-$CARGO_BIN install cbindgen
+command -v cbindgen >/dev/null || $CARGO_BIN install cbindgen
 # Setup Poetry
-echo "Installing poetry"
-curl -sSL https://install.python-poetry.org | python3 - --version "1.7.1"
-if [[ "$OSTYPE" == "msys"* ]]; then # Windows
+if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows
   POETRY_BIN="$APPDATA/Python/Scripts/poetry"
 else
   POETRY_BIN="$HOME/.local/bin/poetry"
 fi
-$POETRY_BIN self add 'poetry-dynamic-versioning[plugin]'
+# Skip if already installed, for the same re-run reason as rustup above.
+if [ -x "$POETRY_BIN" ]; then
+  echo "Poetry already installed, skipping"
+else
+  echo "Installing poetry"
+  # Windows Python installs often provide `python` but not `python3`
+  PYTHON_FOR_POETRY=$(command -v python3 || command -v python)
+  curl -sSL https://install.python-poetry.org | "$PYTHON_FOR_POETRY" - --version "1.7.1"
+  "$POETRY_BIN" self add 'poetry-dynamic-versioning[plugin]'
+fi
 echo "Done installing dependencies"
 
 echo "Downloading spidermonkey source code"
 # Read the commit hash for mozilla-central from the `mozcentral.version` file
 MOZCENTRAL_VERSION=$(cat mozcentral.version)
-wget -c -q -O firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip
-unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source
+# Skip if already extracted so this script can be re-run after a later failure.
+if [ ! -d firefox-source ]; then
+  # curl rather than wget: it ships with Windows, macOS and most Linux distros
+  curl -fsSL -o firefox-source-${MOZCENTRAL_VERSION}.zip https://github.com/mozilla-firefox/firefox/archive/${MOZCENTRAL_VERSION}.zip
+  unzip -q firefox-source-${MOZCENTRAL_VERSION}.zip && mv firefox-${MOZCENTRAL_VERSION} firefox-source
+else
+  echo "firefox-source already exists, skipping download+extract"
+fi
 echo "Done downloading spidermonkey source code"
 
 echo "Building spidermonkey"
@@ -69,6 +99,7 @@ sed -i'' -e '/MOZ_CRASH_UNSAFE_PRINTF/,/__PRETTY_FUNCTION__);/d' ./mfbt/LinkedLi
 sed -i'' -e '/MOZ_ASSERT(stackRootPtr == nullptr);/d' ./js/src/vm/JSContext.cpp # would assert false in Debug Build since we extensively use `new JS::Rooted`
 sed -i'' -e 's/"-fuse-ld=ld"/"-ld64" if c_compiler.version > "14.0.0" else "-fuse-ld=ld"/' ./build/moz.configure/toolchain.configure # XCode 15 changed the linker behaviour. See https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes#Linking
 sed -i'' -e 's/defined(XP_WIN)/defined(_WIN32)/' ./mozglue/baseprofiler/public/BaseProfilerUtils.h # this header file is introduced to js/Debug.h in https://phabricator.services.mozilla.com/D221102, but it would be compiled without XP_WIN in this building configuration
+sed -i'' -e 's/os\.environ\["MOZILLABUILD"\]/os.environ.get("MOZILLABUILD", "")/g' ./python/mozbuild/mozbuild/backend/visualstudio.py # avoid KeyError: we don't use the official Mozilla Build package, so this is never set
 
 cd js/src
 mkdir -p _build
@@ -77,16 +108,14 @@ mkdir -p ../../../../_spidermonkey_install/
 ../configure --target=$(clang --print-target-triple) \
   --prefix=$(realpath $PWD/../../../../_spidermonkey_install) \
   --with-intl-api \
-  $(if [[ "$OSTYPE" != "msys"* ]]; then echo "--without-system-zlib"; fi) \
+  $(if [[ "$OSTYPE" != "msys"* && "$OSTYPE" != "cygwin"* ]]; then echo "--without-system-zlib"; fi) \
   --disable-debug-symbols \
   --disable-jemalloc \
   --disable-tests \
   $(if [[ "$OSTYPE" == "darwin"* ]]; then echo "--enable-linker=ld64"; fi) \
-  --enable-optimize \
-  --disable-explicit-resource-management
-# disable-explicit-resource-management: Disable the `using` syntax that is enabled by default in SpiderMonkey nightly, otherwise the header files will disagree with the compiled lib .so file
-#                                       when it's using a `IF_EXPLICIT_RESOURCE_MANAGEMENT` macro, e.g., the `enum JSProtoKey` index would be off by 1 (header `JSProto_Uint8Array` 27 will be interpreted as `JSProto_Int8Array` in lib as lib has an extra element)
-#                                       https://bugzilla.mozilla.org/show_bug.cgi?id=1940342
+  --enable-optimize
+# --disable-explicit-resource-management (Bugzilla 1940342 workaround) is no
+# longer a recognized flag; the feature it gated has shipped.
 make -j$CPUS
 echo "Done building spidermonkey"
 
@@ -120,7 +149,7 @@ if test -f .git/hooks/pre-commit; then
   cd uncrustify-source
   mkdir -p build
   cd build
-  if [[ "$OSTYPE" == "msys"* ]]; then # Windows
+  if [[ "$OSTYPE" == "msys"* || "$OSTYPE" == "cygwin"* ]]; then # Windows
     cmake ../
     cmake --build . -j$CPUS --config Release
     cp Release/uncrustify.exe ../../uncrustify.exe

src/BufferType.cc

@@ -14,6 +14,7 @@
 #include <jsapi.h>
 #include <js/ArrayBuffer.h>
 #include <js/experimental/TypedData.h>
+#include <js/GCAPI.h>
 #include <js/ScalarType.h>
 
 #include <limits.h>
@@ -80,17 +81,15 @@ PyObject *BufferType::fromJsTypedArray(JSContext *cx, JS::HandleObject typedArra
   JS::Scalar::Type subtype = JS_GetArrayBufferViewType(typedArray);
   auto byteLength = JS_GetTypedArrayByteLength(typedArray);
 
-  // Retrieve/Create the underlying ArrayBuffer object for side-effect.
-  //
-  // If byte length is less than `JS_MaxMovableTypedArraySize()`,
-  // the ArrayBuffer object would be created lazily and the data is stored inline in the TypedArray.
-  // We don't want inline data because the data pointer would be invalidated during a GC as the TypedArray object is moved.
-  bool isSharedMemory;
-  if (!JS_GetArrayBufferViewBuffer(cx, typedArray, &isSharedMemory)) return nullptr;
+  // Python keeps the returned pointer (as a memoryview) long after this call,
+  // so the data must not live inline in a GC-movable object. This moves both
+  // TypedArray-inline and small-ArrayBuffer-inline data out of line.
+  if (!JS::EnsureNonInlineArrayBufferOrView(cx, typedArray)) return nullptr;
 
-  uint8_t __destBuf[0] = {}; // we don't care about its value as it's used only if the TypedArray still having inline data
-  uint8_t *data = JS_GetArrayBufferViewFixedData(typedArray, __destBuf, 0 /* making sure we don't copy inline data */);
-  if (data == nullptr) { // shared memory or still having inline data
+  JS::AutoCheckCannotGC nogc(cx);
+  bool isSharedMemory;
+  uint8_t *data = static_cast<uint8_t *>(JS_GetArrayBufferViewData(typedArray, &isSharedMemory, nogc));
+  if (isSharedMemory || data == nullptr) {
     PyErr_SetString(PyExc_TypeError, "PythonMonkey cannot coerce TypedArrays backed by shared memory.");
     return nullptr;
   }

src/JSFunctionProxy.cc

@@ -16,6 +16,7 @@
 #include "include/setSpiderMonkeyException.hh"
 
 #include <jsapi.h>
+#include <jsfriendapi.h>
 
 #include <Python.h>
 
@@ -59,6 +60,10 @@ PyObject *JSFunctionProxyMethodDefinitions::JSFunctionProxy_call(PyObject *self,
     return NULL;
   }
 
+  // Python->JS callbacks (e.g. setTimeout handlers) can settle promises, and
+  // nothing else drains those reaction jobs. See JobQueue::runJobs.
+  js::RunJobs(cx);
+
   if (PyErr_Occurred()) {
     return NULL;
   }

src/JSMethodProxy.cc

@@ -16,6 +16,7 @@
 #include "include/setSpiderMonkeyException.hh"
 
 #include <jsapi.h>
+#include <jsfriendapi.h>
 
 #include <Python.h>
 
@@ -70,6 +71,9 @@ PyObject *JSMethodProxyMethodDefinitions::JSMethodProxy_call(PyObject *self, PyO
     return NULL;
   }
 
+  // Same checkpoint as JSFunctionProxy_call, for bound methods.
+  js::RunJobs(cx);
+
   if (PyErr_Occurred()) {
     return NULL;
   }