[8.19] [ML] Backport Sandbox2 stack (#3181, #3182, #3185, #3187, #3188, #3215)

.buildkite/scripts/steps/create_dra.sh

@@ -18,6 +18,9 @@
 # 4. Combine the platform-specific non 3rd party dependencies into a 'deps' bundle
 # 4. Create a dependency report containing licensing info on the 3rd party dependencies.
 
+# Shared helper asserting the controller-protocol.version marker is packaged.
+. "${REPO_ROOT}/dev-tools/verify_controller_protocol_version.sh"
+
 rm -rf build/distributions
 
 # Default to a snapshot build
@@ -47,30 +50,47 @@ for it in darwin-aarch64 darwin-x86_64 linux-aarch64 linux-x86_64 windows-x86_64
   unzip -o build/distributions/ml-cpp-${VERSION}-${it}.zip -d build/temp;
 done
 cd build/temp
-zip ../distributions/ml-cpp-${VERSION}.zip -r platform
+# Include controller-protocol.version at the zip root alongside 'platform' so the
+# all-platform uber zip carries the marker too, matching the Gradle buildUberZip
+# task (which pulls it in via buildZip). Each platform zip stages the marker at
+# its root, so unzipping above leaves a copy at build/temp/.
+zip ../distributions/ml-cpp-${VERSION}.zip -r platform controller-protocol.version
 
-# Create a zip excluding dependencies from combined platform-specific C++ distributions
+# Create a zip excluding dependencies from combined platform-specific C++ distributions.
+# controller-protocol.version is staged at the bundle root by the packaging step
+# (dev-tools/docker/docker_entrypoint.sh and the Gradle buildZip task); it must
+# ship in the -nodeps bundle alongside the controller it makes claims about so
+# Elasticsearch's verifyControllerProtocolVersion gate can assert against it.
 find . \( -path "**/libMl*" -o \
        -path "**/platform/darwin*/controller.app/Contents/MacOS/*" -o \
        -path "**/platform/linux*/bin/*" -o \
        -path "**/platform/windows*/bin/*.exe" -o \
        -path "**/ml-en.dict" -o \
        -path "**/Info.plist" -o \
        -path "**/date_time_zonespec.csv" -o \
+       -path "**/controller-protocol.version" -o \
        -path "**/licenses/**" \) -print -exec touch -t 2401010000 {} \; | sort | xargs zip -X ../distributions/ml-cpp-${VERSION}-nodeps.zip
 
-# Create a zip of dependencies only from combined platform-specific C++ distributions
+# Create a zip of dependencies only from combined platform-specific C++ distributions.
+# controller-protocol.version must be pruned here so it ships only in the -nodeps bundle:
+# it is not a 3rd-party dependency, and leaving it in both bundles makes Elasticsearch's
+# ml plugin bundle merge (which unzips -deps and -nodeps together) fail with a duplicate
+# 'controller-protocol.version' entry.
 find . \( -path "**/libMl*" -o \
           -path "**/platform/darwin*/controller.app/Contents/MacOS/*" -o \
           -path "**/platform/linux*/bin/*" -o \
           -path "**/platform/windows*/bin/*.exe" -o \
           -path "**/ml-en.dict" -o \
           -path "**/Info.plist" -o \
           -path "**/date_time_zonespec.csv" -o \
+          -path "**/controller-protocol.version" -o \
           -path "**/licenses/**" \) -prune -o -print -exec touch -t 2401010000 {} \; | sort | xargs zip -X ../distributions/ml-cpp-${VERSION}-deps.zip
 
 cd -
 
+verify_controller_protocol_version build/distributions/ml-cpp-${VERSION}.zip || exit 1
+verify_controller_protocol_version build/distributions/ml-cpp-${VERSION}-nodeps.zip || exit 1
+
 # Create a CSV report on 3rd party dependencies we redistribute
 ./3rd_party/dependency_report.sh --csv build/distributions/dependencies-${VERSION}.csv 
 

3rd_party/3rd_party.cmake

@@ -181,6 +181,21 @@ function(install_libs _target _source_dir _prefix _postfix)
 
   set(LIBRARIES ${ARGN})
 
+  # Each requested library must be found in its own right. A coarse
+  # "did the source directory contain anything matching *${_prefix}*${_postfix}?"
+  # guard is not enough: an unrelated library that happens to share the prefix
+  # and suffix (e.g. libmkl_scalapack_lp64.so.2 when every library actually
+  # requested has moved to .so.3) satisfies it, and every individual library is
+  # then skipped silently. That ships a distribution whose binaries cannot
+  # resolve their NEEDED libraries, and the only symptom is the dynamic loader
+  # killing the process with exit code 127 before it can log anything.
+  foreach(LIBRARY ${LIBRARIES})
+    file(GLOB _CHECK_LIBS ${_source_dir}/*${_prefix}${LIBRARY}*${_postfix})
+    if(NOT _CHECK_LIBS)
+      message(FATAL_ERROR "${_target}: no library matching '${_prefix}${LIBRARY}*${_postfix}' found in ${_source_dir}")
+    endif()
+  endforeach()
+
   file(GLOB _LIBS ${_source_dir}/*${_prefix}*${_postfix})
 
   if(_LIBS)
@@ -235,22 +250,47 @@ install_libs("zlib" ${ZLIB_LOCATION} "" "${ZLIB_EXTENSION}" "zlib")
 install_libs("Torch libraries" ${TORCH_LOCATION} "" "${TORCH_EXTENSION}" "${TORCH_LIBRARIES}")
 install_libs("Intel MKL libraries" ${MKL_LOCATION} "${MKL_PREFIX}" "${MKL_EXTENSION}" "${MKL_LIBRARIES}")
 
-# On Linux, replace the RPATH for 3rd party libraries that already have one.
+# On Linux, set the RPATH of every bundled 3rd party library to $ORIGIN.
 # (Only Linux targets will have a location for the gcc runtime library.)
+#
+# These libraries are all installed flat into the same directory, so $ORIGIN lets
+# each one find its siblings at runtime. This must be done unconditionally rather
+# than only for libraries that already declare an RPATH: some prebuilt libraries
+# (notably Boost, depending on how it was built) ship with no RPATH at all, and
+# because DT_RUNPATH is not inherited transitively, a library with a sibling
+# dependency (e.g. libboost_log -> libboost_atomic) fails to load at runtime even
+# though the dependency sits right beside it. The native controller has its
+# environment cleared by Elasticsearch's Spawner, so RPATH is the sole resolution
+# mechanism - there is no LD_LIBRARY_PATH fallback.
+#
+# The one exception is Intel MKL, which must be left exactly as Intel ships it.
+# Rewriting its RPATH makes pytorch_inference die with SIGSEGV during the first
+# inference of real models (ELSER, E5) on hosts with glibc 2.34 (Amazon Linux
+# 2023, the ES integration test agents), while tiny test models and newer glibc
+# versions are unaffected. The libraries also do not need it: libmkl_core,
+# libmkl_intel_lp64 and libmkl_gnu_thread are NEEDED by libtorch_cpu, which already
+# has an $ORIGIN RPATH, and the CPU-specific kernels MKL dlopen()s later only NEED
+# libmkl_core, which is resolved by SONAME because it is already loaded.
 if (GCC_RT_LOCATION)
   execute_process(COMMAND find . -type f COMMAND egrep -v "^core|-debug$|libMl" COMMAND xargs COMMAND sed -e "s/ /;/g" OUTPUT_VARIABLE FOUND_LIBRARIES WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
   foreach(LIBRARY ${FOUND_LIBRARIES})
-    execute_process(COMMAND patchelf --print-rpath ${LIBRARY} COMMAND grep lib OUTPUT_VARIABLE RPATH_VAR ERROR_VARIABLE RPATH_ERR WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
-    if(RPATH_VAR)
-      message(STATUS "Attempting to overwrite existing RPATH ${RPATH_VAR} in ${LIBRARY}")
-      execute_process(COMMAND patchelf --force-rpath --set-rpath "$ORIGIN" ${LIBRARY} OUTPUT_VARIABLE SET_RPATH_OUT ERROR_VARIABLE SET_RPATH_ERR WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
+    get_filename_component(LIBRARY_NAME ${LIBRARY} NAME)
+    if(LIBRARY_NAME MATCHES "^libmkl_")
+      message(STATUS "Leaving RPATH of Intel MKL library ${LIBRARY} unchanged")
+      continue()
+    endif()
+    # Only ELF objects can carry an RPATH. patchelf --print-rpath exits non-zero
+    # on anything else, so use it to skip non-ELF files without failing the build.
+    execute_process(COMMAND patchelf --print-rpath ${LIBRARY} RESULT_VARIABLE IS_ELF_RESULT OUTPUT_QUIET ERROR_QUIET WORKING_DIRECTORY "${INSTALL_DIR}")
+    if(IS_ELF_RESULT EQUAL 0)
+      execute_process(COMMAND patchelf --force-rpath --set-rpath "$ORIGIN" ${LIBRARY} ERROR_VARIABLE SET_RPATH_ERR WORKING_DIRECTORY "${INSTALL_DIR}" OUTPUT_STRIP_TRAILING_WHITESPACE)
       if(SET_RPATH_ERR)
         message(FATAL_ERROR "Error setting RPATH in ${LIBRARY}: ${SET_RPATH_ERR}")
       else()
-        message(STATUS "Set RPATH in ${LIBRARY}")
+        message(STATUS "Set RPATH to $ORIGIN in ${LIBRARY}")
       endif()
     else()
-      message(STATUS "Did not set RPATH in ${LIBRARY}")
+      message(STATUS "Skipping non-ELF file ${LIBRARY}")
     endif()
   endforeach()
 endif()

3rd_party/CMakeLists.txt

@@ -27,15 +27,151 @@ execute_process(
 )
 
 # Pull the Eigen repo as part of the configuration step
-# thus avoiding any race conditions with parallel builds
+# thus avoiding any race conditions with parallel builds.
+# COMMAND_ERROR_IS_FATAL ANY propagates a FATAL_ERROR raised inside the child
+# script: without it the failure is logged but configure continues, leaving an
+# empty 3rd_party/eigen and surfacing as a cryptic "Eigen/Core: No such file"
+# compile error much later.
 execute_process(
     COMMAND ${CMAKE_COMMAND} -P ./pull-eigen.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND_ERROR_IS_FATAL ANY
 )
 
 # Pull the Valijson repo as part of the configuration step
 # thus avoiding any race conditions with parallel builds
 execute_process(
     COMMAND ${CMAKE_COMMAND} -P ./pull-valijson.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
+    COMMAND_ERROR_IS_FATAL ANY
+)
+
+# Build Abseil and Sandbox2 on Linux only. MlSandbox (lib/sandbox) is a
+# dormant target: it is built everywhere Sandbox2 is available, but nothing
+# in the controller/pytorch_inference wiring routes to it yet. The sandbox
+# policy, spawner, and controller routing land in follow-up PRs.
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
+  include(FetchContent)
+
+  # Save and restore CMake cache state this block flips so it cannot change
+  # the caller's build configuration for anything outside Sandbox2/Abseil.
+  set(_saved_BUILD_TESTING ${BUILD_TESTING})
+  set(BUILD_TESTING OFF CACHE BOOL "" FORCE)
+  set(_saved_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS})
+  set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+
+  # The vendored Abseil and Sandboxed API sources are not unity-build safe:
+  # e.g. absl_time_zone defines kDigits in an anonymous namespace in both
+  # time_zone_fixed.cc and time_zone_posix.cc, which collide when merged
+  # into one unity translation unit. The top-level build configures
+  # -DCMAKE_UNITY_BUILD=ON, so disable it for these third-party targets only.
+  set(_saved_CMAKE_UNITY_BUILD ${CMAKE_UNITY_BUILD})
+  set(CMAKE_UNITY_BUILD OFF)
+
+  # The top-level build applies ml-cpp's strict warning set (including
+  # -Wconversion, see cmake/compiler/*.cmake) to every target via
+  # add_compile_options(${ML_CXX_FLAGS}), and the debug Linux CI build sets
+  # CMAKE_COMPILE_WARNING_AS_ERROR=ON. Both are inherited by any vendored
+  # third-party sources compiled in this block, whose own diagnostics we do
+  # not control and should not gate our build on. Keep the warnings visible
+  # but non-fatal for this third-party subtree only; ml-cpp's own targets are
+  # unaffected and continue to treat warnings as errors.
+  set(_saved_CMAKE_COMPILE_WARNING_AS_ERROR ${CMAKE_COMPILE_WARNING_AS_ERROR})
+  set(CMAKE_COMPILE_WARNING_AS_ERROR OFF)
+
+  set(ABSL_PROPAGATE_CXX_STD ON CACHE INTERNAL "" FORCE)
+  set(ABSL_USE_EXTERNAL_GOOGLETEST OFF CACHE INTERNAL "" FORCE)
+  set(ABSL_FIND_GOOGLETEST OFF CACHE INTERNAL "" FORCE)
+  set(ABSL_ENABLE_INSTALL OFF CACHE INTERNAL "" FORCE)
+  set(ABSL_BUILD_TESTING OFF CACHE INTERNAL "" FORCE)
+  set(ABSL_BUILD_TEST_HELPERS OFF CACHE INTERNAL "" FORCE)
+  set(SAPI_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+  set(SAPI_BUILD_TESTING OFF CACHE BOOL "" FORCE)
+
+  set(ML_SANDBOXED_API_TAG v20241008)
+  set(ML_SANDBOXED_API_GIT_SHA 9e07542a03fefa2cf982ba093b099805362df05d)
+  set(ML_SANDBOXED_API_PATCH_DIR ${CMAKE_CURRENT_SOURCE_DIR}/patches/sandboxed-api)
+  set(ML_SANDBOXED_API_PATCHES
+    0001-abseil-cpp-disable-gtest.patch
+    0002-no-fno-exceptions-propagation.patch
+    0003-python3-optional.patch
+    0004-forkserver-zlib-static-libstdcxx.patch
+  )
+
+  FetchContent_Declare(
+    sandboxed-api
+    GIT_REPOSITORY https://github.com/google/sandboxed-api.git
+    GIT_TAG ${ML_SANDBOXED_API_GIT_SHA}
+  )
+
+  FetchContent_GetProperties(sandboxed-api)
+  if(NOT sandboxed-api_POPULATED)
+    FetchContent_Populate(sandboxed-api)
+
+    find_package(Git REQUIRED)
+    foreach(_patch ${ML_SANDBOXED_API_PATCHES})
+      # Re-running configure in an existing build directory can re-enter this
+      # block even though the checked-out source was already patched in an
+      # earlier configure (observed: FetchContent's populated-tracking does
+      # not reliably short-circuit this across separate `cmake` invocations
+      # on every CMake/generator combination). `git apply --check` alone
+      # cannot distinguish "already applied" from "genuinely drifted" - both
+      # fail to apply cleanly - so try a reverse-check first: if the patch
+      # reverses cleanly, its change is already present and this is the
+      # idempotent-rerun case, not drift.
+      execute_process(
+        COMMAND ${GIT_EXECUTABLE} apply --reverse --check ${ML_SANDBOXED_API_PATCH_DIR}/${_patch}
+        WORKING_DIRECTORY ${sandboxed-api_SOURCE_DIR}
+        RESULT_VARIABLE _patch_already_applied_result
+        OUTPUT_QUIET
+        ERROR_QUIET
+      )
+      if(_patch_already_applied_result EQUAL 0)
+        message(STATUS "sandboxed-api patch already applied (reconfigure): ${_patch}")
+        continue()
+      endif()
+
+      execute_process(
+        COMMAND ${GIT_EXECUTABLE} apply --check ${ML_SANDBOXED_API_PATCH_DIR}/${_patch}
+        WORKING_DIRECTORY ${sandboxed-api_SOURCE_DIR}
+        RESULT_VARIABLE _patch_check_result
+        OUTPUT_QUIET
+        ERROR_VARIABLE _patch_check_error
+      )
+      if(NOT _patch_check_result EQUAL 0)
+        message(FATAL_ERROR
+          "sandboxed-api patch ${_patch} no longer applies to pinned tag "
+          "${ML_SANDBOXED_API_TAG} (${ML_SANDBOXED_API_GIT_SHA}) - the "
+          "upstream source has drifted since this patch was written. "
+          "Regenerate it against the current tag content (see "
+          "3rd_party/patches/sandboxed-api/README.md).\n"
+          "${_patch_check_error}")
+      endif()
+      execute_process(
+        COMMAND ${GIT_EXECUTABLE} apply ${ML_SANDBOXED_API_PATCH_DIR}/${_patch}
+        WORKING_DIRECTORY ${sandboxed-api_SOURCE_DIR}
+        RESULT_VARIABLE _patch_apply_result
+        ERROR_VARIABLE _patch_apply_error
+      )
+      if(NOT _patch_apply_result EQUAL 0)
+        message(FATAL_ERROR "sandboxed-api patch ${_patch} failed to apply: ${_patch_apply_error}")
+      endif()
+      message(STATUS "Applied sandboxed-api patch: ${_patch}")
+    endforeach()
+  endif()
+
+  add_subdirectory(${sandboxed-api_SOURCE_DIR} ${sandboxed-api_BINARY_DIR} EXCLUDE_FROM_ALL)
+
+  if(TARGET sandbox2::sandbox2)
+    set(SANDBOX2_LIBRARIES sandbox2::sandbox2 CACHE INTERNAL "Sandbox2 libraries")
+    message(STATUS "Sandbox2 enabled: using sandbox2::sandbox2")
+  else()
+    message(FATAL_ERROR "Sandbox2 required on Linux but sandbox2::sandbox2 was not built")
+  endif()
+
+  # Restore the caller's settings for the rest of the build.
+  set(BUILD_TESTING ${_saved_BUILD_TESTING} CACHE BOOL "" FORCE)
+  set(BUILD_SHARED_LIBS ${_saved_BUILD_SHARED_LIBS} CACHE BOOL "" FORCE)
+  set(CMAKE_UNITY_BUILD ${_saved_CMAKE_UNITY_BUILD})
+  set(CMAKE_COMPILE_WARNING_AS_ERROR ${_saved_CMAKE_COMPILE_WARNING_AS_ERROR})
+endif()

3rd_party/controller-protocol.version

@@ -0,0 +1 @@
+controller-protocol-version=2

3rd_party/licenses/abseil-INFO.csv

@@ -0,0 +1,2 @@
+name,version,revision,url,license,copyright,sourceURL
+abseil-cpp,2024-04-05,61e47a454c81eb07147b0315485f476513cc1230,https://abseil.io,Apache License 2.0,,https://github.com/abseil/abseil-cpp/archive/61e47a454c81eb07147b0315485f476513cc1230.zip