From 85d6cee2313821c42a5cbdb5ec135cbbdc47c821 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 7 Oct 2026 09:21:34 +1300 Subject: [PATCH 1/5] [ML] Retry 3rd-party git clones and propagate clone failures The Eigen and Valijson sources are cloned at CMake configure time from gitlab.com and github.com respectively. Those hosts occasionally return transient errors (e.g. GitLab "currently unable to handle this request due to load"), and a single failed clone was enough to break an entire CI build, requiring a manual rebuild. Wrap each clone in a bounded retry loop (5 attempts, increasing backoff) that starts from a clean slate on every attempt, so a brief hosting outage no longer fails the build. Also propagate the failure from the outer execute_process() calls that run these scripts. Previously the FATAL_ERROR raised inside the child `cmake -P` process was swallowed: configure logged the error but continued with an empty 3rd_party/eigen, so the failure only surfaced much later as a cryptic "Eigen/Core: No such file or directory" compile error. COMMAND_ERROR_IS_FATAL ANY makes configure stop immediately with the clear message once retries are exhausted, finally delivering the behaviour #3164 intended. Co-authored-by: Cursor --- 3rd_party/CMakeLists.txt | 8 +++++++- 3rd_party/pull-eigen.cmake | 36 ++++++++++++++++++++++++++--------- 3rd_party/pull-valijson.cmake | 32 +++++++++++++++++++++++++------ 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/3rd_party/CMakeLists.txt b/3rd_party/CMakeLists.txt index 0a1406089..6890dccb1 100644 --- a/3rd_party/CMakeLists.txt +++ b/3rd_party/CMakeLists.txt @@ -27,10 +27,15 @@ execute_process( ) # Pull the Eigen repo as part of the configuration step -# thus avoiding any race conditions with parallel builds +# thus avoiding any race conditions with parallel builds. +# COMMAND_ERROR_IS_FATAL ANY propagates a FATAL_ERROR raised inside the child +# script: without it the failure is logged but configure continues, leaving an +# empty 3rd_party/eigen and surfacing as a cryptic "Eigen/Core: No such file" +# compile error much later. execute_process( COMMAND ${CMAKE_COMMAND} -P ./pull-eigen.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY ) # Pull the Valijson repo as part of the configuration step @@ -38,6 +43,7 @@ execute_process( execute_process( COMMAND ${CMAKE_COMMAND} -P ./pull-valijson.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY ) # Build Abseil and Sandbox2 on Linux only. MlSandbox (lib/sandbox) is a diff --git a/3rd_party/pull-eigen.cmake b/3rd_party/pull-eigen.cmake index 1a76f5a8c..4d2dee90d 100644 --- a/3rd_party/pull-eigen.cmake +++ b/3rd_party/pull-eigen.cmake @@ -36,15 +36,33 @@ else() endif() if(PULL_EIGEN) - execute_process( - COMMAND ${CMAKE_COMMAND} -E rm -rf eigen - ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=3.4.0 https://gitlab.com/libeigen/eigen.git - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT - ) + # The GitLab host that serves Eigen is prone to transient "unable to handle + # this request due to load" failures. A single failed clone used to take out + # the whole build, so retry a few times with a short, increasing backoff + # before giving up. Each attempt starts from a clean slate because a failed + # clone can leave a partial directory behind. + set(EIGEN_CLONE_MAX_ATTEMPTS 5) + set(EIGEN_CLONE_BACKOFF_SECONDS 5) + set(GIT_RESULT 1) + foreach(attempt RANGE 1 ${EIGEN_CLONE_MAX_ATTEMPTS}) + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf eigen + ) + execute_process( + COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=3.4.0 https://gitlab.com/libeigen/eigen.git + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE GIT_RESULT + ) + if(GIT_RESULT EQUAL 0) + break() + endif() + if(attempt LESS ${EIGEN_CLONE_MAX_ATTEMPTS}) + math(EXPR backoff "${attempt} * ${EIGEN_CLONE_BACKOFF_SECONDS}") + message(WARNING "Failed to clone Eigen (attempt ${attempt}/${EIGEN_CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) + endif() + endforeach() if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Eigen from https://gitlab.com/libeigen/eigen.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") + message(FATAL_ERROR "Failed to clone Eigen from https://gitlab.com/libeigen/eigen.git after ${EIGEN_CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") endif() endif() diff --git a/3rd_party/pull-valijson.cmake b/3rd_party/pull-valijson.cmake index c80d4838d..1760a72b3 100644 --- a/3rd_party/pull-valijson.cmake +++ b/3rd_party/pull-valijson.cmake @@ -16,12 +16,32 @@ # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location if ( NOT EXISTS valijson ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT - ) + # Retry the clone a few times with a short, increasing backoff to ride out + # transient git hosting outages rather than failing the whole build on a + # single blip. Each attempt starts clean because a failed clone can leave a + # partial directory behind. + set(VALIJSON_CLONE_MAX_ATTEMPTS 5) + set(VALIJSON_CLONE_BACKOFF_SECONDS 5) + set(GIT_RESULT 1) + foreach(attempt RANGE 1 ${VALIJSON_CLONE_MAX_ATTEMPTS}) + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf valijson + ) + execute_process( + COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + RESULT_VARIABLE GIT_RESULT + ) + if(GIT_RESULT EQUAL 0) + break() + endif() + if(attempt LESS ${VALIJSON_CLONE_MAX_ATTEMPTS}) + math(EXPR backoff "${attempt} * ${VALIJSON_CLONE_BACKOFF_SECONDS}") + message(WARNING "Failed to clone Valijson (attempt ${attempt}/${VALIJSON_CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) + endif() + endforeach() if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Valijson from https://github.com/tristanpenman/valijson.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") + message(FATAL_ERROR "Failed to clone Valijson from https://github.com/tristanpenman/valijson.git after ${VALIJSON_CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") endif() endif() From be15ae7d22eab941ad615ff0c1a1c9c59ff52859 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 7 Oct 2026 09:23:22 +1300 Subject: [PATCH 2/5] Update docs/changelog/3233.yaml --- docs/changelog/3233.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/3233.yaml diff --git a/docs/changelog/3233.yaml b/docs/changelog/3233.yaml new file mode 100644 index 000000000..2415a4b5e --- /dev/null +++ b/docs/changelog/3233.yaml @@ -0,0 +1,5 @@ +area: Machine Learning +issues: [] +pr: 3233 +summary: Retry 3rd-party git clones and propagate clone failures +type: enhancement From 138861d00221fdbe2a9652b028563ae9afcc9cbb Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 7 Oct 2026 09:31:27 +1300 Subject: [PATCH 3/5] Delete docs/changelog/3233.yaml --- docs/changelog/3233.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/3233.yaml diff --git a/docs/changelog/3233.yaml b/docs/changelog/3233.yaml deleted file mode 100644 index 2415a4b5e..000000000 --- a/docs/changelog/3233.yaml +++ /dev/null @@ -1,5 +0,0 @@ -area: Machine Learning -issues: [] -pr: 3233 -summary: Retry 3rd-party git clones and propagate clone failures -type: enhancement From fd33d4ea489f5488646c72cb0633058590592d84 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 7 Oct 2026 09:52:18 +1300 Subject: [PATCH 4/5] [ML] Factor 3rd-party clone retry loop into a shared cmake helper The retry-with-backoff clone loop added in #3233 was duplicated verbatim in 3rd_party/pull-eigen.cmake and 3rd_party/pull-valijson.cmake. Extract it into ml_clone_git_dependency() in a new cmake/clone_git_dependency.cmake module that both scripts include. The helper lives in its own file rather than cmake/functions.cmake because the pull-*.cmake scripts run in `cmake -P` script mode, where functions.cmake's trailing add_custom_target() calls are invalid; a dedicated module also keeps the change self-contained and backport-clean. Co-authored-by: Cursor --- 3rd_party/pull-eigen.cmake | 38 ++++------------- 3rd_party/pull-valijson.cmake | 37 ++++------------- cmake/clone_git_dependency.cmake | 71 ++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 57 deletions(-) create mode 100644 cmake/clone_git_dependency.cmake diff --git a/3rd_party/pull-eigen.cmake b/3rd_party/pull-eigen.cmake index 4d2dee90d..762cf5b70 100644 --- a/3rd_party/pull-eigen.cmake +++ b/3rd_party/pull-eigen.cmake @@ -16,6 +16,8 @@ # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake) + # This is the file where Eigen stores its version set(VERSION_FILE "eigen/Eigen/src/Core/util/Macros.h") @@ -36,33 +38,11 @@ else() endif() if(PULL_EIGEN) - # The GitLab host that serves Eigen is prone to transient "unable to handle - # this request due to load" failures. A single failed clone used to take out - # the whole build, so retry a few times with a short, increasing backoff - # before giving up. Each attempt starts from a clean slate because a failed - # clone can leave a partial directory behind. - set(EIGEN_CLONE_MAX_ATTEMPTS 5) - set(EIGEN_CLONE_BACKOFF_SECONDS 5) - set(GIT_RESULT 1) - foreach(attempt RANGE 1 ${EIGEN_CLONE_MAX_ATTEMPTS}) - execute_process( - COMMAND ${CMAKE_COMMAND} -E rm -rf eigen - ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=3.4.0 https://gitlab.com/libeigen/eigen.git - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT - ) - if(GIT_RESULT EQUAL 0) - break() - endif() - if(attempt LESS ${EIGEN_CLONE_MAX_ATTEMPTS}) - math(EXPR backoff "${attempt} * ${EIGEN_CLONE_BACKOFF_SECONDS}") - message(WARNING "Failed to clone Eigen (attempt ${attempt}/${EIGEN_CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") - execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) - endif() - endforeach() - if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Eigen from https://gitlab.com/libeigen/eigen.git after ${EIGEN_CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") - endif() + ml_clone_git_dependency( + NAME Eigen + URL https://gitlab.com/libeigen/eigen.git + BRANCH 3.4.0 + DESTINATION eigen + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + ) endif() diff --git a/3rd_party/pull-valijson.cmake b/3rd_party/pull-valijson.cmake index 1760a72b3..a5aa53d2e 100644 --- a/3rd_party/pull-valijson.cmake +++ b/3rd_party/pull-valijson.cmake @@ -15,33 +15,14 @@ # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake) + if ( NOT EXISTS valijson ) - # Retry the clone a few times with a short, increasing backoff to ride out - # transient git hosting outages rather than failing the whole build on a - # single blip. Each attempt starts clean because a failed clone can leave a - # partial directory behind. - set(VALIJSON_CLONE_MAX_ATTEMPTS 5) - set(VALIJSON_CLONE_BACKOFF_SECONDS 5) - set(GIT_RESULT 1) - foreach(attempt RANGE 1 ${VALIJSON_CLONE_MAX_ATTEMPTS}) - execute_process( - COMMAND ${CMAKE_COMMAND} -E rm -rf valijson - ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git - WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT - ) - if(GIT_RESULT EQUAL 0) - break() - endif() - if(attempt LESS ${VALIJSON_CLONE_MAX_ATTEMPTS}) - math(EXPR backoff "${attempt} * ${VALIJSON_CLONE_BACKOFF_SECONDS}") - message(WARNING "Failed to clone Valijson (attempt ${attempt}/${VALIJSON_CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") - execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) - endif() - endforeach() - if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Valijson from https://github.com/tristanpenman/valijson.git after ${VALIJSON_CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") - endif() + ml_clone_git_dependency( + NAME Valijson + URL https://github.com/tristanpenman/valijson.git + BRANCH v1.0.2 + DESTINATION valijson + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} + ) endif() diff --git a/cmake/clone_git_dependency.cmake b/cmake/clone_git_dependency.cmake new file mode 100644 index 000000000..6d01b8a3f --- /dev/null +++ b/cmake/clone_git_dependency.cmake @@ -0,0 +1,71 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Helper used by the 3rd_party/pull-*.cmake scripts to fetch header-only +# dependencies. It is kept in its own module (rather than cmake/functions.cmake) +# so that it can be include()d from `cmake -P` script-mode invocations without +# pulling in the project-configuration targets defined there. + +# +# Clone a 3rd-party git dependency with a bounded retry loop. +# +# The hosts that serve our 3rd-party sources (gitlab.com, github.com) sometimes +# return transient errors under load, and a single failed clone used to take out +# an entire CI build. Retry a few times with a short, increasing backoff before +# giving up, starting from a clean slate on every attempt because a failed clone +# can leave a partial directory behind. A FATAL_ERROR is raised once the retries +# are exhausted so the caller (via COMMAND_ERROR_IS_FATAL) stops immediately with +# a clear message rather than failing later with a cryptic missing-header error. +# +# Named arguments: +# NAME human-readable dependency name used in log messages +# URL git repository URL to clone +# BRANCH branch or tag to check out (shallow, --depth=1) +# DESTINATION directory the repo is cloned into +# WORKING_DIRECTORY directory in which the clone is performed +# MAX_ATTEMPTS optional number of attempts (default 5) +# BACKOFF_SECONDS optional base backoff, multiplied by the attempt number (default 5) +# +function(ml_clone_git_dependency) + cmake_parse_arguments(CLONE "" "NAME;URL;BRANCH;DESTINATION;WORKING_DIRECTORY;MAX_ATTEMPTS;BACKOFF_SECONDS" "" ${ARGN}) + + if(NOT CLONE_MAX_ATTEMPTS) + set(CLONE_MAX_ATTEMPTS 5) + endif() + if(NOT CLONE_BACKOFF_SECONDS) + set(CLONE_BACKOFF_SECONDS 5) + endif() + + set(GIT_RESULT 1) + foreach(attempt RANGE 1 ${CLONE_MAX_ATTEMPTS}) + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + ) + execute_process( + COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=${CLONE_BRANCH} ${CLONE_URL} ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + RESULT_VARIABLE GIT_RESULT + ) + if(GIT_RESULT EQUAL 0) + break() + endif() + if(attempt LESS ${CLONE_MAX_ATTEMPTS}) + math(EXPR backoff "${attempt} * ${CLONE_BACKOFF_SECONDS}") + message(WARNING "Failed to clone ${CLONE_NAME} (attempt ${attempt}/${CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) + endif() + endforeach() + + if(NOT GIT_RESULT EQUAL 0) + message(FATAL_ERROR "Failed to clone ${CLONE_NAME} from ${CLONE_URL} after ${CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") + endif() +endfunction() From f7b7d8e71bb206905b218102c515d813ac8c5e81 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Wed, 7 Oct 2026 14:51:43 +1300 Subject: [PATCH 5/5] [ML] RSS probe clone helper: remove partial checkout on exhausted retries If every clone attempt fails, the helper raised FATAL_ERROR without cleaning up the destination. A partial directory left behind would cause a subsequent configure to skip the clone (pull-valijson.cmake guards on directory existence) and fail much later with a cryptic missing-header compile error. Remove the destination before reporting the fatal error so the next configure re-attempts the clone from a clean slate. Co-authored-by: Cursor --- cmake/clone_git_dependency.cmake | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cmake/clone_git_dependency.cmake b/cmake/clone_git_dependency.cmake index 6d01b8a3f..1587ec179 100644 --- a/cmake/clone_git_dependency.cmake +++ b/cmake/clone_git_dependency.cmake @@ -66,6 +66,14 @@ function(ml_clone_git_dependency) endforeach() if(NOT GIT_RESULT EQUAL 0) + # Remove any partial checkout left by the final failed attempt so that a + # subsequent configure re-attempts the clone instead of seeing a leftover + # directory, skipping the clone, and failing much later with a cryptic + # missing-header compile error. + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + ) message(FATAL_ERROR "Failed to clone ${CLONE_NAME} from ${CLONE_URL} after ${CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") endif() endfunction()