From 8bcce0d3fc3a5b38ff65092961edb675597e4fe9 Mon Sep 17 00:00:00 2001 From: Ed Savage Date: Fri, 9 Oct 2026 09:43:43 +1300 Subject: [PATCH] [ML] Retry 3rd-party git clones and propagate clone failures (#3233) The Eigen and Valijson sources are cloned at CMake configure time from gitlab.com and github.com respectively. Those hosts occasionally return transient errors (e.g. GitLab "currently unable to handle this request due to load"), and a single failed clone was enough to break an entire CI build, requiring a manual rebuild. Wrap each clone in a bounded retry loop (5 attempts, increasing backoff) that starts from a clean slate on every attempt, so a brief hosting outage no longer fails the build. Also propagate the failure from the outer execute_process() calls that run these scripts. Previously the FATAL_ERROR raised inside the child `cmake -P` process was swallowed: configure logged the error but continued with an empty 3rd_party/eigen, so the failure only surfaced much later as a cryptic "Eigen/Core: No such file or directory" compile error. COMMAND_ERROR_IS_FATAL ANY makes configure stop immediately with the clear message once retries are exhausted, finally delivering the behaviour #3164 intended. (cherry picked from commit bcce4aa584a69284516230d988538749b2bda64d) --- 3rd_party/CMakeLists.txt | 8 +++- 3rd_party/pull-eigen.cmake | 16 +++---- 3rd_party/pull-valijson.cmake | 13 +++--- cmake/clone_git_dependency.cmake | 79 ++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 16 deletions(-) create mode 100644 cmake/clone_git_dependency.cmake diff --git a/3rd_party/CMakeLists.txt b/3rd_party/CMakeLists.txt index f2b092f913..6da8b66b82 100644 --- a/3rd_party/CMakeLists.txt +++ b/3rd_party/CMakeLists.txt @@ -27,10 +27,15 @@ execute_process( ) # Pull the Eigen repo as part of the configuration step -# thus avoiding any race conditions with parallel builds +# thus avoiding any race conditions with parallel builds. +# COMMAND_ERROR_IS_FATAL ANY propagates a FATAL_ERROR raised inside the child +# script: without it the failure is logged but configure continues, leaving an +# empty 3rd_party/eigen and surfacing as a cryptic "Eigen/Core: No such file" +# compile error much later. execute_process( COMMAND ${CMAKE_COMMAND} -P ./pull-eigen.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY ) # Pull the Valijson repo as part of the configuration step @@ -38,4 +43,5 @@ execute_process( execute_process( COMMAND ${CMAKE_COMMAND} -P ./pull-valijson.cmake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + COMMAND_ERROR_IS_FATAL ANY ) diff --git a/3rd_party/pull-eigen.cmake b/3rd_party/pull-eigen.cmake index 1a76f5a8c5..762cf5b708 100644 --- a/3rd_party/pull-eigen.cmake +++ b/3rd_party/pull-eigen.cmake @@ -16,6 +16,8 @@ # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake) + # This is the file where Eigen stores its version set(VERSION_FILE "eigen/Eigen/src/Core/util/Macros.h") @@ -36,15 +38,11 @@ else() endif() if(PULL_EIGEN) - execute_process( - COMMAND ${CMAKE_COMMAND} -E rm -rf eigen - ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=3.4.0 https://gitlab.com/libeigen/eigen.git + ml_clone_git_dependency( + NAME Eigen + URL https://gitlab.com/libeigen/eigen.git + BRANCH 3.4.0 + DESTINATION eigen WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT ) - if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Eigen from https://gitlab.com/libeigen/eigen.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") - endif() endif() diff --git a/3rd_party/pull-valijson.cmake b/3rd_party/pull-valijson.cmake index c80d4838d6..a5aa53d2e8 100644 --- a/3rd_party/pull-valijson.cmake +++ b/3rd_party/pull-valijson.cmake @@ -15,13 +15,14 @@ # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location +include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake) + if ( NOT EXISTS valijson ) - execute_process( - COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git + ml_clone_git_dependency( + NAME Valijson + URL https://github.com/tristanpenman/valijson.git + BRANCH v1.0.2 + DESTINATION valijson WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} - RESULT_VARIABLE GIT_RESULT ) - if(NOT GIT_RESULT EQUAL 0) - message(FATAL_ERROR "Failed to clone Valijson from https://github.com/tristanpenman/valijson.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") - endif() endif() diff --git a/cmake/clone_git_dependency.cmake b/cmake/clone_git_dependency.cmake new file mode 100644 index 0000000000..1587ec1797 --- /dev/null +++ b/cmake/clone_git_dependency.cmake @@ -0,0 +1,79 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License +# 2.0 and the following additional limitation. Functionality enabled by the +# files subject to the Elastic License 2.0 may only be used in production when +# invoked by an Elasticsearch process with a license key installed that permits +# use of machine learning features. You may not use this file except in +# compliance with the Elastic License 2.0 and the foregoing additional +# limitation. +# + +# Helper used by the 3rd_party/pull-*.cmake scripts to fetch header-only +# dependencies. It is kept in its own module (rather than cmake/functions.cmake) +# so that it can be include()d from `cmake -P` script-mode invocations without +# pulling in the project-configuration targets defined there. + +# +# Clone a 3rd-party git dependency with a bounded retry loop. +# +# The hosts that serve our 3rd-party sources (gitlab.com, github.com) sometimes +# return transient errors under load, and a single failed clone used to take out +# an entire CI build. Retry a few times with a short, increasing backoff before +# giving up, starting from a clean slate on every attempt because a failed clone +# can leave a partial directory behind. A FATAL_ERROR is raised once the retries +# are exhausted so the caller (via COMMAND_ERROR_IS_FATAL) stops immediately with +# a clear message rather than failing later with a cryptic missing-header error. +# +# Named arguments: +# NAME human-readable dependency name used in log messages +# URL git repository URL to clone +# BRANCH branch or tag to check out (shallow, --depth=1) +# DESTINATION directory the repo is cloned into +# WORKING_DIRECTORY directory in which the clone is performed +# MAX_ATTEMPTS optional number of attempts (default 5) +# BACKOFF_SECONDS optional base backoff, multiplied by the attempt number (default 5) +# +function(ml_clone_git_dependency) + cmake_parse_arguments(CLONE "" "NAME;URL;BRANCH;DESTINATION;WORKING_DIRECTORY;MAX_ATTEMPTS;BACKOFF_SECONDS" "" ${ARGN}) + + if(NOT CLONE_MAX_ATTEMPTS) + set(CLONE_MAX_ATTEMPTS 5) + endif() + if(NOT CLONE_BACKOFF_SECONDS) + set(CLONE_BACKOFF_SECONDS 5) + endif() + + set(GIT_RESULT 1) + foreach(attempt RANGE 1 ${CLONE_MAX_ATTEMPTS}) + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + ) + execute_process( + COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=${CLONE_BRANCH} ${CLONE_URL} ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + RESULT_VARIABLE GIT_RESULT + ) + if(GIT_RESULT EQUAL 0) + break() + endif() + if(attempt LESS ${CLONE_MAX_ATTEMPTS}) + math(EXPR backoff "${attempt} * ${CLONE_BACKOFF_SECONDS}") + message(WARNING "Failed to clone ${CLONE_NAME} (attempt ${attempt}/${CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff}) + endif() + endforeach() + + if(NOT GIT_RESULT EQUAL 0) + # Remove any partial checkout left by the final failed attempt so that a + # subsequent configure re-attempts the clone instead of seeing a leftover + # directory, skipping the clone, and failing much later with a cryptic + # missing-header compile error. + execute_process( + COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION} + WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY} + ) + message(FATAL_ERROR "Failed to clone ${CLONE_NAME} from ${CLONE_URL} after ${CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.") + endif() +endfunction()