[8.19] [ML] Retry 3rd-party git clones and propagate clone failures (#3233)

3rd_party/CMakeLists.txt

@@ -27,15 +27,21 @@ execute_process(
 )
 
 # Pull the Eigen repo as part of the configuration step
-# thus avoiding any race conditions with parallel builds
+# thus avoiding any race conditions with parallel builds.
+# COMMAND_ERROR_IS_FATAL ANY propagates a FATAL_ERROR raised inside the child
+# script: without it the failure is logged but configure continues, leaving an
+# empty 3rd_party/eigen and surfacing as a cryptic "Eigen/Core: No such file"
+# compile error much later.
 execute_process(
     COMMAND ${CMAKE_COMMAND} -P ./pull-eigen.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+    COMMAND_ERROR_IS_FATAL ANY
 )
 
 # Pull the Valijson repo as part of the configuration step
 # thus avoiding any race conditions with parallel builds
 execute_process(
     COMMAND ${CMAKE_COMMAND} -P ./pull-valijson.cmake
     WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
-)
+    COMMAND_ERROR_IS_FATAL ANY
+)

3rd_party/pull-eigen.cmake

@@ -16,6 +16,8 @@
 
 # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location
 
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake)
+
 # This is the file where Eigen stores its version
 set(VERSION_FILE "eigen/Eigen/src/Core/util/Macros.h")
 
@@ -36,15 +38,11 @@ else()
 endif()
 
 if(PULL_EIGEN)
-  execute_process(
-    COMMAND ${CMAKE_COMMAND} -E rm -rf eigen
-    )
-  execute_process(
-    COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=3.4.0 https://gitlab.com/libeigen/eigen.git
+  ml_clone_git_dependency(
+    NAME Eigen
+    URL https://gitlab.com/libeigen/eigen.git
+    BRANCH 3.4.0
+    DESTINATION eigen
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-    RESULT_VARIABLE GIT_RESULT
     )
-  if(NOT GIT_RESULT EQUAL 0)
-    message(FATAL_ERROR "Failed to clone Eigen from https://gitlab.com/libeigen/eigen.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.")
-  endif()
 endif()

3rd_party/pull-valijson.cmake

@@ -15,13 +15,14 @@
 
 # This cmake script is expected to be called from a target or custom command with WORKING_DIRECTORY set to this file's location
 
+include(${CMAKE_CURRENT_LIST_DIR}/../cmake/clone_git_dependency.cmake)
+
 if ( NOT EXISTS valijson )
-  execute_process(
-    COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=v1.0.2 https://github.com/tristanpenman/valijson.git
+  ml_clone_git_dependency(
+    NAME Valijson
+    URL https://github.com/tristanpenman/valijson.git
+    BRANCH v1.0.2
+    DESTINATION valijson
     WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}
-    RESULT_VARIABLE GIT_RESULT
     )
-  if(NOT GIT_RESULT EQUAL 0)
-    message(FATAL_ERROR "Failed to clone Valijson from https://github.com/tristanpenman/valijson.git: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.")
-  endif()
 endif()

cmake/clone_git_dependency.cmake

@@ -0,0 +1,79 @@
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the Elastic License
+# 2.0 and the following additional limitation. Functionality enabled by the
+# files subject to the Elastic License 2.0 may only be used in production when
+# invoked by an Elasticsearch process with a license key installed that permits
+# use of machine learning features. You may not use this file except in
+# compliance with the Elastic License 2.0 and the foregoing additional
+# limitation.
+#
+
+# Helper used by the 3rd_party/pull-*.cmake scripts to fetch header-only
+# dependencies. It is kept in its own module (rather than cmake/functions.cmake)
+# so that it can be include()d from `cmake -P` script-mode invocations without
+# pulling in the project-configuration targets defined there.
+
+#
+# Clone a 3rd-party git dependency with a bounded retry loop.
+#
+# The hosts that serve our 3rd-party sources (gitlab.com, github.com) sometimes
+# return transient errors under load, and a single failed clone used to take out
+# an entire CI build. Retry a few times with a short, increasing backoff before
+# giving up, starting from a clean slate on every attempt because a failed clone
+# can leave a partial directory behind. A FATAL_ERROR is raised once the retries
+# are exhausted so the caller (via COMMAND_ERROR_IS_FATAL) stops immediately with
+# a clear message rather than failing later with a cryptic missing-header error.
+#
+# Named arguments:
+#   NAME              human-readable dependency name used in log messages
+#   URL               git repository URL to clone
+#   BRANCH            branch or tag to check out (shallow, --depth=1)
+#   DESTINATION       directory the repo is cloned into
+#   WORKING_DIRECTORY directory in which the clone is performed
+#   MAX_ATTEMPTS      optional number of attempts (default 5)
+#   BACKOFF_SECONDS   optional base backoff, multiplied by the attempt number (default 5)
+#
+function(ml_clone_git_dependency)
+  cmake_parse_arguments(CLONE "" "NAME;URL;BRANCH;DESTINATION;WORKING_DIRECTORY;MAX_ATTEMPTS;BACKOFF_SECONDS" "" ${ARGN})
+
+  if(NOT CLONE_MAX_ATTEMPTS)
+    set(CLONE_MAX_ATTEMPTS 5)
+  endif()
+  if(NOT CLONE_BACKOFF_SECONDS)
+    set(CLONE_BACKOFF_SECONDS 5)
+  endif()
+
+  set(GIT_RESULT 1)
+  foreach(attempt RANGE 1 ${CLONE_MAX_ATTEMPTS})
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION}
+      WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY}
+      )
+    execute_process(
+      COMMAND git -c advice.detachedHead=false clone --depth=1 --branch=${CLONE_BRANCH} ${CLONE_URL} ${CLONE_DESTINATION}
+      WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY}
+      RESULT_VARIABLE GIT_RESULT
+      )
+    if(GIT_RESULT EQUAL 0)
+      break()
+    endif()
+    if(attempt LESS ${CLONE_MAX_ATTEMPTS})
+      math(EXPR backoff "${attempt} * ${CLONE_BACKOFF_SECONDS}")
+      message(WARNING "Failed to clone ${CLONE_NAME} (attempt ${attempt}/${CLONE_MAX_ATTEMPTS}): git exited with ${GIT_RESULT}. Retrying in ${backoff}s.")
+      execute_process(COMMAND ${CMAKE_COMMAND} -E sleep ${backoff})
+    endif()
+  endforeach()
+
+  if(NOT GIT_RESULT EQUAL 0)
+    # Remove any partial checkout left by the final failed attempt so that a
+    # subsequent configure re-attempts the clone instead of seeing a leftover
+    # directory, skipping the clone, and failing much later with a cryptic
+    # missing-header compile error.
+    execute_process(
+      COMMAND ${CMAKE_COMMAND} -E rm -rf ${CLONE_DESTINATION}
+      WORKING_DIRECTORY ${CLONE_WORKING_DIRECTORY}
+      )
+    message(FATAL_ERROR "Failed to clone ${CLONE_NAME} from ${CLONE_URL} after ${CLONE_MAX_ATTEMPTS} attempts: git exited with ${GIT_RESULT}. Check network connectivity, proxy settings, and git availability.")
+  endif()
+endfunction()