[master] CI/CD updates to be more stable (#20740)

* Since website s3 push and publish is not run inside a container, just use the awscli installed in the jenkins slave (which is updated.) When multiple processes are attempting to install a pip package at the same time, there is a race condition that causes them to fail often.

* Update variable for CUDA archs in windows make script, so we don't end up building for all.

* Fix CUDA arch variable used.

* Remove oneapi repo after installing, to alleviate failures when intel publishes corrupt files.

Author: josephevans

ci/build_windows.py

@@ -118,7 +118,7 @@ class BuildFlavour(Enum):
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
         '-DUSE_DIST_KVSTORE=OFF '
-        '-DMXNET_CUDA_ARCH="5.2" '
+        '-DMXNET_CUDA_ARCH="5.2 7.5" '
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_GPU_ONEDNN': (
@@ -131,7 +131,7 @@ class BuildFlavour(Enum):
         '-DUSE_BLAS=open '
         '-DUSE_LAPACK=ON '
         '-DUSE_DIST_KVSTORE=OFF '
-        '-DMXNET_CUDA_ARCH="5.2" '
+        '-DMXNET_CUDA_ARCH="5.2 7.5" '
         '-DUSE_ONEDNN=ON '
         '-DCMAKE_BUILD_TYPE=Release')
 

ci/docker/Dockerfile.build.ubuntu

@@ -91,7 +91,8 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
         libb2-dev \
         libzstd-dev \
         gfortran && \
-    rm -rf /var/lib/apt/lists/*
+    rm -rf /var/lib/apt/lists/* && \
+    add-apt-repository -r "deb https://apt.repos.intel.com/oneapi all main"
 
 # Build OpenBLAS from source
 RUN export LIBRARY_PATH=$LIBRARY_PATH:/usr/lib/gcc/x86_64-linux-gnu/7/ && \

ci/docker/runtime_functions.sh

@@ -1355,7 +1355,6 @@ build_docs_beta() {
 push_docs() {
     folder_name=$1
     set -ex
-    pip3 install --user awscli
     export PATH=~/.local/bin:$PATH
     pushd docs/_build
     tar -xzf full_website.tgz --strip-components 1
@@ -1471,7 +1470,6 @@ cd_pypi_publish() {
 
 cd_s3_publish() {
     set -ex
-    pip3 install --upgrade --user awscli
     filepath=$(readlink -f wheel_build/dist/*.whl)
     filename=$(basename $filepath)
     variant=$(echo $filename | cut -d'-' -f1 | cut -d'_' -f2 -s)