From 67e76ec5188d399674999f62721be8348bf57470 Mon Sep 17 00:00:00 2001 From: Nayana Thorat Date: Wed, 12 Aug 2026 02:58:23 +0530 Subject: [PATCH] Add installation script for pyarrow 24.0.0 This script installs the pyarrow package version 24.0.0 along with its dependencies on UBI 10.1. It includes steps for configuring, building, and testing the package. --- p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh | 847 +++++++++++++++++++++++++++ 1 file changed, 847 insertions(+) create mode 100644 p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh diff --git a/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh b/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh new file mode 100644 index 0000000000..ff4c959730 --- /dev/null +++ b/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh @@ -0,0 +1,847 @@ +#!/bin/bash -e +# ----------------------------------------------------------------------------- +# +# Package : pyarrow +# Version : apache-arrow-24.0.0 +# Source repo : https://github.com/apache/arrow +# Tested on : UBI:10.1 +# Language : Python, C +# Ci-Check : True +# Script License: Apache License, Version 2 or later +# Maintainer : Nayana Thorat +# Disclaimer: This script has been tested in root mode on given +# ========== platform using the mentioned version of the package. +# It may not work as expected with newer versions of the +# package and/or distribution. In such case, please +# contact "Maintainer" of this script. +# +# ---------------------------------------------------------------------------- + +set -e + +PACKAGE_NAME=pyarrow +PACKAGE_DIR=arrow/python +PACKAGE_VERSION=${1:-apache-arrow-24.0.0} +PACKAGE_URL=https://github.com/apache/arrow +version=$(echo "$PACKAGE_VERSION" | sed 's/^apache-arrow-//') +CURRENT_DIR="${PWD}" + +echo "Install dependencies and tools." + +yum install -y \ + python3.12 \ + python3.12-devel \ + python3.12-pip \ + wget \ + git \ + make \ + cmake \ + xz-devel \ + openssl-devel \ + zlib-devel \ + libjpeg-devel \ + gcc \ + gcc-c++ \ + libevent \ + libtool \ + pkg-config \ + bzip2-devel \ + lz4-devel \ + libffi-devel \ + perl-Unicode-Normalize \ + patch \ + tzdata \ + binutils \ + ninja-build \ + brotli \ + brotli-devel + +SCRIPT_DIR=$(pwd) + +python3.12 -m pip install \ + numpy==2.2.6 \ + ninja \ + setuptools \ + setuptools_scm \ + scikit-build-core \ + wheel \ + Cython + +export CC="$(which gcc)" +export CXX="$(which g++)" + +# Do not globally set CMAKE_CXX_STANDARD here. +# Different dependencies use different C++ standards. + +echo "---------------------------------------c-ares installing---------------------------------------" + +git clone -b cares-1_19_1 https://github.com/c-ares/c-ares +cd c-ares + +export CA_PREFIX="$(pwd)/prefix" +mkdir -p "$CA_PREFIX" + +mkdir -p build +cd build + +AR="$(which ar)" + +cmake -GNinja .. \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$CA_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCARES_STATIC=OFF \ + -DCARES_SHARED=ON \ + -DCARES_INSTALL=ON \ + -DCMAKE_AR="$AR" + +ninja -v +ninja install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------c-ares installed----------------------------------------" + +echo "---------------------------------------xsimd installing----------------------------------------" + +git clone -b 14.0.0 https://github.com/xtensor-stack/xsimd/ +cd xsimd + +git submodule update --init + +mkdir -p prefix +export XS_PREFIX="$(pwd)/prefix" + +INCLUDE_PATH="${XS_PREFIX}/include" +LIBRARY_PATH="${XS_PREFIX}/lib" + +export CXXFLAGS_XSIMD="-fPIC" + +mkdir build-cmake +cd build-cmake + +cmake \ + -GNinja \ + -DCMAKE_PREFIX_PATH="$XS_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$XS_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + .. + +ninja +ninja install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------xsimd installed----------------------------------------" + +echo "---------------------------------------utf8proc installing------------------------------------" + +git clone -b v2.6.1 https://github.com/JuliaStrings/utf8proc +cd utf8proc + +git submodule update --init + +mkdir -p prefix +export UTF_PREFIX="$(pwd)/prefix" + +mkdir -p build +cd build + +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${UTF_PREFIX}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=ON \ + .. + +cmake --build . -j"$(nproc)" +cmake --build . --target install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------utf8proc installed-------------------------------------" + +echo "---------------------------------------re2 installing-----------------------------------------" + +git clone https://github.com/google/re2.git +cd re2 + +git checkout 2022-04-01 +git submodule update --init + +mkdir -p local/re2 +export RE2_PREFIX="$(pwd)/local/re2" + +export CPU_COUNT="$(nproc)" + +mkdir build-cmake +cd build-cmake + +cmake \ + -GNinja \ + -DCMAKE_PREFIX_PATH="$RE2_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$RE2_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DENABLE_TESTING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + .. + +ninja -v install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------re2 installed------------------------------------------" + +echo "---------------------------------------snappy installing--------------------------------------" + +git clone -b 1.2.2 https://github.com/google/snappy +cd snappy + +git submodule update --init + +mkdir -p local/snappy +export S_PREFIX="$(pwd)/local/snappy" + +mkdir -p build +cd build + +cmake \ + -DCMAKE_INSTALL_PREFIX="$S_PREFIX" \ + -DBUILD_SHARED_LIBS=ON \ + -DSNAPPY_BUILD_STATIC=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib \ + .. + +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------snappy installed--------------------------------------" + +echo "---------------------------------------flex installing---------------------------------------" + +wget https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz +tar -xvf flex-2.6.4.tar.gz + +cd flex-2.6.4 + +./configure --prefix=/usr + +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------flex installed----------------------------------------" + +echo "---------------------------------------texinfo installing------------------------------------" + +wget https://ftp.gnu.org/gnu/texinfo/texinfo-7.1.tar.gz +tar -xzf texinfo-7.1.tar.gz + +cd texinfo-7.1 + +./configure --prefix=/usr +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------texinfo installed-------------------------------------" + +echo "---------------------------------------bison installing--------------------------------------" + +wget https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.gz +tar -xvf bison-3.8.2.tar.gz + +cd bison-3.8.2 + +./configure --prefix=/usr +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------bison installed---------------------------------------" + +echo "---------------------------------------gflags installing--------------------------------------" + +git clone https://github.com/gflags/gflags.git +cd gflags +mkdir build && cd build +cmake .. +make -j$(nproc) +make install + +cd $SCRIPT_DIR +echo "---------------------------------------gflags installed---------------------------------------" + +echo "---------------------------------------boost installing--------------------------------------" + +git clone -b boost-1.81.0 https://github.com/boostorg/boost +cd boost + +git submodule update --init + +mkdir -p boostcpp +export BOOST_PREFIX="$(pwd)/boostcpp" + +INCLUDE_PATH="${BOOST_PREFIX}/include" +LIBRARY_PATH="${BOOST_PREFIX}/lib" + +export BOOST_CXXFLAGS="-fPIC" +TOOLSET=gcc + +cat > tools/build/example/site-config.jam <&1 | + grep '^ ' | + awk '{print "-isystem;" substr($1, 1)}' | + tr '\n' ';' +) + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" + +# Disable Gandiva + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" + +# SIMD + +EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_ALTIVEC=ON" + +sed -ie \ + 's;"--with-jemalloc-prefix=je_arrow_";"--with-jemalloc-prefix=je_arrow_" "--with-lg-page=16";g' \ + ../cmake_modules/ThirdpartyToolchain.cmake + +echo "============================================================" +echo "Arrow C++ configuration" +echo "CMAKE_ARGS=${CMAKE_ARGS}" +echo "CXXFLAGS=${CXXFLAGS}" +echo "============================================================" + +cmake \ + -GNinja \ + ${CMAKE_ARGS} \ + ${EXTRA_CMAKE_ARGS} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="${CC}" \ + -DCMAKE_CXX_COMPILER="${CXX}" \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DCMAKE_CXX_EXTENSIONS=OFF \ + -DCMAKE_AR="${AR}" \ + -DCMAKE_RANLIB="${RANLIB}" \ + -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \ + -DCMAKE_INSTALL_PREFIX="${ARROW_HOME}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=ON \ + -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_DATASET=ON \ + -DARROW_FLIGHT=ON \ + -DARROW_HDFS=ON \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_PYTHON=ON \ + -DARROW_S3=OFF \ + -DARROW_JEMALLOC=ON \ + -DARROW_MIMALLOC=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_BOOST_USE_SHARED=ON \ + -DProtobuf_ROOT="${PROTOBUF_PREFIX}" \ + -DProtobuf_DIR="${Protobuf_DIR}" \ + -DPROTOBUF_PROTOC_EXECUTABLE="${PROTOBUF_PREFIX}/bin/protoc" \ + -DgRPC_DIR="${gRPC_DIR}" \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -Dabsl_DIR="${ABSL_DIR}" \ + -DCARES_INCLUDE_DIR="${CA_PREFIX}/include" \ + -DCARES_LIBRARY="${CA_PREFIX}/lib/libcares.so" \ + -DRE2_INCLUDE_DIR="${RE2_PREFIX}/include" \ + -DRE2_LIBRARY="${RE2_PREFIX}/lib/libre2.so" \ + -DSnappy_INCLUDE_DIR="${S_PREFIX}/include" \ + -DSnappy_LIBRARY="${S_PREFIX}/lib/libsnappy.so" \ + -DORC_INCLUDE_DIR="${ORC_PREFIX}/include" \ + -DORC_LIBRARY="${ORC_PREFIX}/lib/liborc.so" \ + -DThrift_INCLUDE_DIR="${THRIFT_PREFIX}/include" \ + -DThrift_LIBRARY="${THRIFT_PREFIX}/lib/libthrift.so" \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_INCLUDE_DIR="${BOOST_PREFIX}/include" \ + -Dutf8proc_INCLUDE_DIR="${UTF_PREFIX}/include" \ + -Dutf8proc_LIB="${UTF_PREFIX}/lib/libutf8proc.so" \ + -DZSTD_ROOT="${ZSTD_HOME}" \ + -DZSTD_INCLUDE_DIR="${ZSTD_HOME}/include" \ + -DZSTD_LIBRARY="${ZSTD_HOME}/lib/libzstd.so" \ + -DLZ4_HOME=/usr \ + -DZLIB_HOME=/usr \ + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-$(which python3.12)}" \ + -DPython3_EXECUTABLE="${Python3_EXECUTABLE:-$(which python3.12)}" \ + -DRapidJSON_ROOT="${RJ_PREFIX}" \ + .. + +ninja -v +ninja install + +export PYARROW_BUNDLE_ARROW_CPP=ON +export PYARROW_BUNDLE_ARROW_CPP_HEADERS=ON + +export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}" +export CMAKE_PREFIX_PATH="${ARROW_HOME}:${CMAKE_PREFIX_PATH}" +export CMAKE_GENERATOR=Ninja + +export Arrow_DIR="${ARROW_HOME}/lib/cmake/Arrow" + +# Build dependencies + +export PARQUET_HOME="$ARROW_HOME" +export SETUPTOOLS_SCM_PRETEND_VERSION="$version" + +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 +export PYARROW_WITH_GANDIVA=0 +export PYARROW_WITH_HDFS=1 +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_S3=0 + +export PYARROW_WITH_CUDA=0 + +echo "--------------------------------------------pyarrow installed--------------------------------" + +echo "--------------------------------------------pyarrow wheel building---------------------------" + +cd "$SCRIPT_DIR/arrow/python" + +# ------------------------------------------------------------------------- +# Remove any previous scikit-build-core/CMake cache. +# This is important because the PyArrow wheel performs a NEW CMake +# configuration and an old cache can contain C++17. +# ------------------------------------------------------------------------- + +rm -rf build +rm -rf _skbuild + +# Make C++20 explicit for the PyArrow wheel. +export CXXFLAGS="-std=c++20 -fPIC -I${BOOST_PREFIX}/include -I${THRIFT_PREFIX}/include" +export CFLAGS="-fPIC" + +export CMAKE_ARGS="-DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD_REQUIRED=ON -DCMAKE_CXX_EXTENSIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DArrow_DIR=${Arrow_DIR}" + +echo "============================================================" +echo "PyArrow wheel configuration" +echo "CMAKE_ARGS=${CMAKE_ARGS}" +echo "CXXFLAGS=${CXXFLAGS}" +echo "Arrow_DIR=${Arrow_DIR}" +echo "============================================================" + +python3.12 -m pip wheel \ + -w "$SCRIPT_DIR" \ + -vv \ + --no-build-isolation \ + --no-deps \ + -C cmake.build-type=Release \ + -C build.verbose=true \ + . + +cd "$SCRIPT_DIR" + +echo "--------------------------------------------pyarrow wheel build completed---------------------------" + +echo "--------------------------------------------Testing pyarrow-----------------------------------------" + +WHEEL=$(find "${SCRIPT_DIR}" -maxdepth 1 -name "${PACKAGE_NAME}-*.whl" | head -1) +python3.12 -m pip install "${WHEEL}" --no-deps +SITE_PACKAGES=$(python3.12 -m pip show pyarrow | awk -F': ' '/^Location:/ {print $2}') + +export LD_LIBRARY_PATH="${GRPC_PREFIX}/lib:${PROTOBUF_PREFIX}/lib64:${ORC_PREFIX}/lib:${THRIFT_PREFIX}/lib:${BOOST_PREFIX}/lib:${RE2_PREFIX}/lib:${CA_PREFIX}/lib:${XS_PREFIX}/lib:${UTF_PREFIX}/lib:${S_PREFIX}/lib:${ZSTD_HOME}/lib:/usr/lib64:/usr/lib:$SITE_PACKAGES/pyarrow:${LD_LIBRARY_PATH}" + +python3.12 -m pip install -r $SCRIPT_DIR/arrow/python/requirements-test.txt "pytest<9" + +PYARROW_LOCATION=$(python3.12 -c "import os; import pyarrow; print(os.path.dirname(pyarrow.__file__))") +export PARQUET_TEST_DATA="$SCRIPT_DIR/arrow/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA=$SCRIPT_DIR/arrow/testing/data + +if ! python3.12 -m pytest -k "not test_foreign_buffer and not test_get_include and not pandas" $PYARROW_LOCATION -vv ; then + echo "------------------$PACKAGE_NAME:test_fails---------------------" + echo "$PACKAGE_URL $PACKAGE_NAME " + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Fail | Test_Fails" + exit 2 +else + echo "------------------$PACKAGE_NAME:test_success-------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME " + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Pass | Test_Success" + exit 0 +fi + +echo "--------------------------------------------Testing pyarrow completed --------------------------------" +