diff --git a/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh b/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh new file mode 100644 index 0000000000..ff4c959730 --- /dev/null +++ b/p/pyarrow/pyarrow_24.0.0_ubi_10.1.sh @@ -0,0 +1,847 @@ +#!/bin/bash -e +# ----------------------------------------------------------------------------- +# +# Package : pyarrow +# Version : apache-arrow-24.0.0 +# Source repo : https://github.com/apache/arrow +# Tested on : UBI:10.1 +# Language : Python, C +# Ci-Check : True +# Script License: Apache License, Version 2 or later +# Maintainer : Nayana Thorat +# Disclaimer: This script has been tested in root mode on given +# ========== platform using the mentioned version of the package. +# It may not work as expected with newer versions of the +# package and/or distribution. In such case, please +# contact "Maintainer" of this script. +# +# ---------------------------------------------------------------------------- + +set -e + +PACKAGE_NAME=pyarrow +PACKAGE_DIR=arrow/python +PACKAGE_VERSION=${1:-apache-arrow-24.0.0} +PACKAGE_URL=https://github.com/apache/arrow +version=$(echo "$PACKAGE_VERSION" | sed 's/^apache-arrow-//') +CURRENT_DIR="${PWD}" + +echo "Install dependencies and tools." + +yum install -y \ + python3.12 \ + python3.12-devel \ + python3.12-pip \ + wget \ + git \ + make \ + cmake \ + xz-devel \ + openssl-devel \ + zlib-devel \ + libjpeg-devel \ + gcc \ + gcc-c++ \ + libevent \ + libtool \ + pkg-config \ + bzip2-devel \ + lz4-devel \ + libffi-devel \ + perl-Unicode-Normalize \ + patch \ + tzdata \ + binutils \ + ninja-build \ + brotli \ + brotli-devel + +SCRIPT_DIR=$(pwd) + +python3.12 -m pip install \ + numpy==2.2.6 \ + ninja \ + setuptools \ + setuptools_scm \ + scikit-build-core \ + wheel \ + Cython + +export CC="$(which gcc)" +export CXX="$(which g++)" + +# Do not globally set CMAKE_CXX_STANDARD here. +# Different dependencies use different C++ standards. + +echo "---------------------------------------c-ares installing---------------------------------------" + +git clone -b cares-1_19_1 https://github.com/c-ares/c-ares +cd c-ares + +export CA_PREFIX="$(pwd)/prefix" +mkdir -p "$CA_PREFIX" + +mkdir -p build +cd build + +AR="$(which ar)" + +cmake -GNinja .. \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$CA_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCARES_STATIC=OFF \ + -DCARES_SHARED=ON \ + -DCARES_INSTALL=ON \ + -DCMAKE_AR="$AR" + +ninja -v +ninja install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------c-ares installed----------------------------------------" + +echo "---------------------------------------xsimd installing----------------------------------------" + +git clone -b 14.0.0 https://github.com/xtensor-stack/xsimd/ +cd xsimd + +git submodule update --init + +mkdir -p prefix +export XS_PREFIX="$(pwd)/prefix" + +INCLUDE_PATH="${XS_PREFIX}/include" +LIBRARY_PATH="${XS_PREFIX}/lib" + +export CXXFLAGS_XSIMD="-fPIC" + +mkdir build-cmake +cd build-cmake + +cmake \ + -GNinja \ + -DCMAKE_PREFIX_PATH="$XS_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$XS_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DCMAKE_BUILD_TYPE=Release \ + .. + +ninja +ninja install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------xsimd installed----------------------------------------" + +echo "---------------------------------------utf8proc installing------------------------------------" + +git clone -b v2.6.1 https://github.com/JuliaStrings/utf8proc +cd utf8proc + +git submodule update --init + +mkdir -p prefix +export UTF_PREFIX="$(pwd)/prefix" + +mkdir -p build +cd build + +cmake -G "Unix Makefiles" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${UTF_PREFIX}" \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DBUILD_SHARED_LIBS=ON \ + .. + +cmake --build . -j"$(nproc)" +cmake --build . --target install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------utf8proc installed-------------------------------------" + +echo "---------------------------------------re2 installing-----------------------------------------" + +git clone https://github.com/google/re2.git +cd re2 + +git checkout 2022-04-01 +git submodule update --init + +mkdir -p local/re2 +export RE2_PREFIX="$(pwd)/local/re2" + +export CPU_COUNT="$(nproc)" + +mkdir build-cmake +cd build-cmake + +cmake \ + -GNinja \ + -DCMAKE_PREFIX_PATH="$RE2_PREFIX" \ + -DCMAKE_INSTALL_PREFIX="$RE2_PREFIX" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DENABLE_TESTING=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=ON \ + .. + +ninja -v install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------re2 installed------------------------------------------" + +echo "---------------------------------------snappy installing--------------------------------------" + +git clone -b 1.2.2 https://github.com/google/snappy +cd snappy + +git submodule update --init + +mkdir -p local/snappy +export S_PREFIX="$(pwd)/local/snappy" + +mkdir -p build +cd build + +cmake \ + -DCMAKE_INSTALL_PREFIX="$S_PREFIX" \ + -DBUILD_SHARED_LIBS=ON \ + -DSNAPPY_BUILD_STATIC=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib \ + .. + +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------snappy installed--------------------------------------" + +echo "---------------------------------------flex installing---------------------------------------" + +wget https://github.com/westes/flex/releases/download/v2.6.4/flex-2.6.4.tar.gz +tar -xvf flex-2.6.4.tar.gz + +cd flex-2.6.4 + +./configure --prefix=/usr + +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------flex installed----------------------------------------" + +echo "---------------------------------------texinfo installing------------------------------------" + +wget https://ftp.gnu.org/gnu/texinfo/texinfo-7.1.tar.gz +tar -xzf texinfo-7.1.tar.gz + +cd texinfo-7.1 + +./configure --prefix=/usr +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------texinfo installed-------------------------------------" + +echo "---------------------------------------bison installing--------------------------------------" + +wget https://ftp.gnu.org/gnu/bison/bison-3.8.2.tar.gz +tar -xvf bison-3.8.2.tar.gz + +cd bison-3.8.2 + +./configure --prefix=/usr +make -j"$(nproc)" +make install + +cd "$SCRIPT_DIR" + +echo "---------------------------------------bison installed---------------------------------------" + +echo "---------------------------------------gflags installing--------------------------------------" + +git clone https://github.com/gflags/gflags.git +cd gflags +mkdir build && cd build +cmake .. +make -j$(nproc) +make install + +cd $SCRIPT_DIR +echo "---------------------------------------gflags installed---------------------------------------" + +echo "---------------------------------------boost installing--------------------------------------" + +git clone -b boost-1.81.0 https://github.com/boostorg/boost +cd boost + +git submodule update --init + +mkdir -p boostcpp +export BOOST_PREFIX="$(pwd)/boostcpp" + +INCLUDE_PATH="${BOOST_PREFIX}/include" +LIBRARY_PATH="${BOOST_PREFIX}/lib" + +export BOOST_CXXFLAGS="-fPIC" +TOOLSET=gcc + +cat > tools/build/example/site-config.jam <&1 | + grep '^ ' | + awk '{print "-isystem;" substr($1, 1)}' | + tr '\n' ';' +) + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA_PC_CXX_FLAGS=${SYSTEM_INCLUDES}" + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_CUDA=OFF" + +# Disable Gandiva + +EXTRA_CMAKE_ARGS="${EXTRA_CMAKE_ARGS} -DARROW_GANDIVA=OFF" + +# SIMD + +EXTRA_CMAKE_ARGS=" ${EXTRA_CMAKE_ARGS} -DARROW_ALTIVEC=ON" + +sed -ie \ + 's;"--with-jemalloc-prefix=je_arrow_";"--with-jemalloc-prefix=je_arrow_" "--with-lg-page=16";g' \ + ../cmake_modules/ThirdpartyToolchain.cmake + +echo "============================================================" +echo "Arrow C++ configuration" +echo "CMAKE_ARGS=${CMAKE_ARGS}" +echo "CXXFLAGS=${CXXFLAGS}" +echo "============================================================" + +cmake \ + -GNinja \ + ${CMAKE_ARGS} \ + ${EXTRA_CMAKE_ARGS} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER="${CC}" \ + -DCMAKE_CXX_COMPILER="${CXX}" \ + -DCMAKE_CXX_STANDARD=20 \ + -DCMAKE_CXX_STANDARD_REQUIRED=ON \ + -DCMAKE_CXX_EXTENSIONS=OFF \ + -DCMAKE_AR="${AR}" \ + -DCMAKE_RANLIB="${RANLIB}" \ + -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \ + -DCMAKE_INSTALL_PREFIX="${ARROW_HOME}" \ + -DCMAKE_INSTALL_LIBDIR=lib \ + -DBUILD_SHARED_LIBS=ON \ + -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_BUILD_STATIC=OFF \ + -DARROW_DATASET=ON \ + -DARROW_FLIGHT=ON \ + -DARROW_HDFS=ON \ + -DARROW_ORC=ON \ + -DARROW_PARQUET=ON \ + -DARROW_PYTHON=ON \ + -DARROW_S3=OFF \ + -DARROW_JEMALLOC=ON \ + -DARROW_MIMALLOC=ON \ + -DARROW_WITH_BROTLI=ON \ + -DARROW_WITH_BZ2=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DARROW_WITH_THRIFT=ON \ + -DARROW_BOOST_USE_SHARED=ON \ + -DProtobuf_ROOT="${PROTOBUF_PREFIX}" \ + -DProtobuf_DIR="${Protobuf_DIR}" \ + -DPROTOBUF_PROTOC_EXECUTABLE="${PROTOBUF_PREFIX}/bin/protoc" \ + -DgRPC_DIR="${gRPC_DIR}" \ + -DgRPC_ABSL_PROVIDER=package \ + -DgRPC_PROTOBUF_PROVIDER=package \ + -DgRPC_ZLIB_PROVIDER=package \ + -DgRPC_SSL_PROVIDER=package \ + -DgRPC_RE2_PROVIDER=package \ + -DgRPC_CARES_PROVIDER=package \ + -Dabsl_DIR="${ABSL_DIR}" \ + -DCARES_INCLUDE_DIR="${CA_PREFIX}/include" \ + -DCARES_LIBRARY="${CA_PREFIX}/lib/libcares.so" \ + -DRE2_INCLUDE_DIR="${RE2_PREFIX}/include" \ + -DRE2_LIBRARY="${RE2_PREFIX}/lib/libre2.so" \ + -DSnappy_INCLUDE_DIR="${S_PREFIX}/include" \ + -DSnappy_LIBRARY="${S_PREFIX}/lib/libsnappy.so" \ + -DORC_INCLUDE_DIR="${ORC_PREFIX}/include" \ + -DORC_LIBRARY="${ORC_PREFIX}/lib/liborc.so" \ + -DThrift_INCLUDE_DIR="${THRIFT_PREFIX}/include" \ + -DThrift_LIBRARY="${THRIFT_PREFIX}/lib/libthrift.so" \ + -DBoost_ROOT="${BOOST_PREFIX}" \ + -DBoost_INCLUDE_DIR="${BOOST_PREFIX}/include" \ + -Dutf8proc_INCLUDE_DIR="${UTF_PREFIX}/include" \ + -Dutf8proc_LIB="${UTF_PREFIX}/lib/libutf8proc.so" \ + -DZSTD_ROOT="${ZSTD_HOME}" \ + -DZSTD_INCLUDE_DIR="${ZSTD_HOME}/include" \ + -DZSTD_LIBRARY="${ZSTD_HOME}/lib/libzstd.so" \ + -DLZ4_HOME=/usr \ + -DZLIB_HOME=/usr \ + -DPYTHON_EXECUTABLE="${PYTHON_EXECUTABLE:-$(which python3.12)}" \ + -DPython3_EXECUTABLE="${Python3_EXECUTABLE:-$(which python3.12)}" \ + -DRapidJSON_ROOT="${RJ_PREFIX}" \ + .. + +ninja -v +ninja install + +export PYARROW_BUNDLE_ARROW_CPP=ON +export PYARROW_BUNDLE_ARROW_CPP_HEADERS=ON + +export LD_LIBRARY_PATH="${ARROW_HOME}/lib:${LD_LIBRARY_PATH}" +export CMAKE_PREFIX_PATH="${ARROW_HOME}:${CMAKE_PREFIX_PATH}" +export CMAKE_GENERATOR=Ninja + +export Arrow_DIR="${ARROW_HOME}/lib/cmake/Arrow" + +# Build dependencies + +export PARQUET_HOME="$ARROW_HOME" +export SETUPTOOLS_SCM_PRETEND_VERSION="$version" + +export PYARROW_WITH_DATASET=1 +export PYARROW_WITH_FLIGHT=1 +export PYARROW_WITH_GANDIVA=0 +export PYARROW_WITH_HDFS=1 +export PYARROW_WITH_ORC=1 +export PYARROW_WITH_PARQUET=1 +export PYARROW_WITH_S3=0 + +export PYARROW_WITH_CUDA=0 + +echo "--------------------------------------------pyarrow installed--------------------------------" + +echo "--------------------------------------------pyarrow wheel building---------------------------" + +cd "$SCRIPT_DIR/arrow/python" + +# ------------------------------------------------------------------------- +# Remove any previous scikit-build-core/CMake cache. +# This is important because the PyArrow wheel performs a NEW CMake +# configuration and an old cache can contain C++17. +# ------------------------------------------------------------------------- + +rm -rf build +rm -rf _skbuild + +# Make C++20 explicit for the PyArrow wheel. +export CXXFLAGS="-std=c++20 -fPIC -I${BOOST_PREFIX}/include -I${THRIFT_PREFIX}/include" +export CFLAGS="-fPIC" + +export CMAKE_ARGS="-DCMAKE_CXX_STANDARD=20 -DCMAKE_CXX_STANDARD_REQUIRED=ON -DCMAKE_CXX_EXTENSIONS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DArrow_DIR=${Arrow_DIR}" + +echo "============================================================" +echo "PyArrow wheel configuration" +echo "CMAKE_ARGS=${CMAKE_ARGS}" +echo "CXXFLAGS=${CXXFLAGS}" +echo "Arrow_DIR=${Arrow_DIR}" +echo "============================================================" + +python3.12 -m pip wheel \ + -w "$SCRIPT_DIR" \ + -vv \ + --no-build-isolation \ + --no-deps \ + -C cmake.build-type=Release \ + -C build.verbose=true \ + . + +cd "$SCRIPT_DIR" + +echo "--------------------------------------------pyarrow wheel build completed---------------------------" + +echo "--------------------------------------------Testing pyarrow-----------------------------------------" + +WHEEL=$(find "${SCRIPT_DIR}" -maxdepth 1 -name "${PACKAGE_NAME}-*.whl" | head -1) +python3.12 -m pip install "${WHEEL}" --no-deps +SITE_PACKAGES=$(python3.12 -m pip show pyarrow | awk -F': ' '/^Location:/ {print $2}') + +export LD_LIBRARY_PATH="${GRPC_PREFIX}/lib:${PROTOBUF_PREFIX}/lib64:${ORC_PREFIX}/lib:${THRIFT_PREFIX}/lib:${BOOST_PREFIX}/lib:${RE2_PREFIX}/lib:${CA_PREFIX}/lib:${XS_PREFIX}/lib:${UTF_PREFIX}/lib:${S_PREFIX}/lib:${ZSTD_HOME}/lib:/usr/lib64:/usr/lib:$SITE_PACKAGES/pyarrow:${LD_LIBRARY_PATH}" + +python3.12 -m pip install -r $SCRIPT_DIR/arrow/python/requirements-test.txt "pytest<9" + +PYARROW_LOCATION=$(python3.12 -c "import os; import pyarrow; print(os.path.dirname(pyarrow.__file__))") +export PARQUET_TEST_DATA="$SCRIPT_DIR/arrow/cpp/submodules/parquet-testing/data" +export ARROW_TEST_DATA=$SCRIPT_DIR/arrow/testing/data + +if ! python3.12 -m pytest -k "not test_foreign_buffer and not test_get_include and not pandas" $PYARROW_LOCATION -vv ; then + echo "------------------$PACKAGE_NAME:test_fails---------------------" + echo "$PACKAGE_URL $PACKAGE_NAME " + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Fail | Test_Fails" + exit 2 +else + echo "------------------$PACKAGE_NAME:test_success-------------------------" + echo "$PACKAGE_URL $PACKAGE_NAME " + echo "$PACKAGE_NAME | $PACKAGE_URL | $PACKAGE_VERSION | $OS_NAME | Github | Pass | Test_Success" + exit 0 +fi + +echo "--------------------------------------------Testing pyarrow completed --------------------------------" +