diff --git a/programs/salmon/build.sh b/programs/salmon/build.sh index 793cf0a..8f529d8 100644 --- a/programs/salmon/build.sh +++ b/programs/salmon/build.sh @@ -167,6 +167,25 @@ case "${system}" in # -DUSE_FJMPI=OFF # ) # ;; + RIKYU) + module purge + module load nvhpc/26.3 + cmake_args=( + "${common_cmake_args[@]}" + -DCMAKE_Fortran_COMPILER=mpif90 + -DCMAKE_C_COMPILER=mpicc + -DOPENMP_FLAGS=-Mnoopenmp + -DUSE_OPENACC=ON + -DUSE_CUDA=ON + -DUSE_MPI_DEFAULT=ON + -DCMAKE_SYSTEM_PROCESSOR=openacc + -DCMAKE_Fortran_FLAGS="-O3 -Wall -fstrict-aliasing -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_C_FLAGS="-O3 -Wall -alias=ansi -acc=strict -gpu=cc100,managed,ptxinfo -cudalib=cublas -cuda -Minfo=accel -DUSE_OPENACC -DUSE_CUDA" + -DCMAKE_CUDA_ARCHITECTURES=100 + -DCMAKE_CUDA_FLAGS=-arch=sm_100 + -DFORTRAN_COMPILER_HAS_MPI_VERSION3=OFF + ) + ;; *) echo "Unknown system: ${system}" >&2 exit 1 diff --git a/programs/salmon/list.csv b/programs/salmon/list.csv index 386804d..ed4e159 100644 --- a/programs/salmon/list.csv +++ b/programs/salmon/list.csv @@ -4,3 +4,7 @@ RC_GH200,yes,1,1,1,0:10:00 RC_DGXSP,yes,1,1,1,1:00:00 RC_GENOA,yes,1,1,48,2:00:00 RC_FX700,no,1,1,48,0:30:00 +RIKYU,yes,1,4,1,0:30:00 +RIKYU,yes,2,4,1,0:30:00 +RIKYU,yes,3,4,1,0:30:00 +RIKYU,yes,4,4,1,0:30:00 diff --git a/programs/salmon/run.sh b/programs/salmon/run.sh index 1689c38..b482be0 100644 --- a/programs/salmon/run.sh +++ b/programs/salmon/run.sh @@ -13,6 +13,7 @@ RESULTS_DIR="${PWD}/results" WORK_DIR="${PWD}/salmon_run" INPUT_ARCHIVE_DEFAULT="/vol0003/rccs-sdt/data/a01010/benchmark_data/SALMON.tar.gz" INPUT_ARCHIVE_CLOUD="/lvs0/dne1/rccs-nghpcadu/CX_input/SALMON/SALMON.tar.gz" +INPUT_ARCHIVE_RIKYU="/data1/rkp00012/CX_input/SALMON/SALMON.tar.gz" AOCL_ROOT_DEFAULT="/lvs0/rccs-nghpcadu/nakamura/aocl/install" mkdir -p "${RESULTS_DIR}" @@ -29,6 +30,11 @@ case "${system}" in exec_gs=(-stdin Si-1-1-1.nml ./salmon) exec_rt=(-stdin Si-1-1-1-tddft.nml ./salmon) ;; + RIKYU) + input_archive="${BK_SALMON_INPUT_ARCHIVE:-${INPUT_ARCHIVE_RIKYU}}" + exec_gs=(./salmon) + exec_rt=(./salmon) + ;; RC_GH200|RC_DGXSP|RC_GENOA) input_archive="${INPUT_ARCHIVE_CLOUD}" exec_gs=(./salmon) @@ -47,7 +53,7 @@ fi uses_stdin_input() { case "$1" in - RC_GH200|RC_DGXSP|RC_GENOA) + RIKYU|RC_GH200|RC_DGXSP|RC_GENOA) return 0 ;; *) @@ -108,6 +114,35 @@ case "${system}" in fi export OMP_NUM_THREADS="${nthreads}" ;; + RIKYU) + module purge + module load nvhpc/26.3 + export OMP_NUM_THREADS="${nthreads}" + for nml in Si-1-1-1.nml Si-1-1-1-tddft.nml; do + awk -v nproc_ob="${n_ranks}" ' + /^¶llel$/ { print; print " nproc_ob = " nproc_ob; print " nproc_k = 1"; print " nproc_rgrid = 1, 1, 1"; in_parallel=1; next } + in_parallel && /^\// { in_parallel=0; print; next } + in_parallel { next } + { print } + ' "$nml" > "$nml.tmp" && mv "$nml.tmp" "$nml" + done + if [[ "${n_ranks}" -gt 1 ]]; then + cat > wrapper.sh <<'WRAPPER' +#!/bin/bash +NCUDA_GPUS=${NCUDA_GPUS:-$(nvidia-smi -L | wc -l)} +if [ "$OMPI_COMM_WORLD_LOCAL_SIZE" -gt "$NCUDA_GPUS" ]; then + if [ "$OMPI_COMM_WORLD_LOCAL_RANK" -eq 0 ]; then + nvidia-cuda-mps-control -d + fi + sleep 10 +fi +export CUDA_VISIBLE_DEVICES=$((${OMPI_COMM_WORLD_LOCAL_RANK} % ${NCUDA_GPUS})) +exec "$@" +WRAPPER + chmod +x wrapper.sh + export UCX_IB_GPU_DIRECT_RDMA=no + fi + ;; # RC_FX700) # FX700 currently fails during GS initialization even with the Fujitsu # topology guard patch applied. Keep this route disabled until verified. @@ -122,6 +157,13 @@ run_salmon() { local logfile="$1" shift case "${system}" in + RIKYU) + if [[ "${n_ranks}" -gt 1 ]]; then + mpirun -n "${n_ranks}" ./wrapper.sh "$@" > "${logfile}" 2>&1 + else + mpirun -n "${n_ranks}" "$@" > "${logfile}" 2>&1 + fi + ;; RC_GENOA) mpirun -n "${n_ranks}" --bind-to core --map-by "ppr:${numproc_node}:node:PE=${nthreads}" "$@" > "${logfile}" 2>&1 ;;