diff --git a/config/queue.csv b/config/queue.csv index 1e7573e..5dc57dc 100644 --- a/config/queue.csv +++ b/config/queue.csv @@ -1,5 +1,5 @@ queue,submit_cmd,template -SLURM_RIKYU,sbatch,"-p ${queue_group} --account=${account} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads} --gpus=${proc}" +SLURM_RIKYU,sbatch,"-p ${queue_group} ${scheduler_extra_args} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads} --gpus=${proc}" FJ,pjsub,"-L rscunit=rscunit_ft01,rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi max-proc-per-node=${numproc_node} -x PJM_LLIO_GFSCACHE=/vol0002:/vol0003:/vol0004:/vol0005" PJM_GENKAI,pjsub,"-L rscgrp=${queue_group},elapse=${elapse},node=${nodes} --mpi proc=${proc}" SLURM_RC,sbatch,"-p ${queue_group} -t ${elapse} -N ${nodes} --ntasks-per-node=${numproc_node} --cpus-per-task=${nthreads}" diff --git a/config/system.csv b/config/system.csv index e602904..e71a4b1 100644 --- a/config/system.csv +++ b/config/system.csv @@ -1,5 +1,5 @@ -system,mode,tag_build,tag_run,queue,queue_group,account -RIKYU,cross,ai4ss_login,ai4ss_jacamar,SLURM_RIKYU,gpu,rkp00012 +system,mode,tag_build,tag_run,queue,queue_group +RIKYU,cross,ai4ss_login,ai4ss_jacamar,SLURM_RIKYU,gpu Fugaku,cross,fugaku_login1,fugaku_jacamar,FJ,small FugakuCN,native,,fugaku_jacamar,FJ,small RC_GH200,native,,cloud_jacamar,SLURM_RC,qc-gh200 diff --git a/docs/cx/BENCHKIT_SPEC.md b/docs/cx/BENCHKIT_SPEC.md index c62e56c..e9b8dde 100644 --- a/docs/cx/BENCHKIT_SPEC.md +++ b/docs/cx/BENCHKIT_SPEC.md @@ -251,7 +251,7 @@ BenchKit の system / queue 定義層である。 - `queue.csv` に scheduler template を持つ - app 側の `list.csv` から system 固有事情を切り離す -`system.csv` は system 名、mode、runner tag、queue、queue_group、scheduler account などの正本である。 +`system.csv` は system 名、mode、runner tag、queue、queue_group などの正本である。 This is the system and queue definition layer. @@ -261,8 +261,7 @@ Main roles: - `queue.csv` holds scheduler templates - system-specific concerns are separated from app-side `list.csv` -`system.csv` is the source of truth for items such as system name, mode, -runner tag, queue, queue_group, and scheduler account. +`system.csv` is the source of truth for items such as system name, mode, runner tag, queue, and queue_group. ### 6.4 ポータル層 / Portal Layer diff --git a/docs/guides/add-site.md b/docs/guides/add-site.md index 96baec5..25094cd 100644 --- a/docs/guides/add-site.md +++ b/docs/guides/add-site.md @@ -547,12 +547,12 @@ command_delay = "30s" ### `config/system.csv` にシステムを追加 ```csv -system,mode,tag_build,tag_run,queue,queue_group,account +system,mode,tag_build,tag_run,queue,queue_group # 既存エントリ... # cross モード(ビルドと実行が別ノード) -NewSystem,cross,newsystem_login,newsystem_jacamar,PBS_NewSystem,default, +NewSystem,cross,newsystem_login,newsystem_jacamar,PBS_NewSystem,default # native モード(同一ノードでビルドと実行) -NewSystemLN,native,,newsystem_login,none,default, +NewSystemLN,native,,newsystem_login,none,default ``` - `system`: システム名(アプリの `list.csv` から参照される) @@ -561,7 +561,6 @@ NewSystemLN,native,,newsystem_login,none,default, - `tag_run`: 実行用GitLab Runnerタグ(`native`の場合はbuild_runジョブ用) - `queue`: `config/queue.csv` のキュー名(ログインノードは `none`) - `queue_group`: キューグループ名 -- `account`: scheduler account/project。不要な拠点では空欄 ### `config/queue.csv` にキューシステムを追加(必要な場合) @@ -572,10 +571,14 @@ queue,submit_cmd,template PBS_NewSystem,qsub,"-q ${queue_group} -l select=${nodes} -l walltime=${elapse} -W group_list=your_group" ``` -テンプレート内で使える変数:`${queue_group}`, `${account}`, `${nodes}`, `${numproc_node}`, `${nthreads}`, `${elapse}`, `${proc}`(`nodes * numproc_node`), `${cpu_per_node}`, `${gpu_per_node}`, `${cpu_sockets}`(`nodes * cpu_per_node`), `${gpu_cards}`(`nodes * gpu_per_node`) +テンプレート内で使える変数:`${queue_group}`, `${scheduler_extra_args}`, `${nodes}`, `${numproc_node}`, `${nthreads}`, `${elapse}`, `${proc}`(`nodes * numproc_node`), `${cpu_per_node}`, `${gpu_per_node}`, `${cpu_sockets}`(`nodes * cpu_per_node`), `${gpu_cards}`(`nodes * gpu_per_node`) `${cpu_per_node}` と `${gpu_per_node}` は `config/system_info.csv` から取得します。CPU socket 数や GPU card 数を scheduler に明示するサイトでは、`system_info.csv` の値も投入条件に使われます。 +`${scheduler_extra_args}` は site-local な追加投入引数です。例えば scheduler account や project group のように、同じ system でも runner や課題枠ごとに変わる値は OSS repo の `system.csv` や `queue.csv` に固定せず、GitLab CI variable などで `BK_SCHEDULER_EXTRA_ARGS` または `BK_SCHEDULER_EXTRA_ARGS_` として渡します。 + +GitLab CI では scheduler parameter は `matrix_generate.sh` 実行時に展開されるため、この値は対象 runner だけでなく matrix 生成 job からも見える必要があります。機密 token ではなく、scheduler に表示されてもよい project/account/group 引数だけに使ってください。 + ### `config/system_info.csv` に表示用メタデータを追加 Result Server や比較画面でシステム情報を見せるために、`system_info.csv` にも同じ system を追加します。 diff --git a/scripts/job_functions.sh b/scripts/job_functions.sh index fe91fec..4ffa69c 100644 --- a/scripts/job_functions.sh +++ b/scripts/job_functions.sh @@ -64,11 +64,19 @@ get_system_queue_group() { return 0 } -# System_CSVからscheduler accountを取得する -# Account-less systems return an empty string. -get_system_account() { +# Return optional site-local scheduler arguments. +# +# These values are intentionally not stored in system.csv/queue.csv because +# project/account/group options can vary by deployment or runner. The variable +# must be visible where matrix_generate.sh runs, because generated GitLab jobs +# embed SCHEDULER_PARAMETERS before the target runner submits the scheduler job. +get_scheduler_extra_args() { local system="$1" - awk -F, -v s="$system" '$1==s {print $7}' "$SYSTEM_FILE" + local system_key + local system_var + system_key=$(printf '%s' "$system" | tr -c '[:alnum:]_' '_') + system_var="BK_SCHEDULER_EXTRA_ARGS_${system_key}" + printf '%s\n' "${!system_var:-${BK_SCHEDULER_EXTRA_ARGS:-}}" return 0 } diff --git a/scripts/matrix_generate.sh b/scripts/matrix_generate.sh index 10383e5..84b8152 100644 --- a/scripts/matrix_generate.sh +++ b/scripts/matrix_generate.sh @@ -69,7 +69,6 @@ for listfile in programs/*/list.csv; do mode=$(get_system_mode "$system") queue_group=$(get_system_queue_group "$system") - account=$(get_system_account "$system") # Skip if mode or queue_group is empty (system not found in System_CSV) if [[ -z "$mode" || -z "$queue_group" ]]; then @@ -89,7 +88,8 @@ for listfile in programs/*/list.csv; do [[ "$gpu_per_node" =~ ^[0-9]+$ ]] || gpu_per_node=0 cpu_sockets=$((nodes * cpu_per_node)) gpu_cards=$((nodes * gpu_per_node)) - export elapse nodes queue_group account numproc_node nthreads proc cpu_per_node gpu_per_node cpu_sockets gpu_cards + scheduler_extra_args=$(get_scheduler_extra_args "$system") + export elapse nodes queue_group scheduler_extra_args numproc_node nthreads proc cpu_per_node gpu_per_node cpu_sockets gpu_cards read -r submit_cmd template <<< "$(get_queue_template "$system")" if [[ -z "$submit_cmd" || -z "$template" ]]; then diff --git a/scripts/test_submit.sh b/scripts/test_submit.sh index 577e65c..3e77c4d 100644 --- a/scripts/test_submit.sh +++ b/scripts/test_submit.sh @@ -85,15 +85,19 @@ fi # --- System_CSV から mode/queue_group を取得 --- mode=$(get_system_mode "$system") queue_group=$(get_system_queue_group "$system") -account=$(get_system_account "$system") +scheduler_extra_args=$(get_scheduler_extra_args "$system") +read -r -a scheduler_extra_args_array <<< "$scheduler_extra_args" # --- 選択された設定を表示 --- echo "Selected configuration from $list_file (line $list_csv_line_num):" echo " $line" echo "" echo "Parsed values:" -echo " system=$system, enable=$enable, mode=$mode (from system.csv), queue_group=$queue_group (from system.csv), account=${account:-}" +echo " system=$system, enable=$enable, mode=$mode (from system.csv), queue_group=$queue_group (from system.csv)" echo " nodes=$nodes, numproc_node=$numproc_node, nthreads=$nthreads, elapse=$elapse" +if [[ -n "$scheduler_extra_args" ]]; then + echo " scheduler_extra_args=$scheduler_extra_args (from BK_SCHEDULER_EXTRA_ARGS or BK_SCHEDULER_EXTRA_ARGS_${system})" +fi # --- 投入用スクリプト作成 --- echo cd $PWD > script.sh @@ -222,14 +226,10 @@ case "$system" in -l elapstim_req=${elapse} -v OMP_NUM_THREADS=${nthreads} script.sh ;; RIKYU) - if [[ -z "$account" ]]; then - echo "Error: RIKYU requires an account in config/system.csv" >&2 - exit 1 - fi proc=$((nodes * numproc_node)) - echo sbatch -p $queue_group --account=$account -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ + echo sbatch -p $queue_group "${scheduler_extra_args_array[@]}" -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ --wrap="bash programs/$code/run.sh $system $nodes $numproc_node $nthreads" - sbatch -p $queue_group --account=$account -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ + sbatch -p $queue_group "${scheduler_extra_args_array[@]}" -N $nodes -t $elapse --ntasks-per-node=${numproc_node} --cpus-per-task=$nthreads --gpus=$proc \ --wrap="bash programs/${code}/run.sh $system $nodes $numproc_node $nthreads" ;; RC_GH200|RC_DGXSP|RC_GENOA|RC_FX700)