diff --git a/docs/changes/157.maintenance.md b/docs/changes/157.maintenance.md new file mode 100644 index 0000000..b807b3c --- /dev/null +++ b/docs/changes/157.maintenance.md @@ -0,0 +1,5 @@ +- Added support for `ANALYSETABLESXGBTRAIN` in IRF production, including fixed zenith, NSB, and wobble grids for XGB training workflows. +- Updated XGB-related helper scripts to use the new `DispXGBs` model directory layout and simplified XGB stereo training output paths. +- Improved anasum job reproducibility by copying runlists into the combine log directory and using that pinned copy in submitted jobs. +- Hardened runlist and parameter parsing by normalizing carriage returns in runlists and fixing `ENERGYBINS` regex handling in TMVA scripts. +- Removed the legacy `scripts/ANALYSIS.anasum.sh` entrypoint and added a small robustness fix to fail fast when XGB training output directories cannot be created. diff --git a/scripts/ANALYSIS.anasum.sh b/scripts/ANALYSIS.anasum.sh deleted file mode 100755 index f3ea5a5..0000000 --- a/scripts/ANALYSIS.anasum.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash -# script to analyse data files with anasum - -# qsub parameters -# shellcheck disable=SC2034 # SGE resource directives, read by job scheduler -h_cpu=8:00:00; h_vmem=4000M; tmpdir_size=10G -# shellcheck source=scripts/helper_scripts/UTILITY.submitJob.sh -source "$(dirname "$0")/helper_scripts/UTILITY.submitJob.sh" - -if [[ $# -lt 3 ]]; then -# begin help message -echo " -ANASUM data analysis: submit jobs from an anasum run list - -ANALYSIS.anasum.sh - [run parameter file] [mscw directory] - -required parameters: - - full anasum run list - - anasum output files are written to this directory - - name of combined anasum file - (written to same location as anasum files) - -optional parameters: - - [run parameter file] anasum run parameter file (located in - \$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/; - default is ANASUM.runparameter) - - [mscw directory] directory containing the mscw.root files - --------------------------------------------------------------------------------- -" -#end help message -exit -fi - -# Run init script -bash "$(dirname "$0")/helper_scripts/UTILITY.script_init.sh" || exit 1 - -# Parse command line arguments -FLIST=$1 -ODIR=$2 -ONAME=$3 -ONAME=${ONAME%%.root} -[[ "$4" ]] && RUNP=$4 || RUNP="ANASUM.runparameter" -[[ "$5" ]] && INDIR=$5 || INDIR="$VERITAS_USER_DATA_DIR/analysis/Results/$EDVERSION/RecID0" - -# Check that run list exists -if [[ ! -f "$FLIST" ]]; then - echo "Error, anasum runlist $FLIST not found, exiting..." - exit 1 -fi - -# Check that run parameter file exists -if [[ "$RUNP" == $(basename "$RUNP") ]]; then - RUNP="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/$RUNP" -fi -if [[ ! -f "$RUNP" ]]; then - echo "Error, anasum run parameter file not found, exiting..." - exit 1 -fi - -# directory for run scripts -DATE=$(date +"%y%m%d") -LOGDIR="$VERITAS_USER_LOG_DIR/$DATE/ANASUM.ANADATA" -echo -e "Log files will be written to:\n $LOGDIR" -mkdir -p "$LOGDIR" - -# output directory -echo -e "Output files will be written to:\n $ODIR" -mkdir -p "$ODIR" - -# Job submission script -SUBSCRIPT="$(dirname "$0")/helper_scripts/ANALYSIS.anasum_sub" - -TIMETAG=$(date +"%s") - -FSCRIPT="$LOGDIR/ANA.$ONAME-$(date +%s)" -sed -e "s|FILELIST|$FLIST|" \ - -e "s|DATADIR|$INDIR|" \ - -e "s|OUTDIR|$ODIR|" \ - -e "s|OUTNAME|$ONAME|" \ - -e "s|RUNPARAM|$RUNP|" "$SUBSCRIPT.sh" > "$FSCRIPT.sh" - -chmod u+x "$FSCRIPT.sh" -echo "$FSCRIPT.sh" - -SUBC=$("$(dirname "$0")/helper_scripts/UTILITY.readSubmissionCommand.sh") -# run locally or on cluster (expand shell variables in submission string) -SUBC=$(eval "echo \"$SUBC\"") -if [[ $SUBC == *"ERROR"* ]]; then - echo "$SUBC" - exit -fi -submit_job "$FSCRIPT.sh" "$FSCRIPT.sh &> $FSCRIPT.log" "$LOGDIR/runscripts.$TIMETAG.dat" -if [[ $SUBC == *qsub* ]]; then - echo "RUN $AFILE JOBID $JOBID" -fi -run_parallel_jobs "$LOGDIR/runscripts.$TIMETAG.dat" diff --git a/scripts/ANALYSIS.anasum_combine.sh b/scripts/ANALYSIS.anasum_combine.sh index 820071f..944b7fb 100755 --- a/scripts/ANALYSIS.anasum_combine.sh +++ b/scripts/ANALYSIS.anasum_combine.sh @@ -72,6 +72,7 @@ DATE=$(date +"%y%m%d") LOGDIR="$VERITAS_USER_LOG_DIR/ANASUM.COMBINE-${DATE}-$(uuidgen)" mkdir -p "$LOGDIR" echo -e "Log files will be written to:\n $LOGDIR" +cp -f "$RUNLIST" "$LOGDIR/" || { echo "Error: failed to copy runlist '$RUNLIST' to '$LOGDIR'"; exit 1; } # Job submission script SUBSCRIPT="$(dirname "$0")/helper_scripts/ANALYSIS.anasum_combine_sub" @@ -88,7 +89,7 @@ if [[ $NANASUMFILES -eq 0 ]]; then fi echo "Found $NANASUMFILES anasum output files in $DDIR" -sed -e "s|RRUNLIST|$RUNLIST|" \ +sed -e "s|RRUNLIST|$LOGDIR/$(basename "$RUNLIST")|" \ -e "s|DDDIR|$DDIR|" \ -e "s|RRUNP|$RUNP|" \ -e "s|OOUTFILE|$OUTFILE|" "$SUBSCRIPT.sh" > "$FSCRIPT.sh" diff --git a/scripts/ANALYSIS.anasum_parallel_from_runlist.sh b/scripts/ANALYSIS.anasum_parallel_from_runlist.sh index b25ced0..ef7f969 100755 --- a/scripts/ANALYSIS.anasum_parallel_from_runlist.sh +++ b/scripts/ANALYSIS.anasum_parallel_from_runlist.sh @@ -201,7 +201,7 @@ if [[ ! -f "$RUNLIST" ]]; then echo "Error, runlist $RUNLIST not found, exiting..." exit 1 fi -RUNS=$(cat "$RUNLIST" | sort -u) +mapfile -t RUNS < <(tr -d '\r' < "$RUNLIST" | sort -u) NRUNS=$(cat "$RUNLIST" | sort -u | wc -l) echo "total number of runs to analyze: $NRUNS" diff --git a/scripts/IRF.generate_effective_area_parts.sh b/scripts/IRF.generate_effective_area_parts.sh index 389ee7b..078fbd9 100755 --- a/scripts/IRF.generate_effective_area_parts.sh +++ b/scripts/IRF.generate_effective_area_parts.sh @@ -69,10 +69,10 @@ SIMTYPE="$8" ANALYSIS_TYPE="${9:-}" DISPBDT="${10:-0}" UUID="${11:-$(date +"%y%m%d")-$(uuidgen)}" -XGBSTEREOFILESUFFIX="xgb_stereo" +XGBSTEREOFILESUFFIX="None" XGBGAMMAHADRONFILESUFFIX="None" -echo "IRF.generate_effective_area_parts for epoch $EPOCH, atmo $ATM, zenith $ZA, wobble $WOBBLE, noise $NOISE (DISP: $DISPBDT, XGB $XGBSTEREOFILESUFFIX $XGBGAMMAHADRONFILESUFFIX)" +echo "IRF.generate_effective_area_parts for epoch $EPOCH, atmo $ATM, zenith $ZA, wobble $WOBBLE, noise $NOISE (DISP: $DISPBDT)" if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then @@ -103,6 +103,12 @@ CUTS_NAME=${CUTS_NAME%%.dat} echo "Cuts: $CUTSFILE $CUTS_NAME" echo "MC file: $MCFILE" echo "Eff area file: $EFFAREAFILE" +if [[ $CUTS_NAME == *XGB* ]]; then + XGBSTEREOFILESUFFIX="xgb_stereo" + XGBGAMMAHADRONFILESUFFIX="None" # keep until development is finalized +fi +echo "XGB: $XGBSTEREOFILESUFFIX $XGBGAMMAHADRONFILESUFFIX" + # run script SUBSCRIPT="$(dirname "$0")/helper_scripts/IRF.effective_area_parallel_sub" FSCRIPT="$LOGDIR/EA.ID${RECID}.${ZA}.${WOBBLE}.${NOISE}.${CUTS_NAME}.sh" diff --git a/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh b/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh index 18e5a29..3fae898 100755 --- a/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh +++ b/scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh @@ -83,7 +83,7 @@ fi ##################################### # energy bins count1=1 -if grep -q "^\* ENERGYBINS" "$RUNPAR"; then +if grep -q "^[*] ENERGYBINS" "$RUNPAR"; then ENBINS=$(awk '$1 == "*" && $2 == "ENERGYBINS" { for (i = 3; i <= NF; i++) print $i; exit }' "$RUNPAR") mapfile -t EBINARRAY <<< "$ENBINS" NENE=$(( ${#EBINARRAY[@]}-count1 )) #get number of bins diff --git a/scripts/IRF.production.sh b/scripts/IRF.production.sh index bd47d1b..da911ce 100755 --- a/scripts/IRF.production.sh +++ b/scripts/IRF.production.sh @@ -23,7 +23,7 @@ required parameters: EVNDISP, MAKETABLES, COMBINETABLES, TRAINMVANGRES, - ANALYSETABLES, + ANALYSETABLES, ANALYSETABLESXGBTRAIN TRAINXGBANGRES, ANAXGBANGRES, TRAINXGBGH, ANAXGBGH, PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS, @@ -190,9 +190,13 @@ elif [[ "${SIMTYPE}" == "CARE_RedHV_Feb2024" ]]; then ZENITH_ANGLES=$(find "${SIMDIR}" -mindepth 1 -maxdepth 1 -type d -name "Zd*" -exec basename {} \; | awk -F "Zd" '{print $2}' | grep -v curved | sort | uniq) # ZENITH_ANGLES=( 60 65 ) ze_first_bin=$(printf '%s\n' "$ZENITH_ANGLES" | head -n 1) - # Note! Assumes same NSB and WOBBLE offsets for flat and curved atmosphere NSB_LEVELS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\2/p' | sort -u) WOBBLE_OFFSETS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\1/p' | sort -u) + if [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]]; then + ZENITH_ANGLES=( 20 30 35 40 45 50 55 60 65 ) + NSB_LEVELS=( 160 200 350 450 ) + WOBBLE_OFFSETS=( 0.25 0.5 0.75 1.0 1.25 1.5 1.75 2.0 ) + fi ###################################### # TEST # NSB_LEVELS=( 300 ) @@ -205,6 +209,11 @@ elif [[ "${SIMTYPE}" == "CARE_202404" ]] || [[ "${SIMTYPE}" == "CARE_24_20" ]]; # assume same NSB and wobble offsets in all bins NSB_LEVELS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\2/p' | sort -u) WOBBLE_OFFSETS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\1/p' | sort -u) + if [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]]; then + ZENITH_ANGLES=( 20 30 35 40 45 50 55 60 65 ) + NSB_LEVELS=( 160 200 350 450 ) + WOBBLE_OFFSETS=( 0.25 0.5 0.75 1.0 1.25 1.5 1.75 2.0 ) + fi ###################################### # TEST # ZENITH_ANGLES=( 00 20 30 35 40 45 ) @@ -457,7 +466,7 @@ for VX in $EPOCH; do done #recID ###################### # analyse table files - elif [[ $IRFTYPE == "ANALYSETABLES" ]] || [[ $IRFTYPE == "ANATABLESEFFAREAS" ]]; then +elif [[ $IRFTYPE == "ANALYSETABLES" ]] || [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]] || [[ $IRFTYPE == "ANATABLESEFFAREAS" ]]; then for ID in $RECID; do TFIL="${TABLECOM}" # note: the IDs dependent on what is written in EVNDISP.reconstruction.runparameter diff --git a/scripts/IRF.trainTMVAforGammaHadronSeparation.sh b/scripts/IRF.trainTMVAforGammaHadronSeparation.sh index 5f83c37..e74a367 100755 --- a/scripts/IRF.trainTMVAforGammaHadronSeparation.sh +++ b/scripts/IRF.trainTMVAforGammaHadronSeparation.sh @@ -109,7 +109,7 @@ RUNPAR_CONTENT=$(cat "$RUNPAR") ##################################### # energy bins -if echo "$RUNPAR_CONTENT" | grep -q "^\* ENERGYBINS"; then +if echo "$RUNPAR_CONTENT" | grep -q "^[*] ENERGYBINS"; then ENBINS=$(echo "$RUNPAR_CONTENT" | awk '$1 == "*" && $2 == "ENERGYBINS" { for (i = 3; i <= NF; i++) print $i; exit }') mapfile -t EBINARRAY <<< "$ENBINS" count1=1 diff --git a/scripts/IRF.trainXGBforAngularReconstruction.sh b/scripts/IRF.trainXGBforAngularReconstruction.sh index 4ed1d96..e38a55a 100755 --- a/scripts/IRF.trainXGBforAngularReconstruction.sh +++ b/scripts/IRF.trainXGBforAngularReconstruction.sh @@ -62,7 +62,7 @@ if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then exit 1 fi # output and log directories -ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TrainXGBStereoAnalysisBinned/${ZA}/" +ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TrainXGBStereoAnalysis/${ZA}/" LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-trainXGBStereoAnalysis-RECID${RECID}-${UUID}" mkdir -p "$ODIR" chmod g+w "$ODIR" diff --git a/scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh b/scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh index de6f6fd..d2a4825 100755 --- a/scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh +++ b/scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh @@ -58,7 +58,7 @@ ZA=$(echo "$RUNINFO" | awk '{print $8}') EPOCH=$(echo "$RUNINFO" | awk '{print $1}') ATM=$(echo "$RUNINFO" | awk '{print $3}') echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg, epoch ${EPOCH}, ATM ${ATM}" -DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}" +DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}" if [[ ! -d "${DISPDIR}" ]]; then echo "Error finding model directory $DISPDIR" exit diff --git a/scripts/helper_scripts/IRF.dispXGB_sub.sh b/scripts/helper_scripts/IRF.dispXGB_sub.sh index 7b10772..4d55cbf 100755 --- a/scripts/helper_scripts/IRF.dispXGB_sub.sh +++ b/scripts/helper_scripts/IRF.dispXGB_sub.sh @@ -43,7 +43,7 @@ ZA=$(echo "$RUNINFO" | awk '{print $8}') EPOCH=$(echo "$RUNINFO" | awk '{print $1}') ATM=$(echo "$RUNINFO" | awk '{print $3}') echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg, epoch ${EPOCH}, ATM ${ATM}" -DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}" +DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}" if [[ ! -d "${DISPDIR}" ]]; then echo "Error finding model directory $DISPDIR" exit diff --git a/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh b/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh index 7724705..22e0260 100755 --- a/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh +++ b/scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh @@ -271,7 +271,7 @@ run_xgb() ZA=${ZA%deg} echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg" - DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}/" + DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}/" STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json" BIN_ID=$(jq -r --arg za "$ZA" ' .zenith[] diff --git a/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh b/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh index 34f7a06..728b4d8 100755 --- a/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh +++ b/scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh @@ -23,7 +23,7 @@ fi echo "Scratch dir: $TEMPDIR" mkdir -p "$TEMPDIR" -mkdir -p "${ODIR}" +mkdir -p "${ODIR}" || exit 1 echo -e "Output files will be written to:\n ${ODIR}" # shellcheck source=scripts/helper_scripts/UTILITY.conda_env.sh