v492 dev23

docs/changes/157.maintenance.md

@@ -0,0 +1,5 @@
+- Added support for `ANALYSETABLESXGBTRAIN` in IRF production, including fixed zenith, NSB, and wobble grids for XGB training workflows.
+- Updated XGB-related helper scripts to use the new `DispXGBs` model directory layout and simplified XGB stereo training output paths.
+- Improved anasum job reproducibility by copying runlists into the combine log directory and using that pinned copy in submitted jobs.
+- Hardened runlist and parameter parsing by normalizing carriage returns in runlists and fixing `ENERGYBINS` regex handling in TMVA scripts.
+- Removed the legacy `scripts/ANALYSIS.anasum.sh` entrypoint and added a small robustness fix to fail fast when XGB training output directories cannot be created.

scripts/ANALYSIS.anasum_combine.sh

@@ -72,6 +72,7 @@ DATE=$(date +"%y%m%d")
 LOGDIR="$VERITAS_USER_LOG_DIR/ANASUM.COMBINE-${DATE}-$(uuidgen)"
 mkdir -p "$LOGDIR"
 echo -e "Log files will be written to:\n $LOGDIR"
+cp -f "$RUNLIST" "$LOGDIR/" || { echo "Error: failed to copy runlist '$RUNLIST' to '$LOGDIR'"; exit 1; }
 
 # Job submission script
 SUBSCRIPT="$(dirname "$0")/helper_scripts/ANALYSIS.anasum_combine_sub"
@@ -88,7 +89,7 @@ if [[ $NANASUMFILES -eq 0 ]]; then
 fi
 echo "Found $NANASUMFILES anasum output files in $DDIR"
 
-sed -e "s|RRUNLIST|$RUNLIST|" \
+sed -e "s|RRUNLIST|$LOGDIR/$(basename "$RUNLIST")|" \
     -e "s|DDDIR|$DDIR|" \
     -e "s|RRUNP|$RUNP|" \
     -e "s|OOUTFILE|$OUTFILE|" "$SUBSCRIPT.sh" > "$FSCRIPT.sh"

scripts/ANALYSIS.anasum_parallel_from_runlist.sh

@@ -201,7 +201,7 @@ if [[ ! -f "$RUNLIST" ]]; then
     echo "Error, runlist $RUNLIST not found, exiting..."
     exit 1
 fi
-RUNS=$(cat "$RUNLIST" | sort -u)
+mapfile -t RUNS < <(tr -d '\r' < "$RUNLIST" | sort -u)
 NRUNS=$(cat "$RUNLIST" | sort -u | wc -l)
 echo "total number of runs to analyze: $NRUNS"
 

scripts/IRF.generate_effective_area_parts.sh

@@ -69,10 +69,10 @@ SIMTYPE="$8"
 ANALYSIS_TYPE="${9:-}"
 DISPBDT="${10:-0}"
 UUID="${11:-$(date +"%y%m%d")-$(uuidgen)}"
-XGBSTEREOFILESUFFIX="xgb_stereo"
+XGBSTEREOFILESUFFIX="None"
 XGBGAMMAHADRONFILESUFFIX="None"
 
-echo "IRF.generate_effective_area_parts for epoch $EPOCH, atmo $ATM, zenith $ZA, wobble $WOBBLE, noise $NOISE (DISP: $DISPBDT, XGB $XGBSTEREOFILESUFFIX $XGBGAMMAHADRONFILESUFFIX)"
+echo "IRF.generate_effective_area_parts for epoch $EPOCH, atmo $ATM, zenith $ZA, wobble $WOBBLE, noise $NOISE (DISP: $DISPBDT)"
 
 
 if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
@@ -103,6 +103,12 @@ CUTS_NAME=${CUTS_NAME%%.dat}
 echo "Cuts: $CUTSFILE $CUTS_NAME"
 echo "MC file: $MCFILE"
 echo "Eff area file: $EFFAREAFILE"
+if [[ $CUTS_NAME == *XGB* ]]; then
+  XGBSTEREOFILESUFFIX="xgb_stereo"
+  XGBGAMMAHADRONFILESUFFIX="None"   # keep until development is finalized
+fi
+echo "XGB: $XGBSTEREOFILESUFFIX $XGBGAMMAHADRONFILESUFFIX"
+
 # run script
 SUBSCRIPT="$(dirname "$0")/helper_scripts/IRF.effective_area_parallel_sub"
 FSCRIPT="$LOGDIR/EA.ID${RECID}.${ZA}.${WOBBLE}.${NOISE}.${CUTS_NAME}.sh"

scripts/IRF.optimizeTMVAforGammaHadronSeparation.sh

@@ -83,7 +83,7 @@ fi
 #####################################
 # energy bins
 count1=1
-if grep -q "^\* ENERGYBINS" "$RUNPAR"; then
+if grep -q "^[*] ENERGYBINS" "$RUNPAR"; then
     ENBINS=$(awk '$1 == "*" && $2 == "ENERGYBINS" { for (i = 3; i <= NF; i++) print $i; exit }' "$RUNPAR")
     mapfile -t EBINARRAY <<< "$ENBINS"
     NENE=$(( ${#EBINARRAY[@]}-count1 )) #get number of bins

scripts/IRF.production.sh

@@ -23,7 +23,7 @@ required parameters:
                             EVNDISP,
                             MAKETABLES, COMBINETABLES,
                             TRAINMVANGRES,
-                            ANALYSETABLES,
+                            ANALYSETABLES, ANALYSETABLESXGBTRAIN
                             TRAINXGBANGRES, ANAXGBANGRES,
                             TRAINXGBGH, ANAXGBGH,
                             PRESELECTEFFECTIVEAREAS, COMBINEPRESELECTEFFECTIVEAREAS,
@@ -190,9 +190,13 @@ elif [[ "${SIMTYPE}" == "CARE_RedHV_Feb2024" ]]; then
     ZENITH_ANGLES=$(find "${SIMDIR}" -mindepth 1 -maxdepth 1 -type d -name "Zd*" -exec basename {} \; | awk -F "Zd" '{print $2}' | grep -v curved | sort | uniq)
     # ZENITH_ANGLES=( 60 65 )
     ze_first_bin=$(printf '%s\n' "$ZENITH_ANGLES" | head -n 1)
-    # Note! Assumes same NSB and WOBBLE offsets for flat and curved atmosphere
     NSB_LEVELS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\2/p' | sort -u)
     WOBBLE_OFFSETS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\1/p' | sort -u)
+    if [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]]; then
+        ZENITH_ANGLES=( 20 30 35 40 45 50 55 60 65 )
+        NSB_LEVELS=( 160 200 350 450 )
+        WOBBLE_OFFSETS=( 0.25 0.5 0.75 1.0 1.25 1.5 1.75 2.0 )
+    fi
     ######################################
     # TEST
     # NSB_LEVELS=( 300 )
@@ -205,6 +209,11 @@ elif [[ "${SIMTYPE}" == "CARE_202404" ]] || [[ "${SIMTYPE}" == "CARE_24_20" ]];
     # assume same NSB and wobble offsets in all bins
     NSB_LEVELS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\2/p' | sort -u)
     WOBBLE_OFFSETS=$(find "${SIMDIR}/Zd${ze_first_bin}" -maxdepth 1 -type f -name "*.zst" -exec basename {} \; | sed -nE 's/.*_([0-9.]+)wob_([0-9]+)MHz.*/\1/p' | sort -u)
+    if [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]]; then
+        ZENITH_ANGLES=( 20 30 35 40 45 50 55 60 65 )
+        NSB_LEVELS=( 160 200 350 450 )
+        WOBBLE_OFFSETS=( 0.25 0.5 0.75 1.0 1.25 1.5 1.75 2.0 )
+    fi
     ######################################
     # TEST
     # ZENITH_ANGLES=( 00 20 30 35 40 45 )
@@ -457,7 +466,7 @@ for VX in $EPOCH; do
                         done #recID
                     ######################
                     # analyse table files
-                    elif [[ $IRFTYPE == "ANALYSETABLES" ]] || [[ $IRFTYPE == "ANATABLESEFFAREAS" ]]; then
+elif [[ $IRFTYPE == "ANALYSETABLES" ]] || [[ $IRFTYPE == "ANALYSETABLESXGBTRAIN" ]] || [[ $IRFTYPE == "ANATABLESEFFAREAS" ]]; then
                         for ID in $RECID; do
                             TFIL="${TABLECOM}"
                             # note: the IDs dependent on what is written in EVNDISP.reconstruction.runparameter

scripts/IRF.trainTMVAforGammaHadronSeparation.sh

@@ -109,7 +109,7 @@ RUNPAR_CONTENT=$(cat "$RUNPAR")
 
 #####################################
 # energy bins
-if echo "$RUNPAR_CONTENT" | grep -q "^\* ENERGYBINS"; then
+if echo "$RUNPAR_CONTENT" | grep -q "^[*] ENERGYBINS"; then
     ENBINS=$(echo "$RUNPAR_CONTENT" | awk '$1 == "*" && $2 == "ENERGYBINS" { for (i = 3; i <= NF; i++) print $i; exit }')
     mapfile -t EBINARRAY <<< "$ENBINS"
     count1=1

scripts/IRF.trainXGBforAngularReconstruction.sh

@@ -62,7 +62,7 @@ if [[ -z "$VERITAS_IRFPRODUCTION_DIR" ]]; then
     exit 1
 fi
 # output and log directories
-ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TrainXGBStereoAnalysisBinned/${ZA}/"
+ODIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/$SIMTYPE/${EPOCH}_ATM${ATM}_gamma/TrainXGBStereoAnalysis/${ZA}/"
 LOGDIR="$VERITAS_IRFPRODUCTION_DIR/$EDVERSION/${ANALYSIS_TYPE}/${SIMTYPE}/${EPOCH}_ATM${ATM}_gamma/submit-trainXGBStereoAnalysis-RECID${RECID}-${UUID}"
 mkdir -p "$ODIR"
 chmod g+w "$ODIR"

scripts/helper_scripts/ANALYSIS.dispXGB_sub.sh

@@ -58,7 +58,7 @@ ZA=$(echo "$RUNINFO" | awk '{print $8}')
 EPOCH=$(echo "$RUNINFO" | awk '{print $1}')
 ATM=$(echo "$RUNINFO" | awk '{print $3}')
 echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg, epoch ${EPOCH}, ATM ${ATM}"
-DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}"
+DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}"
 if [[ ! -d "${DISPDIR}" ]]; then
     echo "Error finding model directory $DISPDIR"
     exit

scripts/helper_scripts/IRF.dispXGB_sub.sh

@@ -43,7 +43,7 @@ ZA=$(echo "$RUNINFO" | awk '{print $8}')
 EPOCH=$(echo "$RUNINFO" | awk '{print $1}')
 ATM=$(echo "$RUNINFO" | awk '{print $3}')
 echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg, epoch ${EPOCH}, ATM ${ATM}"
-DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}"
+DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}"
 if [[ ! -d "${DISPDIR}" ]]; then
     echo "Error finding model directory $DISPDIR"
     exit

scripts/helper_scripts/IRF.mscw_energy_MC_sub.sh

@@ -271,7 +271,7 @@ run_xgb()
     ZA=${ZA%deg}
     echo "MSCW file: ${MSCW_FILE} at zenith ${ZA} deg"
 
-    DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGB/${ANATYPE}/${EPOCH}_ATM${ATM}/"
+    DISPDIR="$VERITAS_EVNDISP_AUX_DIR/DispXGBs/${ANATYPE}/${EPOCH}_ATM${ATM}/"
     STEREO_PAR="$VERITAS_EVNDISP_AUX_DIR/ParameterFiles/XGB-stereo-parameter.json"
     BIN_ID=$(jq -r --arg za "$ZA" '
       .zenith[]

scripts/helper_scripts/IRF.trainXGBforAngularReconstruction_sub.sh

@@ -23,7 +23,7 @@ fi
 echo "Scratch dir: $TEMPDIR"
 mkdir -p "$TEMPDIR"
 
-mkdir -p "${ODIR}"
+mkdir -p "${ODIR}" || exit 1
 echo -e "Output files will be written to:\n ${ODIR}"
 
 # shellcheck source=scripts/helper_scripts/UTILITY.conda_env.sh