Skip to content

Commit 74ffd59

Browse files
hschellmanhschellman
committed
fix bug with json readin
1 parent 0450498 commit 74ffd59

7 files changed

Lines changed: 79 additions & 61 deletions

File tree

‎_includes/extractor_new.py‎

Lines changed: 21 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -374,11 +374,30 @@ def main():
374374
else:
375375
mddict = {}
376376
mddict['name']=os.path.basename(args.infile)
377-
377+
mddict['size'] = os.path.getsize(args.infile)
378+
mddict['created_by'] = os.environ['USERF']
378379
mddict['metadata']={}
379380
print ("EXTRACTOR: building metadata from parent and args as no artroot dump available")
380381
# If --input_json is supplied, open that dict now and add it to the output json
382+
if args.input_json != None:
383+
if os.path.exists(args.input_json):
384+
try:
385+
arbjson = json.load(open(args.input_json,'r'))
386+
if DEBUG: print ("EXTRACTOR: arbjson",arbjson)
387+
388+
for key,newval in arbjson["metadata"].items():
381389

390+
if key in mddict["metadata"]:
391+
if DEBUG: print ("EXTRACTOR: overriding ",key,mddict["metadata"][key],"with", newval, "from json file" )
392+
else:
393+
if DEBUG: print ("EXTRACTOR: adding ",key, newval, "from json file" )
394+
395+
mddict["metadata"][key] = newval
396+
except:
397+
print('Error loading input json file.',args.input_json)
398+
399+
else:
400+
print('warning, could not open the input json file', args.input_json)
382401

383402
if args.appname != None:
384403
mddict['metadata']['core.application.name'] = args.appname
@@ -445,27 +464,7 @@ def main():
445464
mddict['namespace']=args.namespace
446465

447466

448-
if args.input_json != None:
449-
if os.path.exists(args.input_json):
450-
try:
451-
arbjson = json.load(open(args.input_json,'r'))
452-
print ("EXTRACTOR: arbjson",arbjson)
453-
#arbjson.pop('name')
454-
#arbjson.pop('namespace')
455-
if DEBUG: "got here"
456-
for key,val in arbjson["metadata"].items():
457-
458-
if DEBUG: print (key, val)
459-
newval = os.path.expandvars(val)
460-
if DEBUG: print (newval)
461-
if key in mddict["metadata"]:
462-
print ("EXTRACTOR: overriding ",key,mddict["metadata"][key],"with", newval, "from json file" )
463-
mddict["metadata"][key] = newval
464-
except:
465-
print('Error loading input json file.',args.input_json)
466-
467-
else:
468-
print('warning, could not open the input json file', args.input_json)
467+
469468

470469
except TypeError:
471470
print('You have not implemented a defineMetaData function by providing an experiment.')

‎_includes/makercds.sh‎

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,22 @@
11
# give me the directory name as argument
22
echo "----------------------------------------------------------------"
3-
echo "makercds.sh"
4-
echo "first ensure you have a justin token"
5-
justin time
6-
justin get-token
7-
export HERE=`pwd`
8-
# put the tar file on a bigger disk
9-
export THERE=/exp/dune/data/users/$USER/
10-
date
11-
ls -lrt $THERE/$1.tar.gz
12-
echo " upload tar file to cvmfs and store location in cvmfs.location file"
13-
export INPUT_TAR_DIR_LOCAL=`justin-cvmfs-upload $THERE/$1.tar.gz`
14-
echo "file uploaded to $INPUT_TAR_DIR_LOCAL"
15-
echo $INPUT_TAR_DIR_LOCAL > $HERE/cvmfs.location
16-
echo "return to previous directory"
17-
cd $HERE
18-
echo "----------------------------------------------------------------"
3+
echo "makercds.sh $1"
4+
if [ "$1" == "" ] ; then
5+
echo "need to enter the directory name that was used for your tar file"
6+
else
7+
echo "first ensure you have a justin token"
8+
justin time
9+
justin get-token
10+
export HERE=`pwd`
11+
# put the tar file on a bigger disk
12+
export THERE=/exp/dune/data/users/$USER/
13+
date
14+
ls -lrt $THERE/$1.tar.gz
15+
echo " upload tar file to cvmfs and store location in cvmfs.location file"
16+
export INPUT_TAR_DIR_LOCAL=`justin-cvmfs-upload $THERE/$1.tar.gz`
17+
echo "file uploaded to $INPUT_TAR_DIR_LOCAL"
18+
echo $INPUT_TAR_DIR_LOCAL > $HERE/cvmfs.location
19+
echo "return to previous directory"
20+
cd $HERE
21+
echo "----------------------------------------------------------------"
22+
fi

‎_includes/maketar.sh‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,19 @@ export HERE=`pwd`
66
export THERE=/exp/dune/data/users/$USER/
77
cd .. # go up one from current directory
88
date
9-
echo " make tar file"
10-
tar --exclude '.git' --exclude build_slf7.x86_64 -cf $THERE/$1.tar $1
11-
date
12-
echo " gzip step "
13-
gzip -f $THERE/$1.tar
14-
date
15-
echo " tar file is at $THERE/$1.tar.gz"
16-
echo "----------------------------------------------------------------"
9+
if [ "$1" == "" ] ; then
10+
echo "need to enter the directory name and be in that directory"
11+
else
12+
echo " make tar file from $1, excluding build_slf7... "
13+
tarname=$(basename $1)
14+
echo "$tarname"
15+
tar --exclude '.git' --exclude build_slf7.x86_64 -cf ${THERE}/${tarname}.tar $1
16+
date
17+
echo " gzip step "
18+
gzip -f $THERE/${tarname}.tar
19+
date
20+
echo " tar file is at $THERE/${tarname}.tar.gz"
21+
cd $HERE
22+
echo "----------------------------------------------------------------"
23+
fi
24+

‎_includes/submit_local_code.jobscript.sh‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,8 @@ lar -c ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/$FCL_FILE $events_option -o ${outFile
133133
)
134134

135135
larExit=$?
136+
137+
136138
# Subshell exits with exit code of last command
137139

138140

@@ -149,9 +151,11 @@ echo "--------make metadata---------"
149151

150152
#sam_metadata_dumper ${outFile1}
151153

152-
echo "python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=${outFile1} --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='full-reconstructed' --file_format='artroot' --fcl_file=${FCL_FILE} --namespace=${NAMESPACE} # > $outFile1.json"
154+
FCL_FILE_NAME=$(basename $FCL_FILE)
155+
156+
echo "python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=${outFile1} --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='full-reconstructed' --file_format='artroot' --fcl_file=${FCL_FILE_NAME} --namespace=${NAMESPACE} # > $outFile1.json"
153157

154-
python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile1 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='full-reconstructed' --file_format='artroot' --fcl_file=${FCL_FILE} --namespace=${NAMESPACE} #> $outFile1.json
158+
python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile1 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='full-reconstructed' --file_format='artroot' --fcl_file=${FCL_FILE_NAME} --namespace=${NAMESPACE} #> $outFile1.json
155159

156160
file1Exit=$?
157161

@@ -162,9 +166,9 @@ echo "------------ non-artroot metadata -----------"
162166

163167
oldjson=${outFile1}.json
164168

165-
echo " python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile2 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='root-tuple' --file_format='root' --fcl_file=${FCL_FILE} --no_extract --input_json=${PWD}/${oldjson} --namespace=${NAMESPACE} # > ${outFile2}.json"
169+
echo " python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile2 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='root-tuple' --file_format='root' --fcl_file=${FCL_FILE_NAME} --no_extract --input_json=${PWD}/${oldjson} --namespace=${NAMESPACE} # > ${outFile2}.json"
166170

167-
python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile2 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='root-tuple' --file_format='root' --fcl_file=${FCL_FILE} --no_extract --input_json=${PWD}/${oldjson} --namespace=${NAMESPACE} # > ${outFile2}.json
171+
python ${INPUT_TAR_DIR_LOCAL}/${DIRECTORY}/extractor_new.py --infile=$outFile2 --appversion=$DUNE_VERSION --appname=${APP_TAG} --appfamily=larsoft --no_crc --inputDidsFile=justin-input-dids.txt --data_tier='root-tuple' --file_format='root' --fcl_file=${FCL_FILE_NAME} --no_extract --input_json=${PWD}/${oldjson} --namespace=${NAMESPACE} # > ${outFile2}.json
168172

169173
file2Exit=$?
170174

‎_includes/submit_workflow.sh‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,12 @@ echo "DESCRIPTION=$DESCRIPTION"
1616
echo "INPUT_TAR_DIR_LOCAL=$INPUT_TAR_DIR_LOCAL"
1717
echo "NAMESPACE=${NAMESPACE}"
1818

19-
20-
echo "---- do the submission ----"
21-
justin simple-workflow \
22-
--mql "$MQL" \
23-
--jobscript submit_local_code.jobscript.sh --rss-mb 4000 \
24-
--output-pattern "*.root:${FNALURL}/${USERF}" --output-pattern "*.root.json:${FNALURL}/${USERF}" --env APP_TAG=${APP_TAG} --env DIRECTORY=${DIRECTORY} --scope ${NAMESPACE} --lifetime 30 --env INPUT_TAR_DIR_LOCAL=${INPUT_TAR_DIR_LOCAL} --env DUNE_VERSION=${DUNE_VERSION} --env DUNE_QUALIFIER=${DUNE_QUALIFIER} --env FCL_FILE=${FCL_FILE} --env NUM_EVENTS=${NUM_EVENTS} --env USERF=${USERF} --env NAMESPACE=${NAMESPACE} --description "${DESCRIPTION}"
19+
if test -e "./${FCL_FILE}";
20+
echo "---- do the submission ----"
21+
justin simple-workflow \
22+
--mql "$MQL" \
23+
--jobscript submit_local_code.jobscript.sh --rss-mb 4000 \
24+
--output-pattern "*.root:${FNALURL}/${USERF}" --output-pattern "*.root.json:${FNALURL}/${USERF}" --env APP_TAG=${APP_TAG} --env DIRECTORY=${DIRECTORY} --scope ${NAMESPACE} --lifetime 30 --env INPUT_TAR_DIR_LOCAL=${INPUT_TAR_DIR_LOCAL} --env DUNE_VERSION=${DUNE_VERSION} --env DUNE_QUALIFIER=${DUNE_QUALIFIER} --env FCL_FILE=${FCL_FILE} --env NUM_EVENTS=${NUM_EVENTS} --env USERF=${USERF} --env NAMESPACE=${NAMESPACE} --description "${DESCRIPTION}"
25+
else
26+
echo "FCL_FILE must be in $DIRECTORY for now"
27+
fi

‎_includes/test_workflow.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# actual submission
22
#export INPUT_TAR_DIR_LOCAL=`cat cvmfs.location`
33
export INPUT_TAR_DIR_LOCAL=$DUNEDATA
4-
source $DIRECTORY/job_config.sh
4+
source ./job_config.sh
55

66
export NUM_EVENTS=2
77
echo "DIRECTORY=$DIRECTORY"
@@ -18,9 +18,9 @@ echo "INPUT_TAR_DIR_LOCAL=$INPUT_TAR_DIR_LOCAL"
1818

1919
echo "tardir $INPUT_TAR_DIR_LOCAL"
2020
export HERE=$PWD
21-
echo " go up one directory "
22-
cd ..
21+
#echo " go up one directory "
22+
2323
justin-test-jobscript \
2424
--mql "$MQL" \
25-
--jobscript $DIRECTORY/submit_local_code.jobscript.sh --env PROCESS_TYPE=${PROCESS_TYPE} --env DIRECTORY=${DIRECTORY} --env INPUT_TAR_DIR_LOCAL=${INPUT_TAR_DIR_LOCAL} --env DUNE_VERSION=${DUNE_VERSION} --env DUNE_QUALIFIER=${DUNE_QUALIFIER} --env FCL_FILE=${FCL_FILE} --env NUM_EVENTS=${NUM_EVENTS} --env USERF=${USER} --env APP_TAG=${APP_TAG} --env NAMESPACE=${NAMESPACE}
26-
cd $HERE
25+
--jobscript submit_local_code.jobscript.sh --env PROCESS_TYPE=${PROCESS_TYPE} --env DIRECTORY=${DIRECTORY} --env INPUT_TAR_DIR_LOCAL=${INPUT_TAR_DIR_LOCAL} --env DUNE_VERSION=${DUNE_VERSION} --env DUNE_QUALIFIER=${DUNE_QUALIFIER} --env FCL_FILE=${FCL_FILE} --env NUM_EVENTS=${NUM_EVENTS} --env USERF=${USER} --env APP_TAG=${APP_TAG} --env NAMESPACE=${NAMESPACE}
26+
#cd $HERE

‎files/usefulcode.tar‎

512 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)