Skip to content

Commit 77481e7

Browse files
djw8605djw8605
committed
Merge branch 'merge_test'
2 parents 4142881 + a06480b commit 77481e7

3 files changed

Lines changed: 182 additions & 7 deletions

File tree

‎bin/stashcp‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,16 @@ DIFF = TIMEOUT * 10
2424
def doStashCpSingle(sourceFile, destination, cache, debug=False):
2525

2626
logging.debug("Checking size of file.")
27-
xrdfs = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "stat", sourceFile], stdout=subprocess.PIPE).communicate()[0]
27+
(xrdfs_stdout, xrdfs_stderr) = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "stat", sourceFile], stdout=subprocess.PIPE).communicate()
2828
xrdcp_version = subprocess.Popen(['echo $(xrdcp -V 2>&1)'], stdout=subprocess.PIPE, shell=True).communicate()[0][:-1]
29-
fileSize = int(re.findall(r"Size: \d+", xrdfs)[0].split(": ")[1])
29+
try:
30+
fileSize = int(re.findall(r"Size: \d+", xrdfs_stdout)[0].split(": ")[1])
31+
except Exception as e:
32+
sys.stderr.write("Unable to find size of file\n")
33+
print str(xrdfs_stdout)
34+
sys.stderr.write(str(xrdfs_stderr))
35+
sys.stderr.write("\n")
36+
return 1
3037
logging.debug("Size of the file %s is %s", sourceFile, str(fileSize))
3138
#cache=get_best_stashcache()
3239
logging.debug("Using Cache %s", cache)
@@ -35,11 +42,13 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
3542

3643
# Fill out the payload as much as possible
3744
filename = destination + '/' + sourceFile.split('/')[-1]
45+
3846
payload = {}
3947
payload['xrdcp_version'] = xrdcp_version
4048
payload['filesize'] = fileSize
4149
payload['filename'] = sourceFile
4250
payload['sitename'] = sitename
51+
payload.update(parse_job_ad())
4352

4453
# Calculate the starting time
4554
start1 = int(time.time()*1000)
@@ -72,7 +81,7 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
7281
payload['cache']="CVMFS"
7382
es_send(payload)
7483

75-
return
84+
return 0
7685

7786
except IOError, e:
7887
logging.error("Unable to copy with CVMFS, even though file exists: %s", str(e))
@@ -97,6 +106,7 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
97106
payload['start2']=start2
98107
payload['end2']=end2
99108

109+
100110
if xrd_exit=='0': #worked first try
101111
logging.debug("Transfer success using %s", cache)
102112
dltime=end2-start2
@@ -139,17 +149,47 @@ def doStashCpSingle(sourceFile, destination, cache, debug=False):
139149
payload['end3']=end3
140150
payload['cache']=cache
141151
es_send(payload)
152+
if xrd_exit == '0':
153+
return 0
154+
else:
155+
return 1
156+
return 0
142157

143158

159+
def parse_job_ad():
160+
"""
161+
Parse the .job.ad file for the Owner (username) and ProjectName of the callee.
162+
"""
163+
temp_list = {}
164+
try:
165+
if '_CONDOR_JOB_AD' in os.environ:
166+
filename = os.environ['_CONDOR_JOB_AD']
167+
elif os.path.exists(".job.ad"):
168+
filename = ".job.ad"
169+
else:
170+
return {}
171+
with open(filename) as job_file:
172+
for line in job_file.readlines():
173+
match = re.search('^\s*(Owner|ProjectName)\s=\s"(.*)"', line, re.IGNORECASE)
174+
if match:
175+
temp_list[match.group()] = line
176+
except IOError, e:
177+
logging.error("Unable to open the .job.ad file")
178+
179+
return temp_list
180+
144181
def dostashcpdirectory(sourceDir, destination, cache, debug=False):
145182
sourceItems = subprocess.Popen(["xrdfs", "root://stash.osgconnect.net", "ls", sourceDir], stdout=subprocess.PIPE).communicate()[0].split()
146183
for remote_file in sourceItems:
147184
command2 = 'xrdfs root://stash.osgconnect.net stat '+ remote_file + ' | grep "IsDir" | wc -l'
148185
isdir=subprocess.Popen([command2],stdout=subprocess.PIPE,shell=True).communicate()[0].split()[0]
149186
if isdir!='0':
150-
dostashcpdirectory(remote_file, destination, cache, debug)
187+
result = dostashcpdirectory(remote_file, destination, cache, debug)
151188
else:
152-
doStashCpSingle(remote_file,destination, cache, debug)
189+
result = doStashCpSingle(remote_file,destination, cache, debug)
190+
# Stop transfers if something fails
191+
if result != 0:
192+
return result
153193

154194

155195
def es_send(payload):
@@ -323,10 +363,13 @@ def main():
323363
logging.warning("DEPRECIATED: The source path does not begin with a '/', but it is required. This functionality will be removed in an upcoming release")
324364
source = "/" + source
325365

366+
326367
if not args.recursive:
327-
doStashCpSingle(sourceFile=source, destination=destination, cache=cache, debug=args.debug)
368+
result = doStashCpSingle(sourceFile=source, destination=destination, cache=cache, debug=args.debug)
328369
else:
329-
dostashcpdirectory(sourceDir = source, destination = destination, cache=cache, debug=args.debug)
370+
result = dostashcpdirectory(sourceDir = source, destination = destination, cache=cache, debug=args.debug)
371+
# Exit with failure
372+
sys.exit(result)
330373

331374

332375
if __name__ == "__main__":

‎bin/stashcp2/tests/job.ad‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
MaxHosts = 1
2+
AccountingGroup = "group_opportunistic.ConnectTrain.dweitzel"
3+
User = "dweitzel@login01.osgconnect.net"
4+
EncryptExecuteDirectory = false
5+
TransferSocket = "<192.170.227.192:16751?addrs=192.170.227.192-16751&noUDP>"
6+
OnExitHold = false
7+
CoreSize = 0
8+
MachineAttrCpus0 = 1
9+
WantRemoteSyscalls = false
10+
MyType = "Job"
11+
Rank = 0.0
12+
CumulativeSuspensionTime = 0
13+
MinHosts = 1
14+
PeriodicHold = false
15+
PeriodicRemove = false
16+
Err = "_condor_stderr"
17+
ProcId = 0
18+
EnteredCurrentStatus = 1490812015
19+
JOBGLIDEIN_ResourceName = "NWICG_NDCMS"
20+
UserLog = "/home/dweitzel/testjob/test.log"
21+
ShadowIpAddr = "<192.170.227.192:16751?addrs=192.170.227.192-16751&noUDP>"
22+
NumJobStarts = 0
23+
AutoClusterAttrs = "ConcurrencyLimits,NiceUser,Rank,Requirements,MachineLastMatchTime,CheckpointPlatform,JobUniverse,LastCheckpointPlatform,NumCkpts,Owner,DESIRED_Resources,ITB_Factory,ITB_Sites,CVMFS_stash_osgstorage_org_REVISION,DESIRED_Sites,GLIDEIN_ResourceName,GLIDEIN_Site,GLIDEIN_ToRetire,OSGVO_OS_STRING,RecentJobStarts,RequestGPUs,SUBMITTER_stash_osgstorage_org_REVISION,ProjectName,SleepSlot,undeined,User,WantsRHEL7,WantsStashCvmfs,IsValidCheckpointPlatform,HAS_CVMFS_stash_osgstorage_org,x509userproxysubject,_condor_RequestCpus,_condor_RequestDisk,_condor_RequestMemory,RequestCpus,RequestDisk,RequestMemory,WithinResourceLimits,DynamicSlot,PartitionableSlot,Slot1_SelfMonitorAge,Slot1_TotalTimeClaimedBusy,HAS_CVMFS_oasis_opensciencegrid_org,HAS_CVMFS_singularity_opensciencegrid_org,Slot10_RemoteOwner,Slot1_RemoteOwner,Slot2_RemoteOwner,Slot3_RemoteOwner,Slot4_RemoteOwner,Slot5_RemoteOwner,Slot6_RemoteOwner,Slot7_RemoteOwner,Slot8_RemoteOwner,Slot9_RemoteOwner,RunOnSubmitNode,GLIDEIN_PS_CVMFS_stash_osgstorage_org_REVISION,RequiresBigMem,RequiresLongWalltime,RequiresWholeMachine,Slot1_TotalTimeUnclaimedIdle,WantStash,CPU_IOWait_Percentage,TotalLoadAvg5Min,ResidentSetSize,x509userproxyfirstfqan,OSG_NODE_VALIDATED,Automatic,offline:,node,not,marked,online,or,midline,in,memcache!,DiskUsage"
24+
JobUniverse = 5
25+
AutoClusterId = 5066
26+
ProjectName = "ConnectTrain"
27+
In = "/dev/null"
28+
Requirements = ( TARGET.Arch == "X86_64" ) && ( TARGET.OpSys == "LINUX" ) && ( TARGET.Disk >= RequestDisk ) && ( TARGET.Memory >= RequestMemory ) && ( TARGET.HasFileTransfer )
29+
ClusterId = 4907261
30+
PublicClaimId = "<10.32.86.127:31028>#1490784616#17#..."
31+
ProvisionedResources = "Cpus Memory Disk Swap"
32+
WhenToTransferOutput = "ON_EXIT"
33+
Notification = Never
34+
LastMatchTime = 1490812015
35+
CompletionDate = 0
36+
UidDomain = "login01.osgconnect.net"
37+
BufferSize = 524288
38+
Environment = ""
39+
StartdPrincipal = "execute-side@matchsession/129.74.145.70"
40+
CpusProvisioned = 1
41+
TargetType = "Machine"
42+
LeaveJobInQueue = false
43+
JobNotification = 0
44+
Owner = "dweitzel"
45+
CondorPlatform = "$CondorPlatform: X86_64-CentOS_6.8 $"
46+
CommittedTime = 0
47+
QDate = 1490811859
48+
JobLeaseDuration = 2400
49+
TransferIn = false
50+
ExitStatus = 0
51+
StartdIpAddr = "<10.32.86.127:31028?CCBID=129.79.53.179:9720%3faddrs%3d129.79.53.179-9720#1567805&addrs=10.32.86.127-31028&noUDP>"
52+
RootDir = "/"
53+
NumJobMatches = 1
54+
JobCurrentStartDate = 1490812015
55+
CurrentHosts = 1
56+
GlobalJobId = "login02.osgconnect.net#4907261.0#1490811860"
57+
RemoteSysCpu = 0.0
58+
TotalSuspensions = 0
59+
WantsPosixStashCache = false
60+
WantCheckpoint = false
61+
LastJobLeaseRenewal = 1490812015
62+
LastRejMatchTime = 1490811997
63+
PeriodicRelease = false
64+
NumCkpts_RAW = 0
65+
CondorVersion = "$CondorVersion: 8.4.9 Sep 29 2016 $"
66+
Out = "_condor_stdout"
67+
ShouldTransferFiles = "YES"
68+
DiskUsage = 1
69+
JobRunCount = 1
70+
CumulativeSlotTime = 0
71+
CommittedSlotTime = 0
72+
LocalUserCpu = 0.0
73+
RemotePool = "osg-flock.grid.iu.edu"
74+
JobStartDate = 1490812015
75+
ExitBySignal = false
76+
StreamErr = false
77+
ConnectWrapper = "2.0"
78+
NumSystemHolds = 0
79+
NumRestarts = 0
80+
RequestDisk = 1
81+
OrigMaxHosts = 1
82+
JobPrio = 0
83+
NumCkpts = 0
84+
BufferBlockSize = 32768
85+
ImageSize = 1
86+
DiskUsage_RAW = 1
87+
RemoteHost = "glidein_2055971_313579676@d12chas566.crc.nd.edu"
88+
CommittedSuspensionTime = 0
89+
Cmd = "/home/dweitzel/testjob/test.sh"
90+
LocalSysCpu = 0.0
91+
Iwd = "/var/condor/execute/dir_2055950/glide_8zWpdT/execute/dir_63440"
92+
TransferInputSizeMB = 0
93+
MemoryProvisioned = 1536
94+
ShadowVersion = "$CondorVersion: 8.4.9 Sep 29 2016 $"
95+
LastRejMatchReason = "no match found "
96+
LastSuspensionTime = 0
97+
ExecutableSize_RAW = 1
98+
OrigCmd = "/home/dweitzel/testjob/test.sh"
99+
OrigIwd = "/home/dweitzel/testjob"
100+
DiskProvisioned = 339705100
101+
JobStatus = 2
102+
ShadowBday = 1490812015
103+
ExecutableSize = 1
104+
MachineAttrSlotWeight0 = 1
105+
RemoteWallClockTime = 0.0
106+
ImageSize_RAW = 1
107+
OnExitRemove = true
108+
Arguments = ""
109+
RemoteSlotID = 1
110+
StreamOut = false
111+
RequestMemory = 500
112+
NiceUser = false
113+
RemoteUserCpu = 0.0
114+
NumShadowStarts = 1
115+
RequestCpus = 1
116+
MyAddress = "<192.170.227.192:16751?addrs=192.170.227.192-16751&noUDP>"
117+
WantRemoteIO = true
118+
SubmitFile = "test.condor"
119+
WantsStashCache = WantsPosixStashCache
120+
LastJobStatus = 1

‎bin/stashcp2/tests/test_inside_docker.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,18 @@ module load xrootd
3737
# For now, disable pylint failures
3838
pylint /StashCache/bin/stashcp || /bin/true
3939

40+
# Copy in the .job.ad file:
41+
cp /StashCache/bin/stashcp2/job.ad ./.job.ad
42+
43+
# Test against a file that is known to not exist
44+
set +e
45+
/StashCache/bin/stashcp --cache=$XRD_CACHE /blah/does/not/exist
46+
if [ $? -eq 0 ]; then
47+
echo "Failed to exit with non-zero exit status when it should have"
48+
exit 1
49+
fi
50+
set -e
51+
4052
# Try copying with no forward slash
4153
/StashCache/bin/stashcp --cache=$XRD_CACHE user/dweitzel/public/blast/queries/query1 ./
4254

0 commit comments

Comments
 (0)