Merge branch 'add_auth_reading'

Author: djw8605

bin/stashcp2/tests/test_inside_docker.sh

@@ -107,7 +107,15 @@ if [ "$result" != "12bdb9a96cd5e8ca469b727a81593201" ]; then
 fi
 rm -f query.test
 
+# Scheme test
+/StashCache/stashcp.py --cache=$XRD_CACHE --method=http,xrootd -d stash:///user/dweitzel/public/blast/queries/query1 file://$PWD
 
+result=`md5sum query1 | awk '{print $1;}'`
+
+if [ "$result" != "12bdb9a96cd5e8ca469b727a81593201" ]; then
+  exit 1
+fi
+rm -f query.test
 
 /StashCache/stashcp.py --cache=$XRD_CACHE -d -r /user/dweitzel/public/blast/queries ./
 ls -lah

setup.py

@@ -42,7 +42,7 @@
     # For a discussion on single-sourcing the version across setup.py and the
     # project code, see
     # https://packaging.python.org/en/latest/single_source_version.html
-    version='5.2.1',  # Required
+    version='5.3.0',  # Required
 
     # This is a one-line description or tagline of what your project does. This
     # corresponds to the "Summary" metadata field:
@@ -83,7 +83,7 @@
 
     # This should be a valid email address corresponding to the author listed
     # above.
-    author_email='stashcache@opensciencegrid.org',  # Optional
+    author_email='xcache@opensciencegrid.org',  # Optional
 
     # Classifiers help users find your project by categorizing it.
     #

stashcp.py

@@ -12,6 +12,7 @@
 import socket
 import random
 import shutil
+from urlparse import urlparse
 
 try:
     from pkg_resources import resource_string
@@ -23,7 +24,7 @@
 from urlparse import urlparse
 
 # Version information for user-agent
-VERSION = "5.2.1"
+VERSION = "5.3.0"
 
 main_redirector = "root://redirector.osgstorage.org"
 stash_origin = "root://stash.osgconnect.net"
@@ -38,6 +39,9 @@
 # Global variable for the location of the caches.json file
 caches_json_location = None
 
+# Global variable for the location of the token to use for reading / writing
+token_location = None
+
 TIMEOUT = 300
 DIFF = TIMEOUT * 10
 
@@ -49,29 +53,11 @@ def doWriteBack(source, destination):
     :param str destination: The location of the remote file, in stash:// format
     """
     start1 = int(time.time()*1000)
-    
-     # Get the scitoken content
-    scitoken_file = None
-    if '_CONDOR_CREDS' in os.environ:
-        # First, look for the scitokens.use file
-        # Format: _CONDOR_CREDS=/var/lib/condor/execute/dir_908/.condor_creds
-        scitoken_file = os.path.join(os.environ['_CONDOR_CREDS'], 'scitokens.use')
-        if not os.path.exists(scitoken_file):
-            scitoken_file = None
-    
-    if not scitoken_file and os.path.exists(".condor_creds/scitokens.use"):
-        scitoken_file = ".condor_creds/scitokens.use"
-    
-    if not scitoken_file:
+
+    scitoken_contents = getToken()
+    if scitoken_contents is None:
         logging.error("Unable to find scitokens.use file")
         return 1
-
-    
-    with open(scitoken_file, 'r') as scitoken_obj:
-        scitoken_contents = scitoken_obj.read().strip()
-
-    # Remove the stash:// at the beginning, don't need it
-    destination = destination.replace("stash://", "")
 
     # Check if the source file is zero-length
     statinfo = os.stat(source)
@@ -119,6 +105,48 @@ def doWriteBack(source, destination):
     es_send(payload)
     return curl_exit
 
+def getToken():
+    """
+    Get the token / scitoken from the environment in order to read / write
+    """
+    # Get the scitoken content
+    scitoken_file = None
+
+    # Command line
+    if token_location:
+        scitoken_file = token_location
+    # Environ
+    if 'TOKEN' in os.environ:
+        scitoken_file = os.environ['TOKEN']
+
+    # Backwards compatibility for getting scitokens
+    if not scitoken_file and "_CONDOR_CREDS" in os.environ:
+        # Token wasn't specified on the command line, try the default scitokens.use
+        if os.path.exists(os.path.join(os.environ["_CONDOR_CREDS"], "scitokens.use")):
+            scitoken_file = os.path.join(os.environ["_CONDOR_CREDS"], "scitokens.use")
+        elif os.path.exists(".condor_creds/scitokens.use"):
+            scitoken_file = os.path.join(os.path.abspath(".condor_creds/scitokens.use"))
+
+    if not scitoken_file or not os.path.exists(scitoken_file):
+        logging.info("Unable to find token file")
+        return None
+
+    # If the scitoken file is relative, then assume it's relative
+    # to the _CONDOR_CREDS directory.
+    if not os.path.isabs(scitoken_file) and "_CONDOR_CREDS" in os.environ:
+        os.path.join(os.environ['_CONDOR_CREDS'], scitoken_file)
+
+    # Read in the JSON
+    with open(scitoken_file, 'r') as scitoken_obj:
+        try:
+            token_json = json.load(scitoken_obj)
+            scitoken_contents = token_json['access_token']
+        except ValueError as jsonfail:
+            logging.info("Falling back to old style scitoken parsing")
+            scitoken_obj.seek(0)
+            scitoken_contents = scitoken_obj.read()
+
+    return scitoken_contents
 
 def doStashCpSingle(sourceFile, destination, methods, debug=False):
     """
@@ -127,13 +155,32 @@ def doStashCpSingle(sourceFile, destination, methods, debug=False):
 
     global nearest_cache
 
-    # Check if the desitnation is a protocol like stash:///user/blah
-    if destination.startswith("stash://"):
-        # Source file exists, must be a writeback
-        return doWriteBack(sourceFile, destination)
-    
+    # Parse the source and destination with urlparse
+    source_url = urlparse(sourceFile)
+    dest_url = urlparse(destination)
+    understoodSchemes = ["stash", "file", ""]
+    if source_url.scheme not in understoodSchemes:
+        logging.error("Do not understand scheme: %s", source_url.scheme)
+        return 1
+
+    if dest_url.scheme not in understoodSchemes:
+        logging.error("Do not understand scheme: %s", dest_url.scheme)
+        return 1
+
+    if dest_url.scheme == "stash":
+        return doWriteBack(source_url.path, dest_url.path)
+
+    if dest_url.scheme == "file":
+        destination = dest_url.path
+
+    if source_url.scheme == "stash":
+        sourceFile = source_url.path
+
+    if not sourceFile.startswith("/"):
+        sourceFile = "/" + sourceFile
+
     sitename = os.environ.setdefault("OSG_SITE_NAME", "siteNotFound")
-    
+
     # Fill out the payload as much as possible
     filename = destination + '/' + sourceFile.split('/')[-1]
 
@@ -299,11 +346,11 @@ def download_http(source, destination, debug, payload):
     global nearest_cache
     global nearest_cache_list
 
-    if not nearest_cache:
-        nearest_cache = get_best_stashcache()
-
     logging.debug("Downloading with HTTP")
 
+    #scitoken_contents = getToken()
+    scitoken_contents = None
+
     if not nearest_cache:
         nearest_cache = get_best_stashcache()
 
@@ -340,11 +387,17 @@ def download_http(source, destination, debug, payload):
             cache = cache.replace('root://', 'http://')
 
         # Append port 8000, which is just a convention for now, not set in stone
-        cache += ":8000"
+        # Check if the cache already has a port attached to it
+        parsed_url = urlparse(cache)
+        if not parsed_url.port:
+            cache += ":8000"
 
         # Quote the source URL, which may have weird, dangerous characters
         quoted_source = urllib2.quote(source)
-        curl_command = "curl %s -L --connect-timeout 30 --speed-limit 1024 %s --fail %s%s" % (output_mode, download_output, cache, quoted_source)
+        if scitoken_contents:
+            curl_command = "curl %s -L --connect-timeout 30 --speed-limit 1024 %s --fail -H \"Authorization: Bearer %s\" %s%s" % (output_mode, download_output, scitoken_contents, cache, quoted_source)
+        else:
+            curl_command = "curl %s -L --connect-timeout 30 --speed-limit 1024 %s --fail %s%s" % (output_mode, download_output, cache, quoted_source)
         logging.debug("About to run curl command: %s", curl_command)
         start = int(time.time()*1000)
         command_object = subprocess.Popen([curl_command], shell=True, cwd=dest_dir)
@@ -447,7 +500,10 @@ def timed_transfer(filename, destination, cache, debug=False, ):
     os.environ.setdefault("XRD_CONNECTIONRETRY", "2")   # How many time should we retry the TCP connection
     os.environ.setdefault("XRD_STREAMTIMEOUT", "30")    # How long to wait for TCP activity
 
-    filepath=cache+":1094//"+ filename
+    if not filename.startswith("/"):
+        filepath=cache+":1094//"+ filename
+    else:
+        filepath=cache+":1094/"+ filename
     if debug:
         command="xrdcp -d 2 --nopbar -f " + filepath + " " + destination
     else:
@@ -591,6 +647,7 @@ def main():
     global nearest_cache
     global nearest_cache_list
     global caches_json_location
+    global token_location
 
     usage = "usage: %prog [options] source destination"
     parser = optparse.OptionParser(usage)
@@ -601,6 +658,7 @@ def main():
     parser.add_option('-j', '--caches-json', dest='caches_json', help="The JSON file containing the list of caches",
                       default=None)
     parser.add_option('--methods', dest='methods', help="Comma separated list of methods to try, in order.  Default: cvmfs,xrootd,http", default="cvmfs,xrootd,http")
+    parser.add_option('-t', '--token', dest='token', help="Token file to use for reading and/or writing")
     args,opts=parser.parse_args()
 
     logging.basicConfig(format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
@@ -613,7 +671,10 @@ def main():
     else:
         logger.setLevel(logging.WARNING)
 
-    caches_json_location = args.caches_json
+    if 'CACHES_JSON' in os.environ:
+        caches_json_location = os.environ['CACHES_JSON']
+    else:
+        caches_json_location = args.caches_json
     if args.closest:
         print get_best_stashcache()
         sys.exit(0)
@@ -625,9 +686,15 @@ def main():
         destination=opts[1]
 
     # Check for manually entered cache to use
-    if args.cache and len(args.cache) > 0:
+    if 'NEAREST_CACHE' in os.environ:
+        nearest_cache = os.environ['NEAREST_CACHE']
+        nearest_cache_list = [nearest_cache]
+    elif args.cache and len(args.cache) > 0:
         nearest_cache = args.cache
-        nearest_cache_list = [ args.cache ]
+        nearest_cache_list = [args.cache]
+    
+    if args.token:
+        token_location = args.token
 
     # Convert the methods
     methods = args.methods.split(',')