Updated ecco_access to accept text searches of ECCO variable lists, and non-ECCO ShortNames

Author: andrewdelman

ecco_access/__init__.py

@@ -17,8 +17,12 @@
 
 from .ecco_acc_dates import date_adjustment
 
+from .ecco_varlist import ecco_podaac_varlist_query
+
+
 
 __all__ = ['ecco_access',
            'ecco_download',
            'ecco_s3_retrieve',
-           'ecco_acc_dates']
+           'ecco_acc_dates',
+           'ecco_varlist']

ecco_access/ecco_access.py

@@ -1,6 +1,5 @@
 ### This function allows users to query ECCO variables and datasets, and then gain access via direct download, or opening files remotely on S3
 
-
 from .ecco_download import ecco_podaac_query
 from .ecco_download import ecco_podaac_download
 from .ecco_download import ecco_podaac_download_diskaware
@@ -15,6 +14,11 @@
 
 from .ecco_acc_dates import date_adjustment
 
+from .ecco_varlist import ecco_podaac_varlist_query
+
+
+import requests
+
 
 def ecco_podaac_access(query,version='v4r4',grid=None,time_res='all',\
                 StartDate=None,EndDate=None,snapshot_interval=None,\
@@ -29,9 +33,9 @@ def ecco_podaac_access(query,version='v4r4',grid=None,time_res='all',\
     Parameters
     ----------    
     query: str, list, or dict, defines datasets or variables to access.
-           If query is str, it specifies either a dataset ShortName (which is 
-           assumed if the string begins with 'ECCO_'), or a text string that 
-           can be used to search the ShortNames, variable names, and descriptions.
+           If query is str, it specifies either a dataset ShortName (if query 
+           matches a NASA Earthdata ShortName), or a text string that can be 
+           used to search the ECCO ShortNames, variable names, and descriptions.
            A query may also be a list of multiple ShortNames and/or text searches, 
            or a dict that contains grid,time_res specifiers as keys and ShortNames 
            or text searches as values, e.g.,
@@ -44,8 +48,8 @@ def ecco_podaac_access(query,version='v4r4',grid=None,time_res='all',\
     
     grid: ('native','latlon',None), specifies whether to query datasets with output
           on the native grid or the interpolated lat/lon grid.
-          The default None will query both types of grids, unless specified 
-          otherwise in a query dict (e.g., the example above).
+          The default None will query both types of grids (and datasets with no spatial
+          dimension), unless specified otherwise in a query dict (e.g., the example above).
     
     time_res: ('monthly','daily','snapshot','all'), specifies which time resolution 
               to include in query and downloads. 'all' includes all time resolutions, 
@@ -143,10 +147,15 @@ def ecco_podaac_access(query,version='v4r4',grid=None,time_res='all',\
     def shortnames_find(query_list,grid,time_res):
         shortnames_list = []
         for query_item in query_list:
-            if 'ECCO_' in query_item:
+            # see if the query is an existing NASA Earthdata ShortName
+            # if not, then do a text search of the ECCO variable lists
+            response = requests.get(url="https://cmr.earthdata.nasa.gov/search/collections.json", 
+                                    params={'ShortName':query_item})
+            if len(response.json()['feed']['entry']) > 0:
                 shortnames_list.append(query_item)
             else:
-                print('query is not a ShortName')
+                shortname_match = ecco_podaac_varlist_query(query_item,version,grid,time_res)
+                shortnames_list.append(shortname_match)
 
         return shortnames_list
 
@@ -156,6 +165,11 @@ def shortnames_find(query_list,grid,time_res):
     if isinstance(query,dict):
         shortnames = []
         for gridtime_spec,curr_query in query.items():
+            try:
+                curr_grid,curr_time_res = gridtime_spec.split(',')
+            except:
+                raise ValueError("Keys of dict 'query' must be of the form grid,time_res\n'\
+                                 +'with 1 comma in the middle")
             if isinstance(curr_query,str):
                 curr_query = [curr_query]
             shortnames += shortnames_find(curr_query,\
@@ -288,9 +302,9 @@ def ecco_podaac_to_xrdataset(query,version='v4r4',grid=None,time_res='all',\
     Parameters
     ----------    
     query: str, list, or dict, defines datasets or variables to access.
-           If query is str, it specifies either a dataset ShortName (which is 
-           assumed if the string begins with 'ECCO_'), or a text string that 
-           can be used to search the ShortNames, variable names, and descriptions.
+           If query is str, it specifies either a dataset ShortName (if query 
+           matches a NASA Earthdata ShortName), or a text string that can be 
+           used to search the ECCO ShortNames, variable names, and descriptions.
            A query may also be a list of multiple ShortNames and/or text searches, 
            or a dict that contains grid,time_res specifiers as keys and ShortNames 
            or text searches as values, e.g.,

ecco_access/ecco_download.py

@@ -88,7 +88,8 @@ def setup_earthdata_login_auth(url: str='urs.earthdata.nasa.gov'):
 
     ### Helper subroutines to make the API calls to search CMR and parse response
     def set_params(params: dict):
-        params.update({'scroll': "true", 'page_size': 2000})
+#         params.update({'scroll': "true", 'page_size': 2000})
+        params.update({'page_size': 2000})
         return {par: val for par, val in params.items() if val is not None}
 
     def get_results(params: dict, headers: dict=None):
@@ -100,13 +101,14 @@ def get_results(params: dict, headers: dict=None):
 
     def get_granules(params: dict):
         response, headers = get_results(params=params)
-        scroll = headers['CMR-Scroll-Id']
+#         scroll = headers['CMR-Scroll-Id']
         hits = int(headers['CMR-Hits'])
         if hits==0:
             raise Exception("No granules matched your input parameters.")
         df = pd.read_csv(StringIO(response.text)) 
         while hits > df.index.size:
-            response, _ = get_results(params=params, headers={'CMR-Scroll-Id': scroll})
+#             response, _ = get_results(params=params, headers={'CMR-Scroll-Id': scroll})
+            response, _ = get_results(params=params)
             data = pd.read_csv(StringIO(response.text))
             df = pd.concat([df, data])
         return df

ecco_access/ecco_s3_retrieve.py

@@ -91,7 +91,8 @@ def ecco_podaac_s3_query(ShortName,StartDate,EndDate,snapshot_interval='monthly'
 
     ### Helper subroutines to make the API calls to search CMR and parse response
     def set_params(params: dict):
-        params.update({'scroll': "true", 'page_size': 2000})
+#         params.update({'scroll': "true", 'page_size': 2000})
+        params.update({'page_size': 2000})
         return {par: val for par, val in params.items() if val is not None}
 
     def get_results(params: dict, headers: dict=None):