Create an envo lookup table for performance

Author: Doruk Ozturk

server/data/envo_lookup.json

@@ -0,0 +1 @@
+{"ENVO:00000446": "terrestrial biome", "ENVO:00000292": "watershed", "ENVO:00001998": "soil", "ENVO:00000447": "marine biome", "ENVO:00000032": "bay", "ENVO:00002150": "coastal sea water", "ENVO:00000137": "coastal inlet", "ENVO:01000023": "marine pelagic biome", "ENVO:00002149": "sea water", "ENVO:01000048": "ocean biome", "ENVO:00000137\n": "coastal inlet", "ENVO:00002150\n": "coastal sea water", "ENVO:00002119": "alkaline hot spring", "ENVO:01000008": "microbial mat", "ENVO:01000157": "microbial mat material", "ENVO:01000024": "marine benthic biome", "ENVO:00000208": "marine pelagic zone", "ENVO:00002113": "deep marine sediment", "ENVO:01000065": "marine oxygen minimum zone", "ENVO:01000177": "grassland biome", "ENVO:00005750": "grassland soil", "ENVO:01000219": "anthropogenic terrestrial biome", "ENVO:00000078": "farm", "ENVO:00005749": "farm soil", "ENVO:01000252": "freshwater lake biome", "ENVO:00000021": "freshwater lake", "ENVO:00002011": "fresh water", "ENVO:00002131": "epilimnion", "ENVO:00002268": "sphagnum bog", "ENVO:01001020": "hypersaline lake", "ENVO:00002012": "hypersaline water"}

server/data/ingest_1000.py

@@ -1,19 +1,33 @@
 import csv
 import girder_client
 import requests
+import json
+
 
 gc = girder_client.GirderClient(apiUrl='http://localhost:8080/api/v1')
 gc.authenticate('admin', 'letmein')
 parent = gc.resourceLookup('collection/ResonantEco/datasets/LLNL')
 
+lookup_table = {}
 
 def lookup_envo_number(envo):
     response = requests.get('https://www.ebi.ac.uk/ols/api/select',
                      params={'q': envo})
     json_response = response.json()
-    return json_response['response']['docs'][0]['label']
-
-
+    try:
+        label = json_response['response']['docs'][0]['label']
+        lookup_table[envo] = label
+        return label
+    except IndexError:
+        return None
+
+def get_envo_number(envo):
+    with open('envo_lookup.json', 'r') as f:
+        lookup = json.load(f)
+        try:
+            return lookup[envo]
+        except KeyError:
+            return None
 
 def create_item_from_row(row):
     print("Ingesting {}".format(row[0]))
@@ -25,9 +39,9 @@ def create_item_from_row(row):
     }
     metadata['timestamp'] = row[48]
     metadata['name'] = row[1]
-    metadata['biome'] = lookup_envo_number(row[5])
-    metadata['feature'] = lookup_envo_number(row[6])
-    metadata['material'] = lookup_envo_number(row[7])
+    metadata['biome'] = get_envo_number(row[5])
+    metadata['feature'] = get_envo_number(row[6])
+    metadata['material'] = get_envo_number(row[7])
     gc.addMetadataToItem(item['_id'], {'meta': metadata})
 
 def create_items_from_csv(path):
@@ -39,3 +53,5 @@ def create_items_from_csv(path):
 
 if __name__ == '__main__':
     create_items_from_csv('metadata_1K_datasets_curated_JGI.csv')
+    # with open("envo_lookup.json", "w") as f:
+    #     f.write(json.dumps(lookup_table))