Merge pull request #5 from OpenDataAnalytics/ingest-thousand

Ingest thousand dataset

Author: dorukozturk

server/data/envo_lookup.json

@@ -0,0 +1 @@
+{"ENVO:00000446": "terrestrial biome", "ENVO:00000292": "watershed", "ENVO:00001998": "soil", "ENVO:00000447": "marine biome", "ENVO:00000032": "bay", "ENVO:00002150": "coastal sea water", "ENVO:00000137": "coastal inlet", "ENVO:01000023": "marine pelagic biome", "ENVO:00002149": "sea water", "ENVO:01000048": "ocean biome", "ENVO:00000137\n": "coastal inlet", "ENVO:00002150\n": "coastal sea water", "ENVO:00002119": "alkaline hot spring", "ENVO:01000008": "microbial mat", "ENVO:01000157": "microbial mat material", "ENVO:01000024": "marine benthic biome", "ENVO:00000208": "marine pelagic zone", "ENVO:00002113": "deep marine sediment", "ENVO:01000065": "marine oxygen minimum zone", "ENVO:01000177": "grassland biome", "ENVO:00005750": "grassland soil", "ENVO:01000219": "anthropogenic terrestrial biome", "ENVO:00000078": "farm", "ENVO:00005749": "farm soil", "ENVO:01000252": "freshwater lake biome", "ENVO:00000021": "freshwater lake", "ENVO:00002011": "fresh water", "ENVO:00002131": "epilimnion", "ENVO:00002268": "sphagnum bog", "ENVO:01001020": "hypersaline lake", "ENVO:00002012": "hypersaline water"}

server/data/ingest_1000.py

@@ -0,0 +1,57 @@
+import csv
+import girder_client
+import requests
+import json
+
+
+gc = girder_client.GirderClient(apiUrl='http://localhost:8080/api/v1')
+gc.authenticate('admin', 'letmein')
+parent = gc.resourceLookup('collection/ResonantEco/datasets/LLNL')
+
+lookup_table = {}
+
+def lookup_envo_number(envo):
+    response = requests.get('https://www.ebi.ac.uk/ols/api/select',
+                     params={'q': envo})
+    json_response = response.json()
+    try:
+        label = json_response['response']['docs'][0]['label']
+        lookup_table[envo] = label
+        return label
+    except IndexError:
+        return None
+
+def get_envo_number(envo):
+    with open('envo_lookup.json', 'r') as f:
+        lookup = json.load(f)
+        try:
+            return lookup[envo]
+        except KeyError:
+            return None
+
+def create_item_from_row(row):
+    print("Ingesting {}".format(row[0]))
+    item = gc.createItem(parent['_id'], row[0], reuseExisting=True)
+    metadata = {
+        'latitude': None,
+        'longitude': None,
+        'source': 'JGI'
+    }
+    metadata['timestamp'] = row[48]
+    metadata['name'] = row[1]
+    metadata['biome'] = get_envo_number(row[5])
+    metadata['feature'] = get_envo_number(row[6])
+    metadata['material'] = get_envo_number(row[7])
+    gc.addMetadataToItem(item['_id'], {'meta': metadata})
+
+def create_items_from_csv(path):
+    with open(path) as csvfile:
+        reader = csv.reader(csvfile, delimiter=',')
+        next(reader)
+        for row in reader:
+            create_item_from_row(row)
+
+if __name__ == '__main__':
+    create_items_from_csv('metadata_1K_datasets_curated_JGI.csv')
+    # with open("envo_lookup.json", "w") as f:
+    #     f.write(json.dumps(lookup_table))