Merge branch 'update_nom_metadata_gen' into 'master'

Update nom metadata gen

See merge request mass-spectrometry/corems!145

Author: corilo

support_code/nmdc/nom/api_info_retriever.py

@@ -1,15 +1,16 @@
 import requests
 from typing import Optional
+import yaml
 
 # TODO: Update og_url to be regular nmdc api url when berkeley is implemented
 
-class ApiInfoRetriever:
+class NmdcApiInfoRetriever:
     def __init__(self, collection_name: str):
         self.collection_name = collection_name
 
     def get_id_by_slot_from_collection(self, slot_name: str, slot_field_value: str):
         """
-        Retrieve the identifier from a specified collection based on a slot name and field value.
+        Retrieve the NMDC identifier from a specified collection based on a slot name and field value.
 
         Parameters
         ----------
@@ -34,7 +35,7 @@ def get_id_by_slot_from_collection(self, slot_name: str, slot_field_value: str):
         filter = f'{{"{slot_name}": "{slot_field_value}"}}'
         field = "id"
 
-        og_url = f'https://api-berkeley.microbiomedata.org/nmdcschema/{self.collection_name}?&filter={filter}&projection={field}'
+        og_url = f'https://api.microbiomedata.org/nmdcschema/{self.collection_name}?&filter={filter}&projection={field}'
         resp = requests.get(og_url)
 
         # Check if the response status is 200
@@ -45,9 +46,79 @@ def get_id_by_slot_from_collection(self, slot_name: str, slot_field_value: str):
 
         # Ensure there is at least one resource in the response
         if not data['resources']:
-            raise ValueError(f"No resources found for {slot_name} with value {slot_field_value}")
+            raise ValueError(f"No resources in Mongo found for '{slot_name}' slot in {self.collection_name} with value {slot_field_value}")
 
         identifier = data['resources'][0]['id']
 
         return identifier
 
+class BioOntologyInfoRetriever:
+    """
+    Client for retrieving ENVO term information from BioPortal API.
+
+    A class to handle authentication and retrieval of Environmental Ontology (ENVO)
+    terms using the BioPortal REST API service.
+
+    Parameters
+    ----------
+    config_path : str
+        Path to YAML configuration file containing BioPortal API credentials
+
+    Notes
+    -----
+    The configuration file should contain an 'api_key' field with a valid
+    BioPortal API key.
+
+    Examples
+    --------
+    >>> retriever = BioOntologyInfoRetriever('config.yaml')
+    >>> envo_terms = retriever.get_envo_terms('ENVO:00002042')
+    >>> print(envo_terms)
+    {'ENVO:00002042': 'surface water'}
+    """
+    def __init__(self, config_path: str):
+        self.config = config_path
+
+    def get_envo_terms(self, envo_id: dict):
+        """
+        Look up an ENVO term label using BioPortal API.
+
+        Parameters
+        ----------
+        envo_id : str
+            The ENVO identifier to look up (e.g., 'ENVO:00002042')
+
+        Returns
+        -------
+        dict
+            Dictionary with envo_id as key and term label as value
+            Example: {'ENVO:00002042': 'surface water'}
+
+        Raises
+        ------
+        requests.exceptions.RequestException
+            If the API request fails
+        KeyError
+            If the response doesn't contain expected data format
+        yaml.YAMLError
+            If the config file cannot be parsed
+        FileNotFoundError
+            If the config file is not found
+
+        Notes
+        -----
+        Makes an authenticated request to BioPortal API to retrieve the
+        preferred label (prefLabel) for the given ENVO term.
+        """
+        
+        config = yaml.safe_load(open(self.config))
+        api_key = config['api_key']
+
+        url = f"http://data.bioontology.org/ontologies/ENVO/classes/{envo_id}"
+        headers = {"Authorization": f"apikey token={api_key}"}
+
+        response = requests.get(url, headers=headers)
+        response.raise_for_status()
+
+        data = response.json()
+        return {envo_id: data['prefLabel']}

support_code/nmdc/nom/config.toml

@@ -7,4 +7,4 @@ registration_file = "path/to/where/database/dump/lives.json"
 execution_resource = "EMSL-RZR"
 field_strength = 7
 workflow_version = "2.2.3"
-minting_client_config_path = "enviroMS/nmdc_metadata_generation/config.yaml"
+config_path = "enviroMS/nmdc_metadata_generation/config.yaml"

support_code/nmdc/nom/main.py

@@ -2,6 +2,7 @@
 import toml
 from pathlib import Path
 
+
 def main():
     """
     Runs the MetadataGenerator using the configuration provided in a TOML file.
@@ -38,8 +39,10 @@ def main():
         - 12
         - 15
         - 21
-    - minting_client_config_path: str
-        The path to the NMDC minting client configuration file. Defaults to 'enviroMS/nmdc_metadata_generation/config.yaml'.
+    - config_path: str
+        The path to the NMDC minting client and Bioportal API key configuration yaml file. Defaults to 'enviroMS/nmdc_metadata_generation/config.yaml'.
+        Should have client_id and client_secret for the NMDC minting client. And api_key for BioPortal API access. To get a BioPortal api_key register here:
+        https://bioportal.bioontology.org/accounts/new and go to settings
 
     Notes
     -----
@@ -53,7 +56,7 @@ def main():
 
 
     # Load arguments from TOML file
-    config_data = toml.load('enviroMS/nmdc_metadata_generation/config.toml')
+    config_data = toml.load('support_code/nmdc/nom/config.toml')
 
     generator = MetadataGenerator(
         metadata_file=config_data['metadata_file'],
@@ -65,7 +68,7 @@ def main():
         execution_resource=config_data['execution_resource'],
         field_strength=config_data['field_strength'],
         workflow_version=config_data['workflow_version'],
-        minting_client_config_path=config_data['minting_client_config_path']
+        config_path=config_data['config_path']
     )
 
     generator.run()

support_code/nmdc/nom/metadata_file_example.csv

@@ -0,0 +1,7 @@
+biosample_id,LC-MS filename,DMS Dataset ID,MyEMSL link,Sample Name,Sample Type,env_medium,habitat,ecosystem_category,img_identifiers,name,geo_loc_name,lat_lon: has_raw_value,latitude,longitude,env_local_scale,location,Ecosystem_type,ncbi_taxonomy_name,ecosystem,env_broad_scale,samp_taxon_id:has_raw_value,sample_collection_site,samp_name,ecosystem_subtype,type,description,collection_date:has_raw_value,NMDC Study ID,FTICR_extraction_method,samp_store_temp.has_raw_value,samp_size.has_raw_value,samp_collec_device,elev,size_frac.has_raw_value,air_temp_regm.has_raw_value,biosample_categories,depth_has_numeric_value,depth.has_minimum_numeric_value,depth.has_maximum_numeric_value,depth.has_unit,soil_horizon,light_regm.has_raw_value,soil_type.has_raw_value,instrument_used,eluent_intro,mass_spec_config,chrom_config_name
+nmdc:Biosamp1,Lybrand_GHG_01_C_30Aug19_Alder_Infuse_p05_1_01_47965,775133,https://status.my.emsl.pnl.gov/view/1839138,Lybrand_GHG_01_C,soil,ENVO:00001998,,,,Lybrand_GHG_01_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+nmdc:Biosamp2,Lybrand_GHG_02_C_30Aug19_Alder_Infuse_p05_1_01_47966,775114,https://status.my.emsl.pnl.gov/view/1839103,Lybrand_GHG_02_C,soil,ENVO:00001998,,,,Lybrand_GHG_02_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_02_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_03_C_30Aug19_Alder_Infuse_p05_1_01_47967,775105,https://status.my.emsl.pnl.gov/view/1839083,Lybrand_GHG_03_C,soil,ENVO:00001998,,,,Lybrand_GHG_03_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_03_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,gas_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_09_C_30Aug19_Alder_Infuse_p05_1_01_47973,775109,https://status.my.emsl.pnl.gov/view/1839094,Lybrand_GHG_09_C,soil,ENVO:00001998,,,,Lybrand_GHG_09_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_09_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"Freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_1_W_27Aug19_Alder_Infuse_p2_000001,772705,https://status.my.emsl.pnl.gov/view/1834769,Lybrand_GHG_01_W,soil,ENVO:00001998,,,,Lybrand_GHG_01_W,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_W,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,water,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_11_C_30Aug19_Alder_Infuse_p05_1_01_47975,775026,https://status.my.emsl.pnl.gov/view/1838954,Lybrand_GHG_11_C,soil,ENVO:00001998,,,,Lybrand_GHG_11_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_11_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"Freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites