add metabolomics skelton functionality

Author: brynnz22

support_code/nmdc/metabolomics/config.toml

@@ -0,0 +1,77 @@
+from metab_metadata_generator import MetabolomicsMetadataGenerator
+import toml
+from pathlib import Path
+
+
+def main():
+    """
+    Runs the MetadataGenerator using the configuration provided in a TOML file.
+
+    The TOML configuration file must include the following fields:
+
+    - metadata_file: str
+        The path to the .csv, .xlsx, or .xls file where the biosample metadata is stored (include file extension).
+        Example: "metadata_file.csv"
+    - data_dir: str
+        The directory path where the data lives or will be stored.
+        Example: "/path/to/data/"
+    - ref_calibration_path: str
+        The path to the .ref file where the reference calibration data is stored (include .ref).
+        Example: "/path/to/calibration_file.ref"
+    - raw_data_object_type: str
+        The raw data object type. Must match one of the following options:
+        - 'Direct Infusion FT ICR-MS Raw Data'
+        - 'LC-DDA-MS/MS Raw Data'
+    - processed_data_object_type: str
+        The processed data object type. Must match one of the following options:
+        - 'FT ICR-MS Analysis Results'
+        - 'GC-MS Metabolomics Results'
+    - registration_file: str
+        The desired name of the output file where the data will be dumped (include .json).
+        Example: "output_file.json"
+    - execution_resource: str
+        The execution resource used for the analysis. Must match one of the following options:
+        - 'EMSL-RZR'
+        - 'EMSL'
+    - field_strength: int
+        The field strength for the NOM analysis. Must match one of the following values:
+        - 7
+        - 12
+        - 15
+        - 21
+    - config_path: str
+        The path to the NMDC minting client and Bioportal API key configuration yaml file. Defaults to 'enviroMS/nmdc_metadata_generation/config.yaml'.
+        Should have client_id and client_secret for the NMDC minting client. And api_key for BioPortal API access. To get a BioPortal api_key register here:
+        https://bioportal.bioontology.org/accounts/new and go to settings
+
+    Notes
+    -----
+    This function assumes:
+    - If a biosample_id does not exist for the sample in the metadata_file, a biosample will be generated. On the other hand, 
+      if a biosample_id does exist in the spreadsheet, no biosample will be generated. And the biosample_id present will be used to 
+      generate all other metadata.
+    - The metadata_file conforms to a predefined structure. See example spreadsheet: https://docs.google.com/spreadsheets/d/1-xHGkkG5Gpw5Pen1iM_JUP2XmphuF19ps2ZYONvtDUs/edit?gid=1112301083#gid=1112301083
+    - Necessary configuration and calibrations are already added to MongoDB. If new configurations are needed, they must be added beforehand.
+    """
+
+
+    # Load arguments from TOML file
+    config_data = toml.load('support_code/nmdc/nom/config.toml')
+
+    generator = MetabolomicsMetadataGenerator(
+        metadata_file=config_data['metadata_file'],
+        data_dir=Path(config_data['data_dir']),
+        ref_calibration_path=Path(config_data['ref_calibration_path']),
+        raw_data_object_type=config_data['raw_data_object_type'],
+        processed_data_object_type=config_data['processed_data_object_type'],
+        database_dump_json_path=config_data['registration_file'],
+        execution_resource=config_data['execution_resource'],
+        field_strength=config_data['field_strength'],
+        workflow_version=config_data['workflow_version'],
+        config_path=config_data['config_path']
+    )
+    
+    generator.run()
+
+if __name__ == "__main__":
+    main()

support_code/nmdc/metabolomics/main.py

@@ -0,0 +1,7 @@
+biosample_id,LC-MS filename,DMS Dataset ID,MyEMSL link,Sample Name,Sample Type,env_medium,habitat,ecosystem_category,img_identifiers,name,geo_loc_name,lat_lon: has_raw_value,latitude,longitude,env_local_scale,location,Ecosystem_type,ncbi_taxonomy_name,ecosystem,env_broad_scale,samp_taxon_id:has_raw_value,sample_collection_site,samp_name,ecosystem_subtype,type,description,collection_date:has_raw_value,NMDC Study ID,FTICR_extraction_method,samp_store_temp.has_raw_value,samp_size.has_raw_value,samp_collec_device,elev,size_frac.has_raw_value,air_temp_regm.has_raw_value,biosample_categories,depth_has_numeric_value,depth.has_minimum_numeric_value,depth.has_maximum_numeric_value,depth.has_unit,soil_horizon,light_regm.has_raw_value,soil_type.has_raw_value,instrument_used,eluent_intro,mass_spec_config,chrom_config_name
+biosamp:1,Lybrand_GHG_01_C_30Aug19_Alder_Infuse_p05_1_01_47965,775133,https://status.my.emsl.pnl.gov/view/1839138,Lybrand_GHG_01_C,soil,ENVO:00001998,,,,Lybrand_GHG_01_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_02_C_30Aug19_Alder_Infuse_p05_1_01_47966,775114,https://status.my.emsl.pnl.gov/view/1839103,Lybrand_GHG_02_C,soil,ENVO:00001998,,,,Lybrand_GHG_02_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_02_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_03_C_30Aug19_Alder_Infuse_p05_1_01_47967,775105,https://status.my.emsl.pnl.gov/view/1839083,Lybrand_GHG_03_C,soil,ENVO:00001998,,,,Lybrand_GHG_03_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_03_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,gas_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_09_C_30Aug19_Alder_Infuse_p05_1_01_47973,775109,https://status.my.emsl.pnl.gov/view/1839094,Lybrand_GHG_09_C,soil,ENVO:00001998,,,,Lybrand_GHG_09_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_09_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"Freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_1_W_27Aug19_Alder_Infuse_p2_000001,772705,https://status.my.emsl.pnl.gov/view/1834769,Lybrand_GHG_01_W,soil,ENVO:00001998,,,,Lybrand_GHG_01_W,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_W,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,water,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_11_C_30Aug19_Alder_Infuse_p05_1_01_47975,775026,https://status.my.emsl.pnl.gov/view/1838954,Lybrand_GHG_11_C,soil,ENVO:00001998,,,,Lybrand_GHG_11_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_11_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"Freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites

support_code/nmdc/metabolomics/metab_metadata_generator.py

@@ -1,4 +1,4 @@
-from metadata_generator import MetadataGenerator
+from nom_metadata_generator import NOMMetadataGenerator
 import toml
 from pathlib import Path
 
@@ -58,7 +58,7 @@ def main():
     # Load arguments from TOML file
     config_data = toml.load('support_code/nmdc/nom/config.toml')
 
-    generator = MetadataGenerator(
+    generator = NOMMetadataGenerator(
         metadata_file=config_data['metadata_file'],
         data_dir=Path(config_data['data_dir']),
         ref_calibration_path=Path(config_data['ref_calibration_path']),

support_code/nmdc/metabolomics/metadata_file_example.csv

@@ -1,6 +1,6 @@
 biosample_id,LC-MS filename,DMS Dataset ID,MyEMSL link,Sample Name,Sample Type,env_medium,habitat,ecosystem_category,img_identifiers,name,geo_loc_name,lat_lon: has_raw_value,latitude,longitude,env_local_scale,location,Ecosystem_type,ncbi_taxonomy_name,ecosystem,env_broad_scale,samp_taxon_id:has_raw_value,sample_collection_site,samp_name,ecosystem_subtype,type,description,collection_date:has_raw_value,NMDC Study ID,FTICR_extraction_method,samp_store_temp.has_raw_value,samp_size.has_raw_value,samp_collec_device,elev,size_frac.has_raw_value,air_temp_regm.has_raw_value,biosample_categories,depth_has_numeric_value,depth.has_minimum_numeric_value,depth.has_maximum_numeric_value,depth.has_unit,soil_horizon,light_regm.has_raw_value,soil_type.has_raw_value,instrument_used,eluent_intro,mass_spec_config,chrom_config_name
-nmdc:Biosamp1,Lybrand_GHG_01_C_30Aug19_Alder_Infuse_p05_1_01_47965,775133,https://status.my.emsl.pnl.gov/view/1839138,Lybrand_GHG_01_C,soil,ENVO:00001998,,,,Lybrand_GHG_01_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
-nmdc:Biosamp2,Lybrand_GHG_02_C_30Aug19_Alder_Infuse_p05_1_01_47966,775114,https://status.my.emsl.pnl.gov/view/1839103,Lybrand_GHG_02_C,soil,ENVO:00001998,,,,Lybrand_GHG_02_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_02_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+biosamp:1,Lybrand_GHG_01_C_30Aug19_Alder_Infuse_p05_1_01_47965,775133,https://status.my.emsl.pnl.gov/view/1839138,Lybrand_GHG_01_C,soil,ENVO:00001998,,,,Lybrand_GHG_01_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
+,Lybrand_GHG_02_C_30Aug19_Alder_Infuse_p05_1_01_47966,775114,https://status.my.emsl.pnl.gov/view/1839103,Lybrand_GHG_02_C,soil,ENVO:00001998,,,,Lybrand_GHG_02_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_02_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,12,24,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
 ,Lybrand_GHG_03_C_30Aug19_Alder_Infuse_p05_1_01_47967,775105,https://status.my.emsl.pnl.gov/view/1839083,Lybrand_GHG_03_C,soil,ENVO:00001998,,,,Lybrand_GHG_03_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_03_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,gas_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
 ,Lybrand_GHG_09_C_30Aug19_Alder_Infuse_p05_1_01_47973,775109,https://status.my.emsl.pnl.gov/view/1839094,Lybrand_GHG_09_C,soil,ENVO:00001998,,,,Lybrand_GHG_09_C,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_09_C,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,chloroform,-20 degree Celsius,,permafrost corer,670,not sieved,"Freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,40,54,cm,B horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites
 ,Lybrand_GHG_1_W_27Aug19_Alder_Infuse_p2_000001,772705,https://status.my.emsl.pnl.gov/view/1834769,Lybrand_GHG_01_W,soil,ENVO:00001998,,,,Lybrand_GHG_01_W,"USA: Alaska, Healy",63.87980 -149.21539,63.8798,-149.21539,ENVO:01000861,,,,,ENVO:00000446,,HEAL,Lybrand_GHG_01_W,,nmdc:Biosample,Lybrand NEON (EMSL 50267),2016-04-01,nmdc:sty-11-db67n062,water,-20 degree Celsius,,permafrost corer,670,not sieved,"No freeze-thaw|Following freeze-thaw incubation, soils were incubated at 20C for 14 days.",NEON,,16,30,cm,O horizon,Dark,,12T FT-ICR MS,liquid_chromatography,"EMSL NOM reduced profile mass spectrometry, negative",EMSL LC method for non-polar metabolites