Merge remote-tracking branch 'origin/master' into asmsbugfixes2024

Author: wkew-pnnl

MANIFEST.in

@@ -1,4 +1,5 @@
 include requirements.txt
 include disclaimer.txt
 include Dockerfile
-include ext_lib/*
+include ext_lib/*
+include ext_lib/dotnet/*

README.md

@@ -1,4 +1,4 @@
-![CoreMS Logo](https://github.com/EMSL-Computing/CoreMS/tree/master/docs/CoreMS.COLOR_small.png?raw=true)  
+![CoreMS Logo](docs/CoreMS.COLOR_small.png?raw=true)  
 
 <div align="left">
 

support_code/nmdc/nom/nmdc_metadata_gen.py

@@ -15,9 +15,18 @@
 
 env_mediums = {'ENVO_00002042': 'surface water',
                'ENVO_00002007': 'sediment',
+               'ENVO:00001998': 'soil'
                }
-env_local_scales = {'ENVO_00000022': 'river'}    
-env_broad_scales = {'ENVO_01000253': 'freshwater river biome'}    
+env_local_scales = {'ENVO_00000022': 'river',
+                    'ENVO:01000861': 'area of dwarf scrub',
+                    'ENVO:00000516': 'hummock',
+                    'ENVO:01000869': 'area of scrub',
+                    'ENVO:01000887': 'area of sedge- and forb-dominated herbaceous vegetation',
+                    'ENVO:01001370': 'tundra ecosystem'
+                    }    
+env_broad_scales = {'ENVO_01000253': 'freshwater river biome',
+                    'ENVO:00000446': 'terrestrial biome'
+                    }    
 
 
 @dataclass
@@ -26,13 +35,15 @@ class NomAnalysisActivity:
     cluster_name:str = "EMSL-RZR"
     nom_21T_instrument_name: str = "21T_Agilent"
     nom_12T_instrument_name: str = "12T_FTICR_B"
+    nom_7T_instrument_name: str = "7T_FT_ICR_MS"
 
 @dataclass
 class OmicsProcessing:
     nom_omics_processing_type:str = "Organic Matter Characterization"
     nom_omics_processing_description:str = "High resolution MS spectra only"
     nom_21T_instrument_name: str = "21T Agilent"
     nom_12T_instrument_name: str = "12T_FTICR_B"
+    nom_7T_instrument_name: str = "7T_FT_ICR_MS"
 
 @dataclass
 class DataObject:
@@ -42,13 +53,13 @@ class DataObject:
     nom_dp_data_object_description:str = "EnviroMS FT ICR-MS natural organic matter workflow molecular formula assignment output details"
 
 @dataclass
-class BioSample:
+class Biosample:
     pass
 
 @dataclass
 class NMDC_Types: 
 
-    BioSample:str = "nmdc:Biosample"
+    Biosample:str = "nmdc:Biosample"
     OmicsProcessing:str = "nmdc:OmicsProcessing"
     NomAnalysisActivity:str = "nmdc:NomAnalysisActivity"
     DataObject:str = "nmdc:DataObject"
@@ -68,7 +79,7 @@ def __dict__(self):
     @property
     def json(self):
         return dumps(self.__dict__)
-    
+
 def mint_nmdc_id(type:NMDC_Types, how_many:int = 1) -> List[str]: 
 
     config = yaml.safe_load(open('./config.yaml','r'))
@@ -93,7 +104,7 @@ def mint_nmdc_id(type:NMDC_Types, how_many:int = 1) -> List[str]:
 
 def get_biosample_object(emsl_metadata:EMSL_Metadata) -> nmdc.Biosample:
 
-    nmdc_id = mint_nmdc_id({'id': NMDC_Types.BioSample})[0]
+    nmdc_id = mint_nmdc_id({'id': NMDC_Types.Biosample})[0]
 
     env_medium = {
                  'has_raw_value': emsl_metadata.env_medium,
@@ -121,7 +132,7 @@ def get_biosample_object(emsl_metadata:EMSL_Metadata) -> nmdc.Biosample:
                 "longitude": emsl_metadata.longitude,
             }
 
-    collection_date = { 'has_raw_value': emsl_metadata.collection_date}
+    collection_date = {'has_raw_value': emsl_metadata.collection_date}
 
     geo_loc_name =  {'has_raw_value': emsl_metadata.geo_loc_name}
 
@@ -166,7 +177,7 @@ def get_data_object(file_path:Path, base_url:str, was_generated_by:str,
                 "description": description,
                 "type": "nmdc:DataObject"
                 } 
-    
+
     data_object = nmdc.DataObject(**data_dict)
 
     return data_object
@@ -228,46 +239,45 @@ def create_nmdc_metadata(raw_data_path:Path, data_product_path:Path, base_url:st
 
     if not biosample_id:
 
-        #biosample_id = mint_nmdc_id({'id': NMDC_Types.BioSample})[0]
+        biosample_id = mint_nmdc_id({'id': NMDC_Types.Biosample})[0]
         bioSample = get_biosample_object(emsl_metadata)
         biosample_id = bioSample.id
 
     else:
 
         ''' needs to finish the logic for creating biosamples, this will fail because it is missing some required fields'''
-        bioSample =  nmdc.BioSample(id=biosample_id)
+        bioSample = None
 
     omicsProcessing = get_omics_processing(raw_data_path,
-                                           OmicsProcessing.nom_12T_instrument_name,
-                                           biosample_id, None, 
+                                           OmicsProcessing.nom_7T_instrument_name,
+                                           biosample_id, 'nmdc:placeholder', 
                                            OmicsProcessing.nom_omics_processing_type,
                                            OmicsProcessing.nom_omics_processing_description,
-                                           emsl_metadata.nmdc_study
+                                           emsl_metadata.nmdc_study 
                                            )
 
-    rawDataObject = get_data_object(raw_data_path, base_url + 'nom/grow/raw/', 
+    rawDataObject = get_data_object(raw_data_path, base_url + 'nom/1000soils/raw/', 
                                     was_generated_by=omicsProcessing.id, 
                                     data_object_type =DataObject.nom_raw_data_object_type,
                                     description =DataObject.nom_raw_data_object_description)
 
     nomAnalysisActivity = get_nom_analysis_activity(NomAnalysisActivity.cluster_name,
                                                 NomAnalysisActivity.codebase_url,
-                                                rawDataObject.id, None, False, 
+                                                rawDataObject.id, 'nmdc:placeholder', False, 
                                                 omicsProcessing.id,
-                                                NomAnalysisActivity.nom_12T_instrument_name)
+                                                NomAnalysisActivity.nom_7T_instrument_name)
 
-    dataProductDataObject = get_data_object(data_product_path, base_url + 'nom/grow/results/', 
+    dataProductDataObject = get_data_object(data_product_path, base_url + 'nom/1000soils/results/', 
                                     was_generated_by=nomAnalysisActivity.id, 
                                     data_object_type =DataObject.nom_dp_data_object_type,
                                     description =DataObject.nom_dp_data_object_description)
 
-    
     #circular dependencies : great! 
-    nomAnalysisActivity.has_input = [rawDataObject.id]
     nomAnalysisActivity.has_output = [dataProductDataObject.id]
     omicsProcessing.has_output = [rawDataObject.id]
 
-    nom_metadata_db.biosample_set.append(bioSample)
+    if bioSample:
+        nom_metadata_db.biosample_set.append(bioSample)
     nom_metadata_db.data_object_set.append(rawDataObject)
     nom_metadata_db.nom_analysis_activity_set.append(nomAnalysisActivity)
     nom_metadata_db.omics_processing_set.append(omicsProcessing)
@@ -276,4 +286,3 @@ def create_nmdc_metadata(raw_data_path:Path, data_product_path:Path, base_url:st
 def dump_nmdc_database(ndmc_database:nmdc.Database, output_filepath:str):
 
     json_dumper.dump(ndmc_database, output_filepath)
-    

support_code/nmdc/nom/nom_grow_workflow.py

@@ -37,6 +37,7 @@ class EMSL_Metadata:
     description: str
     collection_date: str
     nmdc_study: str
+    biosample_id: str
 
 def parse_metadata(metadata_file_path:Path) -> EMSL_Metadata:
 
@@ -70,6 +71,7 @@ def parse_metadata(metadata_file_path:Path) -> EMSL_Metadata:
         description = full_list_worksheet['Z']
         collection_date = full_list_worksheet['AA']
         nmdc_study = full_list_worksheet['AB']
+        biosample_id = full_list_worksheet['AC']
 
         for x in range(1, len(full_list_worksheet['A'])):
 
@@ -96,7 +98,8 @@ def parse_metadata(metadata_file_path:Path) -> EMSL_Metadata:
                                 ecosystem_subtype = ecosystem_subtype[x].value,
                                 description = description[x].value,
                                 collection_date = collection_date[x].value,
-                                nmdc_study = nmdc_study[x].value
+                                nmdc_study = nmdc_study[x].value,
+                                biosample_id = biosample_id[x].value
                                 )
 
             yield metadata
@@ -141,7 +144,7 @@ def run_nom_nmdc_data_processing():
     registration_dir.mkdir(parents=True, exist_ok=True)
     registration_file = registration_dir / args.registration_file_name
 
-    field_strength = 12
+    field_strength = 7
 
     ref_calibration_path = Path(args.ref_calibration_path)
     failed_list = []
@@ -150,7 +153,7 @@ def run_nom_nmdc_data_processing():
 
     for each_data in parse_metadata(metadata_file_path):
 
-        raw_file_path = data_dir / each_data.data_path / each_data.data_path.with_suffix(file_ext)    
+        raw_file_path = data_dir / each_data.data_path.with_suffix(file_ext)    
 
         print(raw_file_path)
 
@@ -174,7 +177,7 @@ def run_nom_nmdc_data_processing():
                                                         output_file_path,
                                                         "https://nmdcdemo.emsl.pnnl.gov/", 
                                                         nmdc_database,
-                                                        each_data, biosample_id=None)
+                                                        each_data, biosample_id=each_data.biosample_id)
 
             else:
 

support_code/nmdc/nom/nom_workflow.py

@@ -43,8 +43,9 @@ def run_bruker(file_location):
 
 def run_thermo(file_location):
 
-    MSParameters.mass_spectrum.noise_threshold_methodmethod = 'log'
+    MSParameters.mass_spectrum.noise_threshold_method = 'log'
     MSParameters.mass_spectrum.noise_threshold_min_std = 3
+    MSParameters.ms_peak.peak_min_prominence_percent = 0.2
 
     parser = rawFileReader.ImportMassSpectraThermoMSFileReader(file_location)
 
@@ -58,40 +59,40 @@ def run_thermo(file_location):
 def calspec(msobj, refmasslist, order=2):
 
     calfn = MzDomainCalibration(msobj, refmasslist)
-    ref_mass_list_fmt = calfn.load_ref_mass_list(refmasslist)
+    ref_mass_list_fmt = calfn.load_ref_mass_list()
 
-    imzmeas, mzrefs = calfn.find_calibration_points(msobj, ref_mass_list_fmt,
+    imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                     calib_ppm_error_threshold=(-1.0, 1.0),
                                                     calib_snr_threshold=4)
 
     if len(mzrefs) < 5:
-        imzmeas, mzrefs = calfn.find_calibration_points(msobj, ref_mass_list_fmt,
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                         calib_ppm_error_threshold=(-1.5, 1.5),
                                                         calib_snr_threshold=4)
 
     if len(mzrefs) < 5:
-        imzmeas, mzrefs = calfn.find_calibration_points(msobj, ref_mass_list_fmt,
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                         calib_ppm_error_threshold=(-3, 3),
                                                         calib_snr_threshold=4)
 
     if len(mzrefs) < 5:
-        imzmeas, mzrefs = calfn.find_calibration_points(msobj,ref_mass_list_fmt,
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                         calib_ppm_error_threshold=(-5, 5),
                                                         calib_snr_threshold=4)    
 
     if len(mzrefs) < 5:
 
-        imzmeas, mzrefs = calfn.find_calibration_points(msobj, ref_mass_list_fmt,
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                         calib_ppm_error_threshold=(-7, 7),
                                                         calib_snr_threshold=4) 
 
     if len(mzrefs) < 5:
 
-        imzmeas, mzrefs = calfn.find_calibration_points(msobj, ref_mass_list_fmt,
+        imzmeas, mzrefs = calfn.find_calibration_points(ref_mass_list_fmt,
                                                         calib_ppm_error_threshold=(-10, 10),
                                                         calib_snr_threshold=4)
 
-    calfn.recalibrate_mass_spectrum(msobj, imzmeas, mzrefs, order=order)
+    calfn.recalibrate_mass_spectrum(imzmeas, mzrefs, order=order)
 
 def set_parameters(mass_spectrum, field_strength=12, pos=False):
 
@@ -115,6 +116,17 @@ def set_parameters(mass_spectrum, field_strength=12, pos=False):
         mass_spectrum.molecular_search_settings.min_ppm_error = -1
         mass_spectrum.molecular_search_settings.max_ppm_error = 1
 
+        mass_spectrum.settings.calib_sn_threshold = 4
+
+    elif field_strength == 7:
+
+        mass_spectrum.settings.max_calib_ppm_error = 3
+        mass_spectrum.settings.min_calib_ppm_error = -3
+
+        mass_spectrum.molecular_search_settings.error_method = 'None'
+        mass_spectrum.molecular_search_settings.min_ppm_error = -1
+        mass_spectrum.molecular_search_settings.max_ppm_error = 1
+
         mass_spectrum.settings.calib_sn_threshold = 4
     # 21T Data
     else:
@@ -126,7 +138,7 @@ def set_parameters(mass_spectrum, field_strength=12, pos=False):
         mass_spectrum.molecular_search_settings.min_ppm_error = -0.5
         mass_spectrum.molecular_search_settings.max_ppm_error = 0.5
 
-    # mass_spectrum.molecular_search_settings.url_database = "postgres://coremsdb:coremsmolform@localhost:5432/molformula"
+    mass_spectrum.molecular_search_settings.url_database = "postgresql+psycopg2://coremsappdb:coremsapppnnl@localhost:5432/coremsapp"
     mass_spectrum.molecular_search_settings.min_dbe = 0
     mass_spectrum.molecular_search_settings.max_dbe = 40
 
@@ -200,7 +212,11 @@ def run_nmdc_workflow(args):
 
         return "all_good", mass_spectrum
 
-    except:
+    except Exception as inst:
+
+        print(type(inst))    # the exception instance
+        print(inst.args)     # arguments stored in .args
+        print(inst)
         return 'error', None
 
 def monitor(target):