Updated mapping scripts with all datasets and removed cptac by default. Removed tons of print statements so debugging the full build would be easier

Author: jjacobson95

build/hcmi/02-getHCMIData.py

@@ -581,14 +581,14 @@ def align_to_schema(data, data_type, chunksize=7500,samples_path='/tmp/hcmi_samp
 
     # Process in chunks
     merged_data = pl.DataFrame()
-    print(f"merged_data:\n {merged_data}")
+    # print(f"merged_data:\n {merged_data}")
 
     for i in range(0, len(data), chunksize):
         chunk = data[i:i + chunksize]
         if data_type == "mutations":
             chunk = chunk.rename({"Variant_Classification": "variant_classification"})
         chunk = chunk.select(selected_columns)
-        print(f"chunk: \n{chunk}")
+        # print(f"chunk: \n{chunk}")
         merged_chunk = samples.join(chunk, left_on='other_names', right_on='aliquot_id', how='inner')
         merged_chunk = merged_chunk.drop(["aliquot_id", "other_names"])
 

build/pancpdo/02-getPancPDOData.py

@@ -415,8 +415,8 @@ def map_and_combine(dataframe_list, data_type, metadata, entrez_map_file):
     df_metadata = pl.DataFrame(metadata_dict)
 
     # Merge the metadata DataFrame with the final dataframe based on 'file_id'
-    print(df_metadata)
-    print(final_dataframe)
+    # print(df_metadata)
+    # print(final_dataframe)
     final_dataframe = final_dataframe.join(df_metadata, on='file_id', how='left')
 
     return final_dataframe
@@ -540,14 +540,14 @@ def align_to_schema(data, data_type, chunksize=7500,samples_path='/tmp/hcmi_samp
 
     # Process in chunks
     merged_data = pl.DataFrame()
-    print(f"merged_data:\n {merged_data}")
+    # print(f"merged_data:\n {merged_data}")
 
     for i in range(0, len(data), chunksize):
         chunk = data[i:i + chunksize]
         if data_type == "mutations":
             chunk = chunk.rename({"Variant_Classification": "variant_classification"})
         chunk = chunk.select(selected_columns)
-        print(f"chunk: \n{chunk}")
+        # print(f"chunk: \n{chunk}")
         merged_chunk = samples.join(chunk, left_on='other_names', right_on='aliquot_id', how='inner')
         merged_chunk = merged_chunk.drop(["aliquot_id", "other_names"])
 

build/utils/build_drug_desc.py

@@ -76,17 +76,17 @@ def main():
 
     cores = multiprocessing.cpu_count()
     ncors = cores-1
-    print("Running with "+str(ncors)+' out of '+str(cores)+' processors')
-    print('Adding drug table for '+args.drugtable)
+    # print("Running with "+str(ncors)+' out of '+str(cores)+' processors')
+    # print('Adding drug table for '+args.drugtable)
     tab = pd.read_csv(args.drugtable,sep='\t')
 
     cansmiles = [a for a in set(tab.canSMILES) if str(a)!='nan']
     #    isosmiles = list(set(tab.isoSMILES))
     morgs = smiles_to_fingerprint(cansmiles)
 
     ids = pd.DataFrame(tab[['improve_drug_id','canSMILES']]).drop_duplicates()
-    print("IDS columns:", ids.columns.tolist())
-    print("MORGS columns:", morgs.columns.tolist())
+    # print("IDS columns:", ids.columns.tolist())
+    # print("MORGS columns:", morgs.columns.tolist())
     id_morg = ids.rename({"canSMILES":'smile'},axis=1).merge(morgs)[['improve_drug_id','structural_descriptor','descriptor_value']]
 
     mords = smiles_to_mordred(cansmiles,nproc=ncors)
@@ -105,8 +105,8 @@ def main():
     full['improve_drug_id'] = full['improve_drug_id'].astype(str).str.strip()
     mask = full['improve_drug_id'].str.match(r'^SMI_\d+$')
     n_dropped = (~mask).sum()
-    if n_dropped:
-        print(f"Dropping {n_dropped} malformed improve_drug_id rows.")
+    # if n_dropped:
+    #     print(f"Dropping {n_dropped} malformed improve_drug_id rows.")
     full = full[mask].copy()
 
 

build/utils/pubchem_retrieval.py

@@ -205,79 +205,6 @@ def timeout_handler(signum, frame):
     should_continue = False
 
 
-# def update_dataframe_and_write_tsv(unique_names, output_filename="drugs.tsv", ignore_chems="ignore_chems.txt",
-#                                    batch_size=1, isname=True, time_limit=48 * 60 * 60):
-#     """
-#     Updates the data frame with drug information and writes it to a TSV file.
-
-#     Parameters:
-#     - unique_names (iterable): List of unique compound names or CIDs.
-#     - output_filename (str): File path to the output TSV file.
-#     - ignore_chems (str): File path to log ignored compounds.
-#     - batch_size (int): Number of compounds to process in each batch.
-#     - isname (bool): True if unique_names are names, False if they're CIDs.
-#     - time_limit (int): Time limit for the script in seconds. This is a remnant of the GitHub Action CI.
-    
-#     Returns:
-#     - None
-#     """
-#     global should_continue, existing_synonyms, existing_pubchemids
-#     signal.signal(signal.SIGALRM, timeout_handler)
-#     signal.alarm(time_limit)
-#     print(f'Starting with {len(unique_names)} unique drug names/IDs')
-
-#     try:
-#         print(f'Reading existing data from {output_filename}')
-#         read_existing_data(output_filename)
-#         if isname:
-#             unique_names = set([str(name).lower() for name in unique_names if not pd.isna(name)])
-#             unique_names = set(unique_names) - set(existing_synonyms)
-#             print(f'Looking at {len(unique_names)} names')
-#         else:
-#             unique_names = set([str(name) for name in unique_names if not pd.isna(name)])
-#             unique_names = set(unique_names) - set(existing_pubchemids)
-#             print(f'Looking at {len(unique_names)} IDs')
-#         ignore_chem_set = set()
-#         if os.path.exists(ignore_chems):
-#             with open(ignore_chems, 'r') as file:
-#                 for line in file:
-#                     ignore_chem_set.add(line.strip())
-#         unique_names = list(set(unique_names) - ignore_chem_set)
-
-#         print(f"{len(unique_names)} Drugs to search")
-#         for i in range(0, len(unique_names), batch_size):
-#             if not should_continue:
-#                 break
-#             if unique_names[i] in existing_synonyms or unique_names[i] in existing_pubchemids:
-#                 continue
-
-#             batch = unique_names[i:i + batch_size]
-#             data = fetch_data_for_batch(batch, ignore_chems, isname)
-#             if data:
-#                 file_exists = os.path.isfile(output_filename)
-#                 mode = 'a' if file_exists else 'w'
-#                 with open(output_filename, mode) as f:
-#                     if not file_exists:
-#                         f.write("improve_drug_id\tchem_name\tpubchem_id\tcanSMILES\tInChIKey\tformula\tweight\n")
-#                     for entry in data:
-#                         f.write(f"{entry['improve_drug_id']}\t{entry['name']}\t{entry.get('CID', '')}\t"
-#                                 f"{entry['SMILES']}\t{entry['InChIKey']}\t"
-#                                 f"{entry['MolecularFormula']}\t{entry['MolecularWeight']}\n")
-
-#                 with open(ignore_chems, "a") as ig_f:
-#                     for entry in data:
-#                         if isname:
-#                             ig_f.write(f"{entry['name']}\n")
-#                         else:
-#                             ig_f.write(f"{entry.get('CID', '')}\n")
-
-#     except Exception as e:
-#         print(f"An unexpected error occurred: {e}")
-#     finally:
-#         signal.alarm(0)
-
-
-
 
 
 def _load_prev_drugs_union(prevDrugFilepath: str) -> pd.DataFrame:
@@ -497,8 +424,8 @@ def update_dataframe_and_write_tsv(unique_names,
         nums_comb = pd.to_numeric(extracted_comb, errors="coerce")
         if not nums_comb.empty:
             new_ids = set(combined.loc[nums_comb > previous_max, "improve_drug_id"])
-            if new_ids:
-                print(f"Newly assigned improve_drug_id(s): {new_ids}")
+            # if new_ids:
+            #     print(f"Newly assigned improve_drug_id(s): {new_ids}")
 
     # --- 9) union and filter final DataFrame by improve_drug_id(s) ---
     keep_ids = hit_ids.union(new_ids)

scripts/align_drug_descriptors.py

@@ -65,8 +65,8 @@ def rewrite_files(files, ref):
                 key = (row['improve_drug_id'], row['structural_descriptor'])
                 correct = ref.get(key)
                 if correct is not None and row['descriptor_value'] != correct:
-                    print(f"Fixing {key} in {os.path.basename(fp)}: "
-                          f"{row['descriptor_value']} to {correct}")
+                    # print(f"Fixing {key} in {os.path.basename(fp)}: "
+                    #       f"{row['descriptor_value']} to {correct}")
                     row['descriptor_value'] = correct
                     changed = True
                 writer.writerow(row)

scripts/map_improve_drug_ids.py

@@ -369,7 +369,7 @@ def main():
                         help='Build date in YYYY-MM-DD. Default=now.')
     parser.add_argument('--version', required=True,
                         help='Build version. Must be unique per build.')
-    parser.add_argument('--datasets', default='gdscv1,ccle,ctrpv2,fimm,gcsi,gdscv2,nci60,prism,beataml,mpnst,mpnstpdx,pancpdo,bladderpdo,sarcpdo',
+    parser.add_argument('--datasets', default='gdscv1,ccle,ctrpv2,fimm,gcsi,gdscv2,nci60,prism,beataml,pancpdo,bladderpdo,sarcpdo,liverpdo,novartispdx,mpnst',
                         help='Comma-separated list of datasets.')
     parser.add_argument('--local_dir', default='data',
                         help='Directory containing TSV files.')
@@ -378,6 +378,7 @@ def main():
     parser.add_argument('--input_files', nargs='+',
                         help='List of input files to process. If specified, only these files will be processed.')
     args = parser.parse_args()
+    
 
     # Set build_date
     build_date = args.build_date or datetime.utcnow().strftime("%Y-%m-%d")

scripts/map_improve_sample_ids.py

@@ -412,7 +412,7 @@ def main():
                         help='Build date in YYYY-MM-DD. Default=now.')
     parser.add_argument('--version', required=True,
                         help='Build version. Must be unique per build.')
-    parser.add_argument('--datasets', default='ccle,ctrpv2,fimm,gcsi,gdscv1,gdscv2,nci60,prism,hcmi,beataml,cptac,mpnst,mpnstpdx,pancpdo,bladderpdo,sarcpdo',
+    parser.add_argument('--datasets', default='ccle,ctrpv2,fimm,gcsi,gdscv1,gdscv2,nci60,prism,hcmi,beataml,cptac,pancpdo,bladderpdo,sarcpdo,liverpdo,novartispdx,mpnst',
                         help='Comma-separated list of datasets, e.g., beataml,ccle')
     parser.add_argument('--local_dir', default='data',
                         help='Directory containing all CSV/TSV files.')