fix for #343

Here we separate out the omics data early on in the pipeline to ensure better assignment.

Author: sgosline

README.md

@@ -1,5 +1,7 @@
 ## Cancer Omics Drug Experiment Response Dataset 
 
+
+
 There is a recent explosion of deep learning algorithms that to tackle the computational problem of predicting drug treatment outcome from baseline molecular measurements. To support this,we have built a benchmark dataset that harmonizes diverse datasets to better assess algorithm performance.
 
 This package collects diverse sets of paired molecular datasets with corresponding drug sensitivity data. All data here is reprocessed and standardized so it can be easily used as a benchmark dataset for the 

build/broad_sanger/02-broadSangerOmics.R

@@ -566,12 +566,16 @@ main<-function(){
     lapply(alltypes,function(dt){
         print(dt)
         temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()
+        readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz'))
         tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()
-        readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz'))
+        readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz'))
+
+#        readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz'))
         rm(tempd)
         rm(temps)
     })
 
 }
 
 main()
+

build/broad_sanger/02a-broad_sanger_proteomics.py

@@ -48,6 +48,9 @@ def main():
 
     full[['study']] = 'DepMap'
     full[['source']] = 'Broad'
+    ##now save to separate files
+    full.dropna(axis=0)
+    full.to_csv('/tmp/broad_proteomics.csv.gz', index=False, compression='gzip')
 
 
     ##now get sanger
@@ -69,9 +72,9 @@ def main():
     full2.loc[:,['study']] = 'Sanger'
     full2.loc[:,['source']] = 'Sanger'
 
-    full3 = pd.concat([full,full2])
-    print(full3)
-    full3.dropna(axis=0)
-    full3.to_csv('/tmp/broad_sanger_proteomics.csv.gz',index=False, compression='gzip')
+    #full3 = pd.concat([full,full2])
+    #print(full3)
+    full2.dropna(axis=0)
+    full2.to_csv('/tmp/sanger_proteomics.csv.gz',index=False, compression='gzip')
 
 main()

build/broad_sanger/05b_separate_datasets.py

@@ -13,14 +13,14 @@ def main():
 
 
     dataset_sources = {
-        "CCLE": ["Broad"],
-        "CTRPv2": ["Broad"],
-        "PRISM": ["Broad"],
-        "GDSCv1": ["Sanger"],
-        "GDSCv2": ["Sanger"],
-        "FIMM": ["Broad"],
-        "gCSI": ["Broad"],  # gCSI generates its own omics data but it is comparable to CCLE. In future, retrive gCSI omics.
-        "NCI60": ["Broad"]
+        "CCLE": ["broad"],
+        "CTRPv2": ["broad"],
+        "PRISM": ["broad"],
+        "GDSCv1": ["sanger"],
+        "GDSCv2": ["sanger"],
+        "FIMM": ["broad"],
+        "gCSI": ["broad"],  # gCSI generates its own omics data but it is comparable to CCLE. In future, retrive gCSI omics.
+        "NCI60": ["broad"]
     }
 
     for dataset in datasets_to_process:
@@ -70,14 +70,16 @@ def main():
 
         #One by one, filter other Omics files, write to file, delete from mem.
         for omics in omics_datatypes:
-            omics_filename_in = f"broad_sanger_{omics}.csv"
+            ds = dataset_sources[dataset][0]
+            #print(ds)
+            omics_filename_in = f"{ds}_{omics}.csv"
             if os.path.isfile(omics_filename_in + ".gz"):
                 omics_filename_in += ".gz"
 
             omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower()
             omics_df = pl.read_csv(omics_filename_in)
             omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids))
-            omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset]))
+#            omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset]))
             omics_df.write_csv(omics_filename_out) #csv
 
             #Rewrite as gzipped if needed