Revert "Reverted Sara's original changes, fixed Source issue in Sanger transcriptomics file"

This reverts commit 6faba50.

Author: jjacobson95

build/broad_sanger/02-broadSangerOmics.R

@@ -206,8 +206,7 @@ sanger_files<-function(fi,value){
       samps<-samps[-c(1:2),]|>as.data.frame()|>
         tibble::rownames_to_column('other_id')|>
         left_join(sanger_samples)|>
-        dplyr::rename(source='data_source',study='dataset_name') %>%
-        mutate(source = "Sanger")
+        dplyr::rename(source='data_source',study='dataset_name')
 
       missing<-subset(samps,is.na(improve_sample_id))|>
         dplyr::select(-c(other_id,improve_sample_id))|>
@@ -567,12 +566,16 @@ main<-function(){
     lapply(alltypes,function(dt){
         print(dt)
         temps<-sanger_files(sanger_filenames[[dt]],dt)|>tidyr::drop_na()
+        readr::write_csv(temps,file=paste0('/tmp/sanger_',dt,'.csv.gz'))
         tempd<-depmap_files(depmap_filenames[[dt]],dt)|>tidyr::drop_na()
-        readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz'))
+        readr::write_csv(tempd,file=paste0('/tmp/broad_',dt,'.csv.gz'))
+
+#        readr::write_csv(rbind(tempd,temps),file=paste0('/tmp/broad_sanger_',dt,'.csv.gz'))
         rm(tempd)
         rm(temps)
     })
 
 }
 
-main()
+main()
+

build/broad_sanger/02a-broad_sanger_proteomics.py

@@ -48,6 +48,9 @@ def main():
 
     full[['study']] = 'DepMap'
     full[['source']] = 'Broad'
+    ##now save to separate files
+    full.dropna(axis=0)
+    full.to_csv('/tmp/broad_proteomics.csv.gz', index=False, compression='gzip')
 
 
     ##now get sanger
@@ -69,9 +72,9 @@ def main():
     full2.loc[:,['study']] = 'Sanger'
     full2.loc[:,['source']] = 'Sanger'
 
-    full3 = pd.concat([full,full2])
-    print(full3)
-    full3.dropna(axis=0)
-    full3.to_csv('/tmp/broad_sanger_proteomics.csv.gz',index=False, compression='gzip')
+    #full3 = pd.concat([full,full2])
+    #print(full3)
+    full2.dropna(axis=0)
+    full2.to_csv('/tmp/sanger_proteomics.csv.gz',index=False, compression='gzip')
 
-main()
+main()

build/broad_sanger/05b_separate_datasets.py

@@ -13,14 +13,14 @@ def main():
 
 
     dataset_sources = {
-        "CCLE": ["Broad"],
-        "CTRPv2": ["Broad"],
-        "PRISM": ["Broad"],
-        "GDSCv1": ["Sanger"],
-        "GDSCv2": ["Sanger"],
-        "FIMM": ["Broad"],
-        "gCSI": ["Broad"],  # gCSI generates its own omics data but it is comparable to CCLE. In future, retrive gCSI omics.
-        "NCI60": ["Broad"]
+        "CCLE": ["broad"],
+        "CTRPv2": ["broad"],
+        "PRISM": ["broad"],
+        "GDSCv1": ["sanger"],
+        "GDSCv2": ["sanger"],
+        "FIMM": ["broad"],
+        "gCSI": ["broad"],  # gCSI generates its own omics data but it is comparable to CCLE. In future, retrive gCSI omics.
+        "NCI60": ["broad"]
     }
 
     for dataset in datasets_to_process:
@@ -70,14 +70,16 @@ def main():
 
         #One by one, filter other Omics files, write to file, delete from mem.
         for omics in omics_datatypes:
-            omics_filename_in = f"broad_sanger_{omics}.csv"
+            ds = dataset_sources[dataset][0]
+            #print(ds)
+            omics_filename_in = f"{ds}_{omics}.csv"
             if os.path.isfile(omics_filename_in + ".gz"):
                 omics_filename_in += ".gz"
 
             omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower()
             omics_df = pl.read_csv(omics_filename_in)
             omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids))
-            omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset]))
+#            omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset]))
             omics_df.write_csv(omics_filename_out) #csv
 
             #Rewrite as gzipped if needed
@@ -118,4 +120,4 @@ def main():
             gc.collect()
 
 if __name__ == "__main__":
-    main()
+    main()