Added pre-split script to remove problematic drugs

Author: jjacobson95

build/broad_sanger/05a_remove_problem_drugs.py

@@ -0,0 +1,43 @@
+import gc
+import polars as pl 
+
+
+
+def main():
+
+    # Remove Problematic Drugs before Splitting Data
+    
+    # Load the datasets
+    all_drugs = pl.read_csv("broad_sanger_drugs.tsv", separator="\t")
+    all_experiments = pl.read_csv("broad_sanger_experiments.tsv", separator="\t")
+
+    # Define the brd_list with lowercase entries for case-insensitive matching
+    brd_list = [
+    'brd-k03911514',
+    'brd-k07442505',
+    'brd-k13185470',
+    'brd-k16130065',
+    'brd-k20514654',
+    'brd-k27188169',
+    'brd-k55473186',
+    'yl54',
+    'brd-k58730230',
+    'brd-k79669418',
+    'brd-k99584050']
+
+    # Identify rows in all_drugs that match brd_list entries (case insensitive)
+    removed_drugs = all_drugs.filter(pl.col("chem_name").str.to_lowercase().is_in(brd_list))
+
+    # Store the improve_drug_id IDs of removed entries
+    improve_drug_id = removed_drugs["improve_drug_id"].to_list()
+
+    # Remove these rows from all_drugs and all_experiments
+    all_drugs = all_drugs.filter(~pl.col("improve_drug_id").is_in(improve_drug_id))
+    all_experiments = all_experiments.filter(~pl.col("improve_drug_id").is_in(improve_drug_id))
+            
+    all_drugs.write_csv("broad_sanger_drugs.tsv", separator="\t")
+    all_experiments.write_csv("broad_sanger_experiments.tsv", separator="\t")
+    
+            
+if __name__ == "__main__":
+    main()

…ild/broad_sanger/05_separate_datasets.py …ld/broad_sanger/05b_separate_datasets.pybuild/broad_sanger/05_separate_datasets.py renamed to build/broad_sanger/05b_separate_datasets.py build/broad_sanger/05_separate_datasets.py renamed to build/broad_sanger/05b_separate_datasets.py

@@ -4,7 +4,6 @@
 
 
 def main():
-
     datasets_to_process = ["CCLE", "CTRPv2", "PRISM", "GDSCv1", "GDSCv2", "FIMM", "gCSI", "NCI60"]
     omics_datatypes = ["transcriptomics","proteomics", "copy_number","mutations"] # csv 
     samples_datatypes = ["samples"] #csv

build/broad_sanger/build_misc.sh

@@ -4,8 +4,12 @@ set -euo pipefail
 trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR
 
 cp /tmp/broad_sanger* .
-echo "Running 05_separate_datasets.py..."
-/opt/venv/bin/python 05_separate_datasets.py
+
+echo "Running 05a_remove_problem_drugs.py..."
+/opt/venv/bin/python 05a_remove_problem_drugs.py
+
+echo "Running 05b_separate_datasets.py..."
+/opt/venv/bin/python 05b_separate_datasets.py
 
 echo "Removing broad_sanger* files..."
 rm broad_sanger*

build/docker/Dockerfile.broad_sanger_omics

@@ -34,7 +34,8 @@ ADD build/broad_sanger/build_samples.sh ./
 ADD build/broad_sanger/build_omics.sh ./
 ADD build/utils/* ./
 ADD build/broad_sanger/build_misc.sh ./
-ADD build/broad_sanger/05_separate_datasets.py ./
+ADD build/broad_sanger/05a_remove_problem_drugs.py ./
+ADD build/broad_sanger/05b_separate_datasets.py ./
 
 ADD build/broad_sanger/requirements.txt .
 ADD build/broad_sanger/omics_requirements.r .