updated with panc pdo build working

Author: sgosline

build/build_dataset.py

@@ -98,7 +98,7 @@ def process_drugs(executor, dataset, use_prev_dataset, should_continue):
     '''
     Build the drugs file for the specified dataset.
     '''
-    if dataset in ['cptac', 'hcmi','pancpdo']:
+    if dataset in ['cptac', 'hcmi']:
         return  # No drugs to process for these datasets
 
     drugs_file = f'local/{dataset}_drugs.tsv'
@@ -166,7 +166,7 @@ def process_experiments(executor, dataset, should_continue):
     '''
     Build the experiments files for the specified dataset.
     '''
-    if dataset in ['cptac', 'hcmi','pancpdo']:
+    if dataset in ['cptac', 'hcmi']:
         return  # No experiments to process for these datasets
 
     experiments_file = f'local/{dataset}_experiments.tsv'

build/docker/Dockerfile.pancpdo

@@ -17,4 +17,4 @@ ENV MPLCONFIGDIR=/app/tmp/matplotlib
 RUN mkdir -p /app/tmp/matplotlib
 
 RUN pip install --no-cache-dir -r requirements.txt
-
+VOLUME ['/tmp']

build/pancpdo/01-createPancPDOSamplesFile.py

@@ -5,6 +5,28 @@
 import numpy as np
 
 
+#this is table S1 - it has a mapping from patient number to organoid
+sample_mapping='https://aacr.silverchair-cdn.com/aacr/content_public/journal/cancerdiscovery/8/9/10.1158_2159-8290.cd-18-0349/5/21598290cd180349-sup-199398_2_supp_4775186_p95dln.xlsx?Expires=1738004990&Signature=yngaaKNaXfIPCr-xLS2bDjX49n9py8JC7NBwi3q7m7ARYnK573eZwavFYmJOZVanL555vUWAr5x5k9b7IKj4VWHtZ-dts7BDzHd14AZh15LbsorJh-r3gjPliF7v1PIoAcGnEXjma2~kosmoDmyK0EDWXQCOE48tAaG5hFtaWAMMAINRMeBNgtDYk937Npc3Wb0IcGAdlgD2TJd8KJW2jQmcRspY1hfYssiS3BcWzuJrP-DVJeb-1V7-BnVNL6cVCkr7zHhau50H6aVgMVzk33F0gjCphl4r90OIx9UwE59hyNHbN9rFeeW26kDQpgCQKCj98Ol6CNQfLDsb2Zc5dQ__&Key-Pair-Id=APKAIE5G5CRDK6RD3PGA'
+
+
+def get_organoid_samples(sample_tab):
+    '''
+    takes as input a processed list of samples from HCMI and appends it with the 'organoid' identifier from the papers table S1 described above
+    '''
+    map = pd.read_excel(sample_mapping, sheet_name='Patient-Derived Organoid Cohort', skiprows=1)
+    pmap = map[['Patient number','Organoid']]
+    pmap = pmap.rename(columns={'Patient number':'common_name','Organoid':'experimentId'})
+    
+    #join with sampletab
+    sample_tab.common_name=[str(a) for a in sample_tab.common_name]
+    pmap.common_name = [str(a) for a in pmap.common_name]
+    ocols=['common_name','other_names','model_type','cancer_type','improve_sample_id','species']
+    red_tab = sample_tab[ocols].merge(pmap)
+    
+    #then add in organoid number
+    newsamp = red_tab.melt(id_vars=ocols,value_vars='experimentId',var_name='other_id_source',value_name='other_id').drop_duplicates()
+    res = pd.concat([sample_tab,newsamp])
+    return res
 
 def align_to_linkml_schema(input_df):
     """
@@ -320,6 +342,7 @@ def main():
     output = filter_and_subset_data(df,maxval,args.map)
     aligned = align_to_linkml_schema(output)
     print(aligned)
+    aligned = get_organoid_samples(aligned)
     aligned.to_csv("/tmp/pancpdo_samples.csv",index=False)
 
 main()

build/pancpdo/03-getPancPDODrugs.py

@@ -2,48 +2,52 @@
 import os
 import argparse
 import synapseclient as sc
-
+from pubchem_retrieval import update_dataframe_and_write_tsv
 
 
 ###figshare link:
 
 filelink='https://aacr.figshare.com/ndownloader/files/39996295'
-synid = 'syn64333325'
+#synid = 'syn64333325'
 ##get third tab and drugsa re listeda cross top
 
-
+##sup table drug list (in column names)
+tablink = 'https://aacr.silverchair-cdn.com/aacr/content_public/journal/cancerdiscovery/8/9/10.1158_2159-8290.cd-18-0349/5/21598290cd180349-sup-199398_2_supp_4775187_p95dln.xlsx?Expires=1738004990&Signature=av8XadTm9AmI20O2Y7J7aHDtPbpluKJIfI5ubsoiYJ15D0zh5p1ltF4a7-DCSWTSMs-qX5TD09shxHeqkQ2NkLWHZsXoCD5KyREGhEgcDAvWZ1V9kwXDm0bjpINipAPPtC20oeuw6c~hPooF3Mtgzp4MzMCCjcVwfn05u27a0kS0yifBi11wQj3nmHlR3ym-2fYkFuqQtnNPCzH8-yIw21y0kTvXrNodAzC5pGA8qUK4PLxBt52xUIvTEPsPiPjXwBnDCfVsLGGdDYIY25lEPKiA403q6kFYvrSQ3bsTvM4kuvltb7yS4AXjK0-tthMOKbqq8~uREmJCcueADUF91g__&Key-Pair-Id=APKAIE5G5CRDK6RD3PGA'
 
 def getDrugNames(token=""):
-    if token !="":
-        syn = sc.login(token)
-    else:
-        syn = sc.login()
-    fpath = syn.get(synid).path
-    print(fpath)
-    tab = pd.read_excel(fpath,sheet_name='concentrations')
-    drugs = [a.lower() for a in tab.columns]
-    return drugs
 
+    #chemo drugs
+    ctab = pd.read_excel(tablink,sheet_name=1,skiprows=1)
+    #targeted drugs
+    ttab = pd.read_excel(tablink,sheet_name=2,skiprows=1)
+    drugs = [a.lower() for a in ctab.columns]+[a.lower() for a in ttab.columns]
+    drugs = set(drugs)-set(['sample id','insensitive'])
+    return drugs
 
 
 def main():
     parser = argparse.ArgumentParser(description='Download and match pancpdodrugs')
-    parser.add_argument('-p', '--pat',help='Synapse authentication token with permission to syn64333325')
-    parser.add_argument('-d', '--prevDrugFile',help='Comma-delimited list of previous drug files')
-    parser.add_argument('-o', '--output', default = '/tmp/pancpdo_drugs.tsv.gz')
+#    parser.add_argument('-p', '--pat',help='Synapse authentication token with permission to syn64333325')
+    parser.add_argument('-d', '--prevDrugFile', default=None, help='Comma-delimited list of previous drug files')
+    parser.add_argument('-o', '--output', default = '/tmp/pancpdo_drugs.tsv')
 
     args = parser.parse_args()
-    newdrugs = getDrugNames(args.pat)
-
-    prevdrugs = [pd.read_csv(t,sep='\t') for t in args.prevDrugFile.split(',')]
-    alldrugs = pd.concat(prevdrugs).drop_duplicates()
-
-    imps = alldrugs[alldrugs.chem_name.isin(newdrugs)]
-    newdrugs = alldrugs[alldrugs.improve_drug_id.isin(imps.improve_drug_id)]
-
-    ##write drugs
-    newdrugs.to_csv(args.output, sep='\t', compression='gzip', index=False)
-
+    newdrugnames = getDrugNames()
+
+    alldrugs = []
+    if args.prevDrugFile is not None and args.prevDrugFile is not "":
+        prevdrugs = [pd.read_csv(t,sep='\t') for t in args.prevDrugFile.split(',')]
+        alldrugs = pd.concat(prevdrugs).drop_duplicates()
+
+        imps = alldrugs[alldrugs.chem_name.isin(newdrugnames)]
+        newdrugs = alldrugs[alldrugs.improve_drug_id.isin(imps.improve_drug_id)]
+        
+        ##write drugs
+        newdrugs.to_csv(args.output, sep='\t', index=False)
+
+    if len(alldrugs)==0 or len(newdrugnames)>len(set(newdrugs.improve_drug_id)): #we have more names we didn't match
+        print('Missing drugs in existing file, querying pubchem')
+        update_dataframe_and_write_tsv(newdrugnames,args.output)
     ##calculate drug descriptors
 
 

build/pancpdo/04-getPancPDOExperiments.py

@@ -5,7 +5,7 @@
 import argparse
 import synapseclient as sc
 import math
-
+import re
 
 def main():
     ##current AUC values are here: https://aacr.figshare.com/ndownloader/files/39996295 tabs 2 and 3
@@ -17,14 +17,17 @@ def main():
 
     args = parser.parse_args()
     newdata = get_data(args.pat)
-    newdata = newdata.rename(columns={'Organoid':'other_id','Drug':'chem_name','Dose':'DOSE','FracResponse':'GROWTH','Passage':'time'})
+    newdata = newdata.rename(columns={'Organoid':'other_id','Drug':'chem_name','Dose':'DOSE','PercResponse':'GROWTH','Passage':'time'})
+#    print(newdata)
     newdata = newdata[['other_id','chem_name','DOSE','GROWTH']]
     newdata[['time']]='120'
     newdata[['time_unit']]='hours'
     newdata[['study']]='pancpdo'
     newdata[['source']]='TiriacEtAl2018'
+    print('collected doses and response for '+str(len(set(newdata.chem_name)))+' drugs and '+str(len(set(newdata.other_id)))+' samples')
 #    'source', 'improve_sample_id', 'Drug', 'study','time','time_unit'
-    mappedresponse = map_drugs_to_samps(newddata,args.drugs,args,samples)
+    mappedresponse = map_to_drugs_samps(newdata,args.drugs,args.samples)
+    print('mapped doses and response for '+str(len(set(mappedresponse.Drug)))+' drugs and '+str(len(set(mappedresponse.improve_sample_id)))+' samples')
     mappedresponse.to_csv(args.output, sep='\t', index=False)
 
 def map_to_drugs_samps(dose_rep,drugfile,sampfile):
@@ -35,8 +38,10 @@ def map_to_drugs_samps(dose_rep,drugfile,sampfile):
     samps = pd.read_csv(sampfile)
 
     merged = dose_rep.merge(drugs).merge(samps)
-    merged = merged[['improve_sample_id','improve_drug_id','DOSE','GROWTH','time','time_unit','study','source']]
-    merged = merged.rename(columns={'improve_drug_id':'Drug'})
+
+    merged = merged.rename(columns={'improve_drug_id':'Drug'}) 
+    merged = merged[['improve_sample_id','Drug','DOSE','GROWTH','time','time_unit','study','source']].drop_duplicates()
+    print(merged)
     return merged
 
 def get_data(token):
@@ -72,10 +77,13 @@ def get_data(token):
 
 
     ##now melt the data into single columns
-    rtab = responses.melt(id_vars = responses.columns[0:4],value_vars=responses.columns[4:10], var_name='Drug',value_name='Response')
-    
+    rtab = responses.melt(id_vars = responses.columns[0:4],value_vars=responses.columns[4:20], var_name='Drug',value_name='Response')
+    print('Collected results from '+str(len(set(rtab.Drug)))+' drugs and '+str(len(set(rtab.Organoid)))+' organoids')
+    #print(set(rtab.Drug))
     ##rename the drugs
     rtab[['Drug','Rep']]=rtab['Drug'].str.lower().str.split('.',expand=True)
+    rtab.Drug=[re.sub('-','',a) for a in rtab.Drug]
+    #print(set(rtab.Drug))
     newrep=[]
     for r in rtab.Rep:
         if r is None:
@@ -94,12 +102,16 @@ def get_data(token):
     ##dosenum isa dummy value to use for merging since we need to repeat the concentrations over and over
     dosenum = [a for a in range(15)]
     rtab['Dosenum']=dosenum*int(rtab.shape[0]/15)
-               
+
+    #print(set(rtab.Drug))
     ##merge the concentrations
     concs = concs.dropna().melt(value_vars=concs.columns,var_name='Drug',value_name='Dose')
+    print(concs)
+    concs.Dose = [d*10.0**6.0 for d in concs.Dose] ## convert M to uM here
+    
     concs.Drug=concs.Drug.str.lower()
     concs['Dosenum'] = dosenum*int(concs.shape[0]/15)##creating dosenum here to merge
-
+    #print(set(concs.Drug))
 
     return rtab.merge(concs)
 

build/pancpdo/05-compare_with_scores.py

@@ -0,0 +1,26 @@
+'''
+This script pulls down pre-computed curves and compares our fits with theirs
+'''
+
+import pandas as pd
+import math
+import argparse
+
+tablink = 'https://aacr.silverchair-cdn.com/aacr/content_public/journal/cancerdiscovery/8/9/10.1158_2159-8290.cd-18-0349/5/21598290cd180349-sup-199398_2_supp_4775187_p95dln.xlsx?Expires=1738004990&Signature=av8XadTm9AmI20O2Y7J7aHDtPbpluKJIfI5ubsoiYJ15D0zh5p1ltF4a7-DCSWTSMs-qX5TD09shxHeqkQ2NkLWHZsXoCD5KyREGhEgcDAvWZ1V9kwXDm0bjpINipAPPtC20oeuw6c~hPooF3Mtgzp4MzMCCjcVwfn05u27a0kS0yifBi11wQj3nmHlR3ym-2fYkFuqQtnNPCzH8-yIw21y0kTvXrNodAzC5pGA8qUK4PLxBt52xUIvTEPsPiPjXwBnDCfVsLGGdDYIY25lEPKiA403q6kFYvrSQ3bsTvM4kuvltb7yS4AXjK0-tthMOKbqq8~uREmJCcueADUF91g__&Key-Pair-Id=APKAIE5G5CRDK6RD3PGA'
+
+
+
+def main():
+    ##so far we have data for 'chemo' tab. how about the targeted tab?
+    
+    chemo = pd.read_excel(tablink,sheet_name=1)
+    targeted = res = pd.read_excel(tablink,sheet_name=2)
+    
+    
+    ##add in these scores to the drug file
+    ##get drug file
+
+    
+
+if __name__=='__main__':
+    main()

build/pancpdo/build_drugs.sh

@@ -1,3 +1,4 @@
 
-/opt/venv/bin/python3 03-getPancPDODrugs.py --pat $SYNAPSE_AUTH_TOKEN --prevDrugFile=$1 --output=/tmp/pancpdo_drugs.tsv.gz
-/opt/venv/bin/python3 build_drug_desc.py --drugtable /tmp/pancpdo_drugs.tsv.gz --desctable /tmp/pancpdo_drug_descriptors.tsv.gz
+python 03-getPancPDODrugs.py --prevDrugFile=$1 --output=/tmp/pancpdo_drugs.tsv
+python build_drug_desc.py --drugtable /tmp/pancpdo_drugs.tsv --desctable /tmp/pancpdo_drug_descriptors.tsv.gz
+

build/pancpdo/build_exp.sh

@@ -4,5 +4,9 @@ set -euo pipefail
 trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit 1' ERR
 
 echo "Running 04-drug_dosage_and_curves.py with drugfile $2 and curSampleFile $1"
-/opt/venv/bin/python 04-getPancPDOExperiments.py --pat $SYNAPSE_AUTH_TOKEN --drugs $2 --samples $1 --output /tmp/pancpdo_doserep.tsv
-/opt/venv/bin/python fit_curv.py --input /tmp/panpdo_doserep.tsv --output /tmp/pancpdo_experiments.tsv.gz
+python 04-getPancPDOExperiments.py --pat $SYNAPSE_AUTH_TOKEN --drugs $2 --samples $1 --output /tmp/pancpdo_doserep.tsv
+python fit_curve.py --input /tmp/pancpdo_doserep.tsv
+
+##now move file and gzip
+mv /tmp/pancpdo_doserep.tsv /tmp/pancpdo_experiments.tsv
+gzip /tmp/pancpdo_experiments.tsv

build/pancpdo/build_omics.sh

@@ -9,5 +9,5 @@ python 02-getPancPDOData.py -m full_manifest.txt -t transcriptomics -o /tmp/panc
 #echo "Running 02-getPancPDOData.py for copy_number."
 #python 02-getPancPDOData.py -m full_manifest.txt -t copy_number -o /tmp/pancpdo_copy_number.csv.gz -g $1 -s $2
 
-echo "Running 02-getPancPDOData.py for mutations."
-python 02-getPancPDOData.py -m full_manifest.txt -t mutations -o /tmp/pancpdo_mutations.csv.gz -g $1 -s $2
+#echo "Running 02-getPancPDOData.py for mutations."
+#python 02-getPancPDOData.py -m full_manifest.txt -t mutations -o /tmp/pancpdo_mutations.csv.gz -g $1 -s $2