udpated with better sample matching

still missing transcriptomics samples

Author: sgosline

build/docker/Dockerfile.pancpdo

@@ -4,6 +4,7 @@ WORKDIR /usr/src/app
 
 COPY build/pancpdo/01-createPancPDOSamplesFile.py .
 COPY build/pancpdo/02-getPancPDOData.py .
+COPY build/pancpdo/02a-getPancPDDataFromSynapse.py .
 COPY build/pancpdo/03-getPancPDODrugs.py .
 COPY build/pancpdo/04-getPancPDOExperiments.py .
 COPY build/pancpdo/05-addPrecalcAUC.py .

build/pancpdo/01-createPancPDOSamplesFile.py

@@ -270,18 +270,26 @@ def filter_and_subset_data(df, maxval, mapfile):
     # Convert 'other_names' to string to ensure consistency
     longtab['other_names'] = longtab['other_names'].astype(str)
 
+    #print(longtab)
     # Reassign 'improve_sample_id's at the end
     unique_other_names = longtab['other_names'].unique()
     print("Number of unique 'other_names' after filtering:", len(unique_other_names))
 
+    ##UPDATE: assign them to common_names instead!
+    unique_common_names = longtab['common_name'].unique()
+    print("Number of unique 'common_names' after filtering:", len(unique_common_names))        
     # Create a new mapping
+    #mapping = pd.DataFrame({
+    #    'other_names': unique_other_names,
+    #    'improve_sample_id': range(int(maxval) + 1, int(maxval) + len(unique_other_names) + 1)
+        #})
     mapping = pd.DataFrame({
-        'other_names': unique_other_names,
-        'improve_sample_id': range(int(maxval) + 1, int(maxval) + len(unique_other_names) + 1)
-    })
+        'common_name':unique_common_names,
+        'improve_sample_id': range(int(maxval) +1, int(maxval) + len(unique_common_names)+1)
+        })
 
     # Merge the mapping back into 'longtab'
-    longtab = pd.merge(longtab, mapping, on='other_names', how='left')
+    longtab = pd.merge(longtab, mapping, on='common_name', how='left')
 
     # Debugging: Check longtab after reassigning IDs
     print("\nlongtab columns after reassigning 'improve_sample_id':", longtab.columns)

build/pancpdo/02-getPancPDOData.py

@@ -183,7 +183,7 @@ def use_gdc_tool(manifest_data, data_type, download_data):
 
         # Initialize retry variables
         retries = 0
-        max_retries = 5
+        max_retries = 1
 
         # Function to get downloaded file IDs
         def get_downloaded_ids(manifest_loc):

build/pancpdo/02a-getPancPDODataFromSynapse.py

@@ -100,7 +100,13 @@ def main():
     genes = pd.read_csv(args.genes)
 
     sc = synapseclient.login(args.token)
-
+    ##to double check identifiers, we use transcriptomics data since that determines what samples were sequenced
+    trans = pd.read_csv('/tmp/pancpdo_transcriptomics.csv.gz')
+    tsamps = samps[samps.improve_sample_id.isin(trans.improve_sample_id)]
+    print(samps.shape)
+    print(tsamps.shape)
+    
+          
     missingsamples = []
     if args.copy:
         ##query synapse view for files
@@ -112,11 +118,17 @@ def main():
             sname = row['name'].split('--')[0]
             print(sid,sname)
             path = sc.get(sid).path
-            if sname in set(samps.other_id):
+            if sname in set(tsamps.other_id):
+                print(sname+' in transcriptomics, using that id')
+                sampid = tsamps.loc[tsamps.other_id==sname]['improve_sample_id'].values[0]
+                missingsamples.append('copy,trans,'+sname)
+            elif sname in set(samps.other_id):
+                print(sname+' in samples but not transcriptomics, using other id')
                 sampid = samps.loc[samps.other_id==sname]['improve_sample_id'].values[0]
+                missingsamples.append("copy,notrans,"+sname)
             else:
-                print('Missing sample id for '+sname)
-                missingsamples.append('copy,'+sname)
+                print('Missing sample id for '+sname,' skipping for now')
+                missingsamples.append('copy,missed,'+sname)
                 continue
             sampid = samps.loc[samps.other_id==sname]['improve_sample_id'].values[0]
             res = parseCNVFile(path,sampid, genes)
@@ -132,8 +144,14 @@ def main():
             sname = row['name'].split('--')[0]
             sid = row.id
             print(sid,sname)
-            if sname in set(samps.other_id):
+            if sname in set(tsamps.other_id):
+                print(sname+' in transcriptomics, using that id')
+                sampid = tsamps.loc[tsamps.other_id==sname]['improve_sample_id'].values[0]
+                missingsamples.append('mutation,trans,'+sname)
+            elif sname in set(samps.other_id):
+                print(sname+' in samples but not transcriptomics, using other id')
                 sampid = samps.loc[samps.other_id==sname]['improve_sample_id'].values[0]
+                missingsamples.append('mutation,notrans,'+sname)
             else:
                 print('Missing sample id for '+sname)
                 missingsamples.append('mutation,'+sname)
@@ -144,6 +162,6 @@ def main():
             alldats.append(res)
         newmut = pd.concat(alldats)
         newmut.to_csv("/tmp/pancpdo_mutations.csv.gz",compression='gzip',index=False)
-    missingsamples.to_csv('missing.csv')
+    pd.DataFrame(missingsamples).to_csv('missing.csv',index=False,quoting=None,header=False)
 if __name__=='__main__':
     main()

build/pancpdo/build_omics.sh

@@ -6,6 +6,9 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit
 echo "Running 02-getPancPDOData.py for transcriptomics."
 python 02-getPancPDOData.py -m full_manifest.txt -t transcriptomics -o /tmp/pancpdo_transcriptomics.csv.gz -g $1 -s $2
 
+echo 'Running 02a-getPancPDODataFromSynapse.py for copy number and mutations'
+python 02a-getPancPDODataFromSynapse.py -g $1 -s $2 -t $SYNAPSE_AUTH_TOKEN -c -m
+
 #echo "Running 02-getPancPDOData.py for copy_number."
 #python 02-getPancPDOData.py -m full_manifest.txt -t copy_number -o /tmp/pancpdo_copy_number.csv.gz -g $1 -s $2