updates and fixes

Author: jjacobson95

build/build_dataset.py

@@ -41,6 +41,7 @@ def process_docker(dataset,validate):
         'hcmi': ['hcmi'],
         'beataml': ['beataml'],
         'mpnst': ['mpnst'],
+        'mpnstpdx': ['mpnstpdx'],
         'cptac': ['cptac'],
         'genes': ['genes'],
         'upload': ['upload']
@@ -121,7 +122,8 @@ def process_omics(executor, dataset, should_continue):
         'mpnst': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'broad_sanger': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
         'cptac': ['copy_number', 'mutations', 'proteomics', 'transcriptomics'],
-        'hcmi': ['mutations', 'transcriptomics']
+        'hcmi': ['mutations', 'transcriptomics'],
+        'mpnstpdx':['copy_number', 'mutations', 'proteomics', 'transcriptomics']
     }
 
     expected_omics = dataset_omics_files.get(dataset, [])

build/docker/Dockerfile.mpnstPDX

@@ -0,0 +1,54 @@
+FROM r-base:4.3.2
+
+# Set environment to noninteractive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Update package list and install required packages
+RUN apt-get update && \
+    apt-get install -y build-essential wget curl libcurl4-openssl-dev libxml2-dev \
+    zlib1g-dev libssl-dev libbz2-dev libreadline-dev libsqlite3-dev libffi-dev
+
+# Download and compile Python 3.10 with shared library support
+RUN wget https://www.python.org/ftp/python/3.10.12/Python-3.10.12.tgz && \
+    tar -xf Python-3.10.12.tgz && \
+    cd Python-3.10.12 && \
+    ./configure --enable-optimizations --enable-shared && \
+    make -j$(nproc) && \
+    make altinstall && \
+    cd .. && \
+    rm -rf Python-3.10.12.tgz Python-3.10.12
+
+# Set Python 3.10 as default
+RUN ln -s /usr/local/bin/python3.10 /usr/bin/python3 && \
+    ln -s /usr/local/bin/pip3.10 /usr/bin/pip3
+
+# Update library paths for Python shared library
+RUN echo "/usr/local/lib" >> /etc/ld.so.conf.d/python3.10.conf && ldconfig
+
+# Create a Python virtual environment
+RUN python3 -m venv /opt/venv
+RUN /opt/venv/bin/pip install --upgrade pip
+
+# Set environment variables for reticulate
+ENV RETICULATE_PYTHON="/opt/venv/bin/python3"
+ENV PYTHONPATH=/app#"${PYTHONPATH}:/app"
+WORKDIR /app
+
+# Set MPLCONFIGDIR to a writable directory
+ENV MPLCONFIGDIR=/app/tmp/matplotlib
+RUN mkdir -p /app/tmp/matplotlib
+
+# Add necessary files to the container
+ADD build/mpnstpdx/requirements.txt .
+ADD build/mpnstpdx/requirements.r .
+ADD build/mpnstpdx/* ./
+ADD build/utils/* ./
+
+# installing python libraries
+RUN /opt/venv/bin/pip3 install -r requirements.txt
+
+# Install all R libraries from requirements.r
+RUN Rscript requirements.r
+
+# Set up volume for temporary storage
+VOLUME ["/tmp"]

build/docker/docker-compose.yml

@@ -44,7 +44,15 @@ services:
         HTTPS_PROXY: ${HTTPS_PROXY}
     platform: linux/amd64
     image: mpnst:latest
-
+    
+  mpnstpdx:
+    build:
+      context: ../../
+      dockerfile: build/docker/Dockerfile.mpnstpdx
+      args:
+        HTTPS_PROXY: ${HTTPS_PROXY}
+    platform: linux/amd64
+    image: mpnstpdx:latest
   cptac:
     build:
       context: ../../

build/mpnst/00_sample_gen.R

@@ -7,25 +7,14 @@ library(dplyr)
 
 ##adding a command line argument
 args = commandArgs(trailingOnly=TRUE)
-if(length(args) > 1 ){
-    stop("Up to one argument is allowed. This is the filepath to the previously run samples file.")
-}
-
+if(length(args)!=2){
+    stop("Need a sample file and synapse token as argument. Rscript 00_sample_gen.R [samplefile] [synapse token]")
 
-if (length(args) == 0 || is.na(args[1]) || args[1] == "" || !file.exists(args[1])) {
-    orig_samples <- ""
-} else {
-    orig_samples <- fread(args[1])
 }
 
+orig_samples<-fread(args[1])
 
-# Check if Synapse token is available from the environment
-synapse_token <- Sys.getenv("SYNAPSE_AUTH_TOKEN")
-if (synapse_token == "") {
-    stop("Error: SYNAPSE_AUTH_TOKEN environment variable is not set.")
-}
-
-synapser::synLogin(authToken=synapse_token)
+synapser::synLogin(authToken=args[2])
 manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|>
                                                              as.data.frame()
 
@@ -43,18 +32,23 @@ manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|>
 ##first create samples for the original tumors
 tumorTable<-manifest|>
     dplyr::select(common_name='Sample')|>
-    dplyr::mutate(other_id_source='NF Data Portal',other_names='',cancer_type="Malignant peripheral nerve sheath tumor",species='Homo sapiens (Human)',model_type='tumor')|>
+    dplyr::mutate(other_id_source='NF Data Portal',other_names='',cancer_type="Malignant peripheral nerve sheath tumor",species='Human',model_type='tumor')|>
     tidyr::unite(col='other_id',c('common_name','model_type'),sep=' ',remove=FALSE)
 
 ##then create samples for the PDX
 sampTable<-manifest|>
     dplyr::select(common_name='Sample',MicroTissueDrugFolder)|>
-    dplyr::mutate(other_id_source='NF Data Portal',other_names='',cancer_type="Malignant peripheral nerve sheath tumor",species='Homo sapiens (Human)',model_type='patient derived xenograft')|>
+    dplyr::mutate(other_id_source='NF Data Portal',other_names='',cancer_type="Malignant peripheral nerve sheath tumor",species='Human',model_type='patient derived xenograft')|>
     tidyr::unite(col='other_id',c('common_name','model_type'),sep=' ',remove=FALSE)
 
 
+pdxmt<-manifest|>
+    dplyr::select(common_name='Sample',MicroTissueDrugFolder)|>
+    dplyr::mutate(other_id_source='NF Data Portal',other_names='',cancer_type="Malignant peripheral nerve sheath tumor",species='Human',model_type='organoid')|>
+    tidyr::unite(col='other_id',c('common_name','model_type'),sep=' ',remove=FALSE)
+
 ##third, generate a sample for the MTs if they were generated
-pdxmt<-subset(sampTable,!is.na(MicroTissueDrugFolder))
+#pdxmt<-subset(sampTable,!is.na(MicroTissueDrugFolder))
 pdxmt$model_type=rep('organoid',nrow(pdxmt))
 print(pdxmt)
 
@@ -64,15 +58,7 @@ main<-rbind(sampTable,pdxmt)|>
 
 #main <- fread("mpnst/NF_MPNST_samples.csv")
 #previous_aml <- fread(args[1])#"beatAML/beataml_samples.csv")
-
-# If there is no previous samples file - start at 1, else, continue where the previous one left off.
-if (identical(orig_samples, "")) {
-    max_id <- 1  
-} else {
-    max_id <- max(orig_samples$improve_sample_id, na.rm = TRUE)
-}
-
-
+max_id <- max(orig_samples$improve_sample_id)
 main$improve_sample_id <- seq(from = max_id + 1, length.out = nrow(main))
 
 #synapse_main <- fread("mpnst/synapse_NF-MPNST_samples.csv")

build/mpnst/01_mpnst_get_omics.R

@@ -34,6 +34,7 @@ samples_df <- fread(patients)|>
 
 pdx_samps<-subset(samples_df,model_type=='patient derived xenograft')
 tumor_samps<-subset(samples_df,model_type=='tumor')
+mt_samps<-subset(samples_df,model_type=='organoid')
 
 ##now get the manifest from synapse
 manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|>
@@ -45,14 +46,19 @@ manifest<-synapser::synTableQuery("select * from syn53503360")$asDataFrame()|>
 ##they each get their own sample identifier
 pdx_data<-manifest|>dplyr::select(common_name,starts_with("PDX"))|>
     left_join(pdx_samps)|>
-    dplyr::select(improve_sample_id,RNASeq='PDX_RNASeq',Mutations='PDX_Somatic_Mutations',CopyNumber='PDX_CNV',Proteomics='PDX_Proteomics')
+    dplyr::select(improve_sample_id,common_name,model_type,RNASeq='PDX_RNASeq',Mutations='PDX_Somatic_Mutations',CopyNumber='PDX_CNV',Proteomics='PDX_Proteomics')
 
 tumor_data<- manifest|>dplyr::select(common_name,starts_with("Tumor"))|>
     left_join(tumor_samps)|>
-    dplyr::select(improve_sample_id,RNASeq='Tumor_RNASeq',Mutations='Tumor_Somatic_Mutations',CopyNumber='Tumor_CNV')|>
+    dplyr::select(improve_sample_id,common_name,model_type,RNASeq='Tumor_RNASeq',Mutations='Tumor_Somatic_Mutations',CopyNumber='Tumor_CNV')|>
     mutate(Proteomics='') ##we dont have tumor proteomics from these samples
 #print(tumor_data)
 
+mt_data<- manifest|>dplyr::select(common_name,starts_with("PDX"))|>
+    left_join(mt_samps)|>
+    dplyr::select(improve_sample_id,common_name,model_type, RNASeq='PDX_RNASeq',Mutations='PDX_Somatic_Mutations',CopyNumber='PDX_CNV',Proteomics='PDX_Proteomics')##we dont have mt data yet, so collecting PDX instead
+#print(tumor_data)
+
 
 combined<-rbind(pdx_data,tumor_data)|>distinct()
 
@@ -61,10 +67,10 @@ genes_df <- fread(genefile)
 
 
 ##added proteomics first
-proteomics<-do.call('rbind',lapply(setdiff(combined$Proteomics,c('',NA,"NA")),function(x){
+proteomics<-do.call('rbind',lapply(setdiff(mt_data$Proteomics,c('',NA,"NA")),function(x){
                                         # if(x!=""){
     #print(x)
-    sample<-subset(combined,Proteomics==x)
+    sample<-subset(mt_data,Proteomics==x)
     #print(sample)
     res<-fread(synGet(x)$path)|>
         #tidyr::separate(Name,into=c('other_id','vers'),sep='\\.')|>
@@ -88,10 +94,10 @@ fwrite(proteomics,'/tmp/mpnst_proteomics.csv.gz')
 
 #### FIRST WE GET RNASeq Data
 
-rnaseq<-do.call('rbind',lapply(setdiff(combined$RNASeq,c(NA,"NA")),function(x){
+rnaseq<-do.call('rbind',lapply(setdiff(mt_data$RNASeq,c(NA,"NA")),function(x){
                                         # if(x!=""){
     #print(x)
-    sample<-subset(combined,RNASeq==x)
+    sample<-subset(mt_data,RNASeq==x)
     #print(sample)
     res<-fread(synGet(x)$path)|>
         tidyr::separate(Name,into=c('other_id','vers'),sep='\\.')|>
@@ -114,11 +120,11 @@ fwrite(rnaseq,'/tmp/mpnst_transcriptomics.csv.gz')
 
 #####NEXT WE DO WES DATA
 print("Getting WES")
-wes<-do.call(rbind,lapply(setdiff(combined$`Mutations`,c(NA,"NA")),function(x){
+wes<-do.call(rbind,lapply(setdiff(mt_data$`Mutations`,c(NA,"NA")),function(x){
 
     x2=x#gsub('"','',gsub("[",'',gsub("]",'',x,fixed=T),fixed=T),fixed=T)
     print(x)
-    sample<-subset(combined,Mutations==x)
+    sample<-subset(mt_data,Mutations==x)
     print(sample$improve_sample_id)
     res<-NULL
     try(res<-fread(synGet(x2)$path)|>
@@ -141,11 +147,11 @@ fwrite(wes,'/tmp/mpnst_mutations.csv.gz')
 
 print(paste("getting CNV"))
 ##next let's do CNVs!
-cnv<-do.call(rbind,lapply(setdiff(combined$CopyNumber,c(NA,"NA")),function(x){
+cnv<-do.call(rbind,lapply(setdiff(mt_data$CopyNumber,c(NA,"NA")),function(x){
 
     x2=x#gsub('"','',gsub("[",'',gsub("]",'',x,fixed=T),fixed=T),fixed=T)
     print(x)
-    sample<-subset(combined,CopyNumber==x)
+    sample<-subset(mt_data,CopyNumber==x)
     print(sample$improve_sample_id)
     res<-fread(synGet(x2)$path)