Merge branch 'main' of github.com:PNNL-CompBio/coderdata

Author: sgosline

LICENSE_DISCLAIMER DISCLAIMERLICENSE_DISCLAIMER renamed to DISCLAIMER

@@ -1,27 +1,3 @@
-Copyright Battelle Memorial Institute 2025
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-1. Redistributions of source code must retain the above copyright notice, this
-list of conditions and the following disclaimer.
-
-2. Redistributions in binary form must reproduce the above copyright notice,
-this list of conditions and the following disclaimer in the documentation
-and/or other materials provided with the distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
-FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
-SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
-CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
-OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-------
 
 This material was prepared as an account of work sponsored by an agency of the
 United States Government.  Neither the United States Government nor the United

LICENSE

@@ -0,0 +1,23 @@
+Copyright Battelle Memorial Institute 2025
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation
+and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

build/beatAML/GetBeatAML.py

@@ -424,6 +424,9 @@ def map_and_combine(df, data_type, entrez_map_file, improve_map_file, map_file=N
         mapped_df.rename(columns={"hgvsc": "mutation"}, inplace=True)
         mapped_df.rename(columns={"labId": "sample_id"}, inplace=True)
         mapped_df.rename(columns={"Entrez_Gene_Id": "entrez_id"}, inplace=True)
+        
+        #remove (gene) information preceeding the colon - this formats it like other datasets.
+        mapped_df["mutation"] = mapped_df["mutation"].astype(str).str.split(":", n=1).str[-1]
 
         variant_mapping = {
             'frameshift_variant': 'Frameshift_Variant',

build/bladderpdo/00_createBladderPDOSampleFile.py

@@ -1,12 +1,49 @@
+#!/usr/bin/env python3
 import synapseclient
 import pandas as pd
 import numpy as np
 import argparse
 import os
+import re
+import subprocess
 
+# Helper functions
+def _clean_geo_id(s):
+    """
+    Normalise GEO sample IDs so they match Synapse naming.
+      • 11.2  → 11_2
+      • **_Tumor → *_Parental
+      • *_orgP2 → *_Organoid_P2
+      • *_xenoorgP4 → *_XenoOrganoid_P4
+    """
+    s = s.strip()
+    s = re.sub(r"(?<=\d)\.(?=\d)", "_", s)          # dots between digits
+    s = s.replace("_tumor", "_Parental")            # tumour alias
+    # lower-case 'orgP' / 'xenoorgP' fix
+    s = re.sub(r"_(org)P(\d+)",      r"_Organoid_P\2",      s, flags=re.IGNORECASE)
+    s = re.sub(r"_(xenoorg)P(\d+)",  r"_XenoOrganoid_P\2",  s, flags=re.IGNORECASE)
+    return s
 
+
+def _parse_model_type(sample_id):
+    """Derive model_type from Sample ID."""
+    low = sample_id.lower()
+    if "_xenoorganoid" in low:
+        return "xenograft derived organoid"
+    if "_organoid" in low:
+        return "organoid"
+    if "_xenograft" in low:
+        return "patient derived xenograft"
+    if "_parental" in low:
+        return "tumor"
+    return "unknown"
+
+#Generate Samples Data
 def get_bladder_pdo_samples(synLoginObject, maxval):
 
+    
+    #Part 1: Get Data from Synapse
+    
     # download from Synapse..
     samples_syn = synLoginObject.get('syn64765486')
     # and read the file
@@ -19,7 +56,43 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
     samples.loc[:,['other_id_source']] = 'Synapse'
     samples.loc[:,['other_names'] ]= ''
     samples.loc[:,['cancer_type']]=samples['cancer_type'].str.lower()
-    samples.loc[:, ['model_type']] = samples['model_type'].str.lower()
+    samples["model_type"] = samples["other_id"].apply(_parse_model_type)
+
+    #Part 2: Get Data from Geo
+    subprocess.call (["Rscript", "--vanilla", "obtainGSMidLink.R"])
+    GEO_ids_link = "./gsmlinkDf.csv"
+
+    geo_map  = pd.read_csv(GEO_ids_link)
+    geo_ids  = geo_map["sampleid"].dropna().map(_clean_geo_id).unique()
+    missing  = sorted(set(geo_ids) - set(samples["other_id"]))
+
+    if missing:                    
+        print(f"Adding {len(missing)} GEO samples not in Synapse sheet")
+
+    rows = []                      
+    for oid in missing:            
+        common = oid.split("_")[0]
+        ctype  = (
+            samples.loc[samples["common_name"] == common, "cancer_type"]
+            .iloc[0]
+            if (samples["common_name"] == common).any()
+            else "bladder urothelial carcinoma"
+        )
+        rows.append(
+            {
+                "other_id":        oid,
+                "common_name":     common,
+                "cancer_type":     ctype,
+                "model_type":      _parse_model_type(oid),
+                "species":         "Homo sapiens(Human)",
+                "other_id_source": "GEO",
+                "other_names":     "",
+            }
+        )
+    if rows:                       
+        samples = pd.concat([samples, pd.DataFrame(rows)], ignore_index=True)
+
+    samples = samples.sort_values("other_id").reset_index(drop=True)
 
     samples['improve_sample_id'] = range(maxval+1, maxval+1+samples.shape[0])
 
@@ -29,11 +102,8 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser(description="This script handles downloading, processing and formatting of sample files for the Sarcoma PDO project into a single samplesheet")
-    
     parser.add_argument('-t', '--token', type=str, help='Synapse Token')
-
     parser.add_argument("-p", '--prevSamples', nargs="?", type=str, default ="", const  = "", help = "Use this to provide previous sample file, will run sample file generation")
-
     args = parser.parse_args()
 
     print("Logging into Synapse")
@@ -46,5 +116,4 @@ def get_bladder_pdo_samples(synLoginObject, maxval):
         prev_max_improve_id = 0
 
     bladder_pdo_samples = get_bladder_pdo_samples(synObject, prev_max_improve_id)
-
     bladder_pdo_samples.to_csv("/tmp/bladderpdo_samples.csv", index=False)

build/bladderpdo/01_createBladderPDOOmicsFiles.py

@@ -7,6 +7,7 @@
 import gzip
 import subprocess
 import math
+import re
 
 def get_copy_call(a):
     """
@@ -31,7 +32,21 @@ def get_copy_call(a):
     else:
         return 'amp'
 
-    return pd.Series([get_copy_call(a) for a in arr])
+
+def normalise_id(s):
+    """
+    Make GEO sample IDs line up with 'other_id' in bladderpdo_samples.csv.
+    """
+    if pd.isna(s):
+        return s
+    s = s.strip()
+    s = re.sub(r"(?<=\d)\.(?=\d)", "_", s)                 # dots → underscore
+    s = s.replace("_tumor", "_Parental")                   # tumour alias
+    s = re.sub(r"_(org)P(\d+)",      r"_Organoid_P\2",     s, flags=re.IGNORECASE)
+    s = re.sub(r"_(xenoorg)P(\d+)", r"_XenoOrganoid_P\2",  s, flags=re.IGNORECASE)
+    return s
+
+
 
 def get_bladder_pdo_transcriptomics(GEO_id_link_table, samples, genes):
 
@@ -40,30 +55,42 @@ def get_bladder_pdo_transcriptomics(GEO_id_link_table, samples, genes):
     transcriptomics = pd.read_csv(transcriptomic_txt, compression='gzip', sep="\t")
     subprocess.call (["/usr/bin/Rscript", "--vanilla", "obtainGSMidLink.R"])
 
-    GEO_ids_link = pd.read_csv("./gsmlinkDf.csv")
+    GEO_ids = pd.read_csv(GEO_id_link_table)
+    print(GEO_ids)
     fpkm_totals = transcriptomics.iloc[:, 1:43].sum()
     transcriptomics.iloc[:, 1:43] = transcriptomics.iloc[:, 1:43].div(fpkm_totals).mul(1e6)
     transcriptomics['ensembl'] = transcriptomics['Unnamed: 0'].str.split("_", expand=True)[0]
     mapped_df = transcriptomics.merge(genes[['entrez_id', 'other_id']].drop_duplicates(), left_on='ensembl', right_on='other_id', how='left')
     # transform data to long format
+    print(mapped_df)
 
-    mapped_df.drop('other_id', axis=1)
+    mapped_df = mapped_df.drop('other_id', axis=1)
     value_variables = transcriptomics.columns[transcriptomics.columns.str.contains("M")]
     melted_txomics = mapped_df.melt(id_vars = "entrez_id", value_vars = value_variables, var_name='sample_name')
     # use info from GEO to get Sample IDS
-    txomics_with_GEOid = melted_txomics.merge(GEO_ids_link, how = 'left', left_on = "sample_name", right_on='RNAid')
+    m1 = melted_txomics.merge(GEO_ids, how="left", left_on="sample_name", right_on="RNAid")
+    m1["sampleid"] = m1["sampleid"].apply(normalise_id)
+    print(m1)
+    print(m1.sampleid.unique())
     # use samplesheet to link sample_ids to improve ids
-    txomics_with_GEOid['sampleid'] = txomics_with_GEOid['sampleid'].str.replace("org", "Organoid_")
-    txomics_with_GEOid['sampleid'] = txomics_with_GEOid['sampleid'].str.replace("tumor", "Tumor")
-    txomics_with_improveid = txomics_with_GEOid.merge(samples, left_on="sampleid", right_on="other_id", how="left")
-    final_transcriptomics = txomics_with_improveid[['entrez_id', 'value', 'improve_sample_id']]
-    final_transcriptomics['source'] = "Gene Expression Omnibus"
-    final_transcriptomics['study'] = "Lee etal 2018 Bladder PDOs"
-    final_transcriptomics.rename({'value' : 'transcriptomics' })
-    # remove duplicates
-    toreturn = final_transcriptomics.drop_duplicates()
-
-    return toreturn
+    tx_with_ids = m1.merge(
+        samples, left_on="sampleid", right_on="other_id", how="left"
+    )
+    print(tx_with_ids)
+
+    final_tx = (
+        tx_with_ids[["entrez_id", "value", "improve_sample_id"]]
+        .drop_duplicates()
+        .assign(source="Gene Expression Omnibus",
+                study="Lee et al. 2018 Bladder PDOs")
+    )
+    final_tx.rename(columns= {"value":"transcriptomics"},inplace=True)
+    final_tx = final_tx.drop_duplicates()
+    final_tx = final_tx.dropna(subset=["entrez_id"])
+    final_tx["improve_sample_id"] = final_tx["improve_sample_id"].astype(int)
+    final_tx["entrez_id"]         = final_tx["entrez_id"].astype(int)
+
+    return final_tx
 
 def get_bladder_pdo_mutations(synObject, samples, genes):
     print(samples.head)
@@ -74,10 +101,11 @@ def get_bladder_pdo_mutations(synObject, samples, genes):
     selectioncols_mutations = mutations_df[['Entrez_Gene_Id',"Variant_Classification", "Tumor_Sample_Barcode", "mutation"]]
     merged_mutations = selectioncols_mutations.merge(samples, left_on="Tumor_Sample_Barcode", right_on="other_id", how="left")
     merged_mutations_renamed = merged_mutations.rename({"Entrez_Gene_Id" : 'entrez_id', 'Variant_Classification' : "variant_classification"}, axis=1)
-    print(merged_mutations_renamed.head)
     final_mutations = merged_mutations_renamed[['entrez_id', "mutation", "variant_classification", "improve_sample_id"]]
     final_mutations['study'] = "Lee etal 2018 Bladder PDOs"
-    print(final_mutations.head)
+    final_mutations = final_mutations.dropna(subset=["entrez_id"])
+    final_mutations["improve_sample_id"] = final_mutations["improve_sample_id"].astype(int)
+    final_mutations["entrez_id"]         = final_mutations["entrez_id"].astype(int)
     return final_mutations
 
 def get_bladder_pdo_copynumber(synObject, samples, genes):
@@ -94,7 +122,9 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
     final_copynumber = copynumber_with_correct_colnames[['entrez_id', 'improve_sample_id', 'copy_number', 'copy_call']]
     final_copynumber['source'] = "Synapse"
     final_copynumber['study'] = "Lee etal 2018 Bladder PDOs"
-
+    final_copynumber = final_copynumber.dropna(subset=["entrez_id"])
+    final_copynumber["improve_sample_id"] = final_copynumber["improve_sample_id"].astype(int)
+    final_copynumber["entrez_id"]         = final_copynumber["entrez_id"].astype(int)
     return final_copynumber
 
 
@@ -108,7 +138,7 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
     parser.add_argument('-c', '--copy', help='Flag to capture copy number data', action='store_true', default=False)
     parser.add_argument('-m', '--mutation', help='Flag to capture mutation data', action='store_true', default=False)
     parser.add_argument('-e', '--expression', help='Flag to capture transcriptomic data', action='store_true', default=False)
-    parser.add_argument('-i', '--geolink', help=".csv file that is the output of 'CNV-segfile-anotation.R")
+    parser.add_argument('-i', '--geolink', default = "./gsmlinkDf.csv", help=".csv file that is the output of 'CNV-segfile-anotation.R")
     parser.add_argument('-t', '--token', help='Synapse token')
 
     args = parser.parse_args()
@@ -129,4 +159,4 @@ def get_bladder_pdo_copynumber(synObject, samples, genes):
         get_bladder_pdo_mutations(synObject, samples, genes).to_csv('/tmp/bladderpdo_mutations.csv', index=False)
 
     if args.copy:
-        get_bladder_pdo_copynumber(synObject, samples, genes).to_csv("/tmp/bladderpdo_copynumber.csv", index=False)
+        get_bladder_pdo_copynumber(synObject, samples, genes).to_csv("/tmp/bladderpdo_copy_number.csv", index=False)

build/bladderpdo/README.md

@@ -0,0 +1,49 @@
+# Bladder Patient Derived Organoid Dataset
+
+This dataset comes from a [2018 publication](https://www.sciencedirect.com/science/article/pii/S0092867418302976?via%3Dihub) (Lee et. al., 2018) that created several bladder PDO lines from xenografts in mice and performed sequencing and drug experiments. 
+
+## Samples
+
+Sample information can be found on Synapse [here](https://www.synapse.org/Synapse:syn64765486).
+
+## Omics Data
+
+This study had copy number variation, RNA-seq and mutation data that was included. 
+
+Copy number data is available at [this synapse link](https://www.synapse.org/Synapse:syn64765499). Mutation data is available at [this synapse link](https://www.synapse.org/Synapse:syn64765525). RNA-Seq data was acquired from GEO, we used the 'GSE1039990_Normalized_counts.txt.gz' table available on GEO [GSE103990](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE103990). 
+
+## Drug Dose-Response Experiments
+
+For these data, we had complete drug dose-response data to recalculate curves, available on Synapse in the [Lee Bladder PDO Datasets](https://www.synapse.org/Synapse:syn64765430) folder. 
+
+## To Run Code 
+
+### First sample and omics steps are the same, by hand locally or in full build process
+```
+python3 00_createBladderPDOSampleFile.py --token $SYNAPSE_AUTH_TOKEN -p prevSamples
+
+# for mutation data (-m)
+python3 01_createBladderPDOOmicsFiles.py --token $SYNAPSE_AUTH_TOKEN -s curSamples -g genes.csv -m
+# for expressiondata (-e)
+python3 01_createBladderPDOOmicsFiles.py --token $SYNAPSE_AUTH_TOKEN -s curSamples -g genes.csv -e
+# for copynumber (-c)
+python3 01_createBladderPDOOmicsFiles.py --token $SYNAPSE_AUTH_TOKEN -s curSamples -g genes.csv -c
+```
+
+### For drug and experiment steps, command depends on location of helper scripts
+```
+# for running locally (from build directory):
+python3 -m bladderpdo.02_createBladderPDODrugsFile --token $SYNAPSE_AUTH_TOKEN -d prevDrugFilePath -o ./bladderpdo/bladderpdo_drugs.tsv
+# for running in Docker as part of full build
+python3 02_createBladderPDODrugsFile.py --token $SYNAPSE_AUTH_TOKEN -d prevDrugFilePath -o /tmp/bladderpdo_drugs.tsv
+
+# for running locally (from build directory): 
+python3 utils/build_drug_desc.py --drugtable ./bladderpdo/bladderpdo_drugs.tsv --desctable ./bladderpdo/bladderpdo_drug_descriptors.tsv.gz
+# for running in docker as part of full build
+python3 build_drug_desc.py --drugtable /tmp/bladderpdo_drugs.tsv --desctable /tmp/bladderpdo_drug_descriptors.tsv.gz
+
+
+python3 03_createBladderPDOExperimentFile.py --token $SYNAPSE_AUTH_TOKEN --drugfile curDrugFile --curSampleFile curSampleFile --output /tmp/bladderpdo_doserep.tsv
+
+python3 fit_curve.py --input /tmp/bladderpdo_doserep.tsv --output /tmp/bladderpdo_experiments.tsv
+```

build/bladderpdo/build_exp.sh

@@ -6,4 +6,6 @@ trap 'echo "Error on or near line $LINENO while executing: $BASH_COMMAND"; exit
 echo "Running 04-drug_dosage_and_curves.py with drugfile $2 and curSampleFile $1"
 python3 03_createBladderPDOExperimentFile.py --token $SYNAPSE_AUTH_TOKEN --drugfile $2 --curSampleFile $1 --output /tmp/bladderpdo_doserep.tsv
 
-python3 fit_curve.py --input /tmp/bladderpdo_doserep.tsv --output /tmp/bladderpdo_experiments.tsv
+python3 fit_curve.py --input /tmp/bladderpdo_doserep.tsv --output /tmp/bladderpdo_experiments.tsv
+rm /tmp/bladderpdo_doserep.tsv
+mv /tmp/bladderpdo_experiments.tsv.0 /tmp/bladderpdo_experiments.tsv

build/docker/Dockerfile.upload

@@ -11,15 +11,18 @@ RUN apt-get update && \
     apt-get install -y git curl gnupg && \
     rm -rf /var/lib/apt/lists/*
 
-# Install GitHub CLI (gh)
-RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \
-    gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg && \
-    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | \
-    tee /etc/apt/sources.list.d/github-cli.list > /dev/null && \
-    apt-get update && \
-    apt-get install -y gh && \
-    rm -rf /var/lib/apt/lists/*
+# # Install GitHub CLI (gh)
+# RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \
+#     gpg --dearmor -o /usr/share/keyrings/githubcli-archive-keyring.gpg && \
+#     echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | \
+#     tee /etc/apt/sources.list.d/github-cli.list > /dev/null && \
+#     apt-get update && \
+#     apt-get install -y gh && \
+#     rm -rf /var/lib/apt/lists/*
+
 
+# RUN git clone https://github.com/PNNL-CompBio/coderdata.git
+# WORKDIR /usr/src/app/coderdata
 
-RUN git clone https://github.com/PNNL-CompBio/coderdata.git
-WORKDIR /usr/src/app/coderdata
+ADD schema schema
+ADD scripts scripts