Update for NCI60, bug fixes, and HCMI GDC tool

Author: jjacobson95

build/broad_sanger/04b-nci60-updated.py

@@ -107,10 +107,11 @@ def main():
 
     finaldf = pl.DataFrame(
         {
-            'source':['NCI60' for a in molar['improve_drug_id']], ##2024 build
+            'source':['NCI60_24' for a in molar['improve_drug_id']], ##2024 build
             'improve_sample_id':molar['improve_sample_id'],
             'Drug':molar['improve_drug_id'],
-            'study': molar['EXPID'],#['NCI60' for a in nonulls['improve_drug_id']],
+            # 'study': molar['EXPID'],#['NCI60' for a in nonulls['improve_drug_id']],
+            'study': "NCI60",
             'time':molar['time'],
             'time_unit':molar['time_unit'],
             'DOSE': [(10**a)*1000000 for a in molar['CONCENTRATION']], ##move from molar to uM to match pharmacoDB

build/build_dataset.py

@@ -218,7 +218,7 @@ def run_docker_validate_cmd(cmd_arr, all_files_dir, name):
     Wrapper for 'docker run' command used during validation and uploads.
     '''
     env = os.environ.copy()
-    docker_run = ['docker', 'run', '-v', f"{env['PWD']}/local/{all_files_dir}:/tmp"]
+    docker_run = ['docker', 'run', '-v', f"{env['PWD']}/local/{all_files_dir}:/tmp", '--platform=linux/amd64']
     docker_run.extend(['upload']) 
     docker_run.extend(cmd_arr)
     print('Executing:', ' '.join(docker_run))
@@ -258,7 +258,7 @@ def run_schema_checker(dataset):
             decompress_file(os.path.join('local', all_files_dir, file))
 
     # Run schema checker
-    schema_check_command = ['python3', 'scripts/check_schema.py', '--datasets'] + datasets
+    schema_check_command = ['python3', 'check_schema.py', '--datasets'] + datasets
     run_docker_validate_cmd(schema_check_command, all_files_dir, 'Validation')
 
 def main():

build/hcmi/02-getHCMIData.py

@@ -17,79 +17,93 @@
 import polars as pl
 import gc
 import hashlib
+from pathlib import Path
 
 def download_tool(url):
     """
-    Download, extract, and make a tool (GDC Client) executable from the provided URL.
+    Download, extract, and prepare the GDC client tool.
 
     Parameters
     ----------
     url : str
-        The URL from where the tool needs to be downloaded.
+        The URL to download the tool from.
 
     Returns
     -------
     str
-        Name of the downloaded file.
+        The path to the `gdc-client` executable.
     """
-    
+    # Download the file
+    print("Downloading tool...")
     filename = wget.download(url)
-    files_before = os.listdir()
-    # shutil.unpack_archive(filename)
-    
-    #This is just set for AWS to debug. This will have to be mapped to OS.  They changed their file structure. This should be updated.
-    shutil.unpack_archive("gdc-client_2.3_Ubuntu_x64.zip") 
-    if not os.path.exists('gdc-client'):
-        raise FileNotFoundError("gdc-client executable not found after extraction.")
-    # Ensure 'gdc-client' is executable
-    st = os.stat('gdc-client')
-    os.chmod('gdc-client', st.st_mode | stat.S_IEXEC)
-    # Return the path to the executable
-    return './gdc-client'
-
-    # files_after = os.listdir()
-    # new_file = str(next(iter((set(files_after) - set(files_before)))))
-    # st = os.stat(new_file)
-    # os.chmod(new_file, st.st_mode | stat.S_IEXEC)
-    # return filename
+    
+    # First extraction
+    print(f"\nExtracting {filename}...")
+    shutil.unpack_archive(filename)
+    os.remove(filename)
+
+    # Check for a nested zip file and extract again
+    extracted_files = [f for f in os.listdir() if os.path.isfile(f) and f.endswith(".zip")]
+    for zip_file in extracted_files:
+        print(f"Extracting nested archive: {zip_file}...")
+        shutil.unpack_archive(zip_file)
+        os.remove(zip_file)
+
+    gdc_client_path = None
+    for root, dirs, files in os.walk("."):
+        if "gdc-client" in files:
+            gdc_client_path = os.path.join(root, "gdc-client")
+            break
+
+    if not gdc_client_path:
+        raise FileNotFoundError("`gdc-client` executable not found after extraction.")
+
+    # Ensure `gdc-client` is executable
+    print(f"Making {gdc_client_path} executable...")
+    st = os.stat(gdc_client_path)
+    os.chmod(gdc_client_path, st.st_mode | stat.S_IEXEC)
+
+    return gdc_client_path
 
 def is_tool(name):
     """
-    Check if a specific tool is available on the system or in the current directory.
+    Check if a specific tool is available on the system.
 
     Parameters
     ----------
     name : str
-        The name of the tool to check.
+        The name of the tool.
 
     Returns
     -------
     bool
         True if the tool is found, otherwise False.
     """
-    
-    return which(name) is not None or name in os.listdir()
+    return shutil.which(name) is not None or name in os.listdir()
 
 def ensure_gdc_client():
     """
-    Ensure that the gdc-client is available on the system.
+    Ensure that the GDC client tool is available on the system.
     
-    If the gdc-client tool isn't found, this function will automatically
-    download the appropriate version based on the operating system.
+    If the tool isn't found, this function downloads and prepares it.
     """
-    
     tool_name = "gdc-client"
     if not is_tool(tool_name):
-        print("Downloading gdc-client")
+        print("GDC client not found. Downloading...")
         urls = {
-            "Darwin": 'https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_OSX_x64-py3.8-macos-14.zip',
-            "Windows": 'https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_Windows_x64-py3.8-windows-2019.zip',
-            "Linux": 'https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_Ubuntu_x64-py3.8-ubuntu-20.04.zip'
+            "Darwin": "https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_OSX_x64-py3.8-macos-14.zip",
+            "Windows": "https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_Windows_x64-py3.8-windows-2019.zip",
+            "Linux": "https://gdc.cancer.gov/system/files/public/file/gdc-client_2.3_Ubuntu_x64-py3.8-ubuntu-20.04.zip"
         }
-        
-        download_tool(urls.get(platform.system()))
+        os_type = platform.system()
+        url = urls.get(os_type)
+        if not url:
+            raise ValueError(f"Unsupported OS: {os_type}")
+        gdc_client_path = download_tool(url)
+        print(f"`gdc-client` downloaded and available at {gdc_client_path}")
     else:
-        print("gdc-client already installed")
+        print("`gdc-client` is already installed.")
+
 
 def extract_uuids_from_manifest(manifest_data):
     """

schema/expected_files.yaml

@@ -39,7 +39,7 @@ datasets:
     - target_class: Drug
       file: /tmp/mpnst_drugs.tsv
 
-  mpnst:
+  mpnstpdx:
     - target_class: Sample
       file: /tmp/mpnstpdx_samples.csv
     - target_class: Transcriptomics