Ready to merge

Author: jjacobson95

build/broad_sanger/05b_separate_datasets.py

@@ -1,7 +1,8 @@
 import gc
 import polars as pl 
-
-
+import os
+import gzip
+import shutil
 
 def main():
     datasets_to_process = ["CCLE", "CTRPv2", "PRISM", "GDSCv1", "GDSCv2", "FIMM", "gCSI", "NCI60"]
@@ -23,58 +24,96 @@ def main():
     }
 
     for dataset in datasets_to_process:
-        exp = pl.read_csv("broad_sanger_experiments.tsv", separator="\t") # Keeping memory down, so I will not be making copies.
+        exp_in_filename = "broad_sanger_experiments.tsv"
+        if os.path.isfile(exp_in_filename + ".gz"):
+            exp_in_filename = exp_in_filename + ".gz"
+            
+        exp = pl.read_csv(exp_in_filename, separator="\t") # Keeping memory down, so I will not be making copies.
         exp = exp.filter(pl.col("study") == dataset)
 
         # Extract information to separate out datasets
         exp_improve_sample_ids = exp["improve_sample_id"].unique().to_list()
         exp_improve_drug_ids = exp["improve_drug_id"].unique().to_list()
 
         # Write Filtered Experiments File to TSV. Then delete it from memory.
-        exp_filename = f"/tmp/{dataset}_experiments.tsv".lower()
-        exp.write_csv(exp_filename, separator="\t")
+        exp_filename_out = f"/tmp/{dataset}_experiments.tsv".lower()
+        exp.write_csv(exp_filename_out, separator="\t")
+        #Rewrite as gzipped if needed
+        if exp_in_filename.endswith(".gz"):
+            with open(exp_filename_out, 'rb') as f_in, gzip.open(exp_filename_out + ".gz", 'wb') as f_out:
+                shutil.copyfileobj(f_in, f_out)
+            os.remove(exp_filename_out)
+            
         del exp
         gc.collect()
 
 
         #Filter Samples files, write to file, delete from mem.
         for samples in samples_datatypes:
             samples_filename_in = f"broad_sanger_{samples}.csv"
+            if os.path.isfile(samples_filename_in + ".gz"):
+                samples_filename_in += ".gz"
+            
             samples_filename_out = f"/tmp/{dataset}_{samples}.csv".lower()
             samples_df = pl.read_csv(samples_filename_in)
             samples_df = samples_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids))
             samples_df.write_csv(samples_filename_out) #csv
+            
+            #Rewrite as gzipped if needed
+            if samples_filename_in.endswith(".gz"):
+                with open(samples_filename_out, 'rb') as f_in, gzip.open(samples_filename_out + ".gz", 'wb') as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+                os.remove(samples_filename_out)
+                
             del samples_df
             gc.collect()
 
         #One by one, filter other Omics files, write to file, delete from mem.
         for omics in omics_datatypes:
             omics_filename_in = f"broad_sanger_{omics}.csv"
+            if os.path.isfile(omics_filename_in + ".gz"):
+                omics_filename_in += ".gz"
+                
             omics_filename_out = f"/tmp/{dataset}_{omics}.csv".lower()
             omics_df = pl.read_csv(omics_filename_in)
             omics_df = omics_df.filter(pl.col("improve_sample_id").is_in(exp_improve_sample_ids))
             omics_df = omics_df.filter(pl.col("source").is_in(dataset_sources[dataset]))
             omics_df.write_csv(omics_filename_out) #csv
+            
+            #Rewrite as gzipped if needed
+            if omics_filename_in.endswith(".gz"):
+                with open(omics_filename_out, 'rb') as f_in, gzip.open(omics_filename_out + ".gz", 'wb') as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+                os.remove(omics_filename_out)
+                
             del omics_df
             gc.collect()
 
 
         #One by one, filter other Drugs files, write to file, delete from mem.
         for drugs in drugs_datatypes:
             drugs_filename_in = f"broad_sanger_{drugs}.tsv"
+            if os.path.isfile(drugs_filename_in + ".gz"):
+                drugs_filename_in += ".gz"
+                
             drugs_filename_out = f"/tmp/{dataset}_{drugs}.tsv".lower()
             if drugs == "drug_descriptors":
                 drugs_df = pl.read_csv(drugs_filename_in,separator="\t",
                                        dtypes={"improve_drug_id": pl.Utf8,
                                                          "structural_descriptor": pl.Utf8,
                                                          "descriptor_value": pl.Utf8}
                                       )
-
             else:
                 drugs_df = pl.read_csv(drugs_filename_in,separator="\t")
 
             drugs_df = drugs_df.filter(pl.col("improve_drug_id").is_in(exp_improve_drug_ids))
             drugs_df.write_csv(drugs_filename_out,separator="\t") #tsv
+            
+            if drugs_filename_in.endswith(".gz"):
+                with open(drugs_filename_out, 'rb') as f_in, gzip.open(drugs_filename_out + ".gz", 'wb') as f_out:
+                    shutil.copyfileobj(f_in, f_out)
+                os.remove(drugs_filename_out)
+            
             del drugs_df
             gc.collect()
 

build/build_all.py

@@ -402,8 +402,8 @@ def get_latest_commit_hash(owner, repo, branch='main'):
     ######
     ### Begin Upload and/or validation
     #####
-    # if args.figshare or args.validate or github_token:
-    if args.figshare or args.validate:
+    if args.figshare or args.validate or github_token:
+    # if args.figshare or args.validate:
         # FigShare File Prefixes:
 
         prefixes = ['beataml', 'hcmi', 'cptac', 'mpnst', 'genes', 'drugs']
@@ -436,6 +436,13 @@ def get_latest_commit_hash(owner, repo, branch='main'):
         for file in glob(os.path.join(all_files_dir, '*.gz')):
             decompress_file(file)
 
+        ### These should be done before schema checking.
+        sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp", '--version', args.version]
+        run_docker_upload_cmd(sample_mapping_command, 'all_files_dir', 'Map_Samples', args.version)
+        
+        drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp", '--version', args.version]
+        run_docker_upload_cmd(drug_mapping_command, 'all_files_dir', 'Map_Drugs', args.version)
+
         # Run schema checker - This will always run if uploading data.
         schema_check_command = ['python3', 'scripts/check_schema.py', '--datasets'] + datasets
         run_docker_upload_cmd(schema_check_command, 'all_files_dir', 'validate', args.version)
@@ -452,40 +459,47 @@ def get_latest_commit_hash(owner, repo, branch='main'):
 
         print("File compression and decompression adjustments are complete.")
 
-        # Upload to Figshare using Docker
+        ### Upload to Figshare using Docker
         if args.figshare and args.version and figshare_token:
-            figshare_command = ['python3', 'scripts/push_to_figshare.py', '--directory', "/tmp", '--title', f"CODERData{args.version}", '--token', os.getenv('FIGSHARE_TOKEN'), '--project_id', '189342', '--publish']
+            figshare_command = ['python3', 'scripts/push_to_figshare.py', '--directory', "/tmp", '--title', f"CODERData{args.version}", '--token', os.getenv('FIGSHARE_TOKEN'), '--project_id', '189342', '--version', args.version, '--publish']
             run_docker_upload_cmd(figshare_command, 'all_files_dir', 'Figshare', args.version)
 
+            ### Push changes to GitHub using Docker
+            # if args.version and args.figshare and figshare_token and github_token and args.github_username and args.github_email:
 
-            # Push changes to GitHub using Docker
-        # if args.version and args.figshare and figshare_token and github_token and args.github_username and args.github_email:
-        if args.version and github_token and args.github_username and args.github_email:
-
-            git_command = [
-                'bash', '-c', (
-                    f'git config --global user.name "{args.github_username}" '
-                    f'&& git config --global user.email "{args.github_email}" '
-                    f'&& cp /tmp/improve_sample_mapping.json /usr/src/app/coderdata/build/improve_sample_mapping.json '
-                    f'&& cp /tmp/improve_drug_mapping.json /usr/src/app/coderdata/build/improve_drug_mapping.json '
-                    f'&& git add build/improve_sample_mapping.json '
-                    f'&& git add build/improve_drug_mapping.json '
-                    f'&& cp /tmp/figshare_latest.yml /usr/src/app/coderdata/docs/_data/figshare_latest.yml '
-                    f'&& git add docs/_data/figshare_latest.yml '
-                    f'&& git commit -m "Data Built and Uploaded. New Tag: {args.version}" '
-                    f'&& git tag {args.version} '
-                    f'&& git push https://{args.github_username}:{github_token}@github.com/PNNL-CompBio/coderdata.git main '
-                    f'&& git push https://{args.github_username}:{github_token}@github.com/PNNL-CompBio/coderdata.git --tags'
-                )
-            ]
+            # You can only upload to Github after Figshare upload is completed - otherwise figshare_latest.yml and dataset.yml won't be available.
+            if args.version and github_token and args.github_username and args.github_email:
+
+                git_command = [
+                    'bash', '-c', (
+                        f'git config --global user.name "{args.github_username}" '
+                        f'&& git config --global user.email "{args.github_email}" '
+                        
+                        # Checkout a new branch
+                        f'&& git checkout -b testing-auto-build-pr-{args.version} '
+                        
+                        # Copy and add the necessary files
+                        f'&& cp /tmp/improve_sample_mapping.json.gz /usr/src/app/coderdata/build/improve_sample_mapping.json.gz '
+                        f'&& cp /tmp/improve_drug_mapping.json.gz /usr/src/app/coderdata/build/improve_drug_mapping.json.gz '
+                        f'&& gunzip /usr/src/app/coderdata/build/*.gz '
+                        f'&& git add -f build/improve_sample_mapping.json build/improve_drug_mapping.json '
+                        f'&& cp /tmp/figshare_latest.yml /usr/src/app/coderdata/docs/_data/figshare_latest.yml '
+                        f'&& cp /tmp/dataset.yml /usr/src/app/coderdata/coderdata/dataset.yml '
+                        f'&& git add -f docs/_data/figshare_latest.yml coderdata/dataset.yml'
+                        
+                        # Tag and push
+                        f'&& git commit -m "Data Built and Uploaded. New Tag: {args.version}" '
+                        f'&& git tag {args.version} '
+                        f'&& git push https://{args.github_username}:{github_token}@github.com/PNNL-CompBio/coderdata.git testing-auto-build-pr-{args.version} '
+                        
+                        # Create a PR using GitHub CLI
+                        f'&& gh pr create --title "Testing Auto PR instead of auto Merge {args.version}" '
+                        f'--body "This PR was automatically generated by the build process." '
+                        f'--base main --head testing-auto-build-pr-{args.version}'
+                    )
+                ]
 
-            sample_mapping_command = ['python3', 'scripts/map_improve_sample_ids.py', '--local_dir', "/tmp", '--version', args.version]
-            run_docker_upload_cmd(sample_mapping_command, 'all_files_dir', 'Map_Samples', args.version)
-    
-            drug_mapping_command = ['python3', 'scripts/map_improve_drug_ids.py', '--local_dir', "/tmp", '--version', args.version]
-            run_docker_upload_cmd(drug_mapping_command, 'all_files_dir', 'Map_Drugs', args.version)
-        
-            run_docker_upload_cmd(git_command, 'all_files_dir', 'GitHub', args.version)
+                run_docker_upload_cmd(git_command, 'all_files_dir', 'GitHub', args.version)
 
 if __name__ == '__main__':
     main()

build/build_dataset.py

@@ -55,7 +55,7 @@ def process_docker(dataset,validate):
 
     datasets_to_build.extend(dataset_map.get(dataset, []))
 
-    compose_command = ['docker-compose', '-f', compose_file, 'build'] + datasets_to_build
+    compose_command = ['docker','compose', '-f', compose_file, 'build'] + datasets_to_build
 
     log_file_path = 'local/docker.log'
     env = os.environ.copy()
@@ -258,7 +258,7 @@ def run_schema_checker(dataset):
             decompress_file(os.path.join('local', all_files_dir, file))
 
     # Run schema checker
-    schema_check_command = ['python3', 'check_schema.py', '--datasets'] + datasets
+    schema_check_command = ['python3', 'scripts/check_schema.py', '--datasets'] + datasets
     run_docker_validate_cmd(schema_check_command, all_files_dir, 'Validation')
 
 def main():

build/docker/Dockerfile.upload

@@ -22,16 +22,4 @@ RUN curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | \
 
 
 RUN git clone https://github.com/PNNL-CompBio/coderdata.git
-WORKDIR /usr/src/app/coderdata
-RUN git checkout sample_id_mapping_update
-
-# COPY ./schema /usr/src/app/schema
-# ADD scripts/check_schema.py ./
-
-# #Add id mapping script
-# ADD scripts/map_improve_sample_ids.py ./
-# ADD scripts/map_improve_drug_ids.py ./
-
-# #Add improve_mapping.json file. If file is not present this won't fail.
-# ADD build/* ./
-
+WORKDIR /usr/src/app/coderdata

scripts/push_to_figshare.py

@@ -7,7 +7,7 @@
 import yaml
 
 
-def upload_to_figshare(token, title, directory, project_id, publish, article_id=None):
+def upload_to_figshare(token, title, directory, project_id, publish, version, article_id=None):
     """
     Uploads a file to Figshare and publishes the article.
 
@@ -187,34 +187,41 @@ def delete_existing_file(article_id, file_id):
         issue_request('DELETE', f'account/articles/{article_id}/files/{file_id}')
 
 
-    def write_figshare_details_to_yaml(article_id, project_id, title):
+    def write_figshare_details_to_yaml(article_id, project_id, title, version):
         """
-        Write details of Figshare to yaml
+        Write details of Figshare to two yaml files.  
+        figshare_latest.yml will be updated for the docs.
+        dataset.yml will be updated for the python package.
         """
-        #convert slashes and periods to underscores so the file links are generated correctly.
+                
+        # update dataset.yml
+        with open("coderdata/dataset.yml", "r") as f:
+            data = yaml.safe_load(f)
+        data["figshare"] = f"https://api.figshare.com/v2/articles/{article_id}"
+        data["version"] = version
+        with open("tmp/dataset.yml", "w") as f:
+            yaml.safe_dump(data, f, sort_keys=False)       
+             
+        
+        # write figshare_latest.yml
         title_updated = title.replace('/', '_')
         title_updated = title_updated.replace('.', '_')
         article_info = issue_request('GET', f'articles/{article_id}')
-        # article_link = f"https://figshare.com/articles/dataset/{title}/{project_id}/file/{article_id}"
         article_link = f"https://figshare.com/articles/dataset/{title_updated}/{article_id}"
 
         # Retrieve the article details
         article_details_response = requests.get(article_info['url'])
         article_details_response.raise_for_status()
         article_details = article_details_response.json()
-
-        # Construct the URLs
-        file_url_links = {file['name']:f"https://figshare.com/articles/dataset/{title_updated}/{article_id}?file={file['id']}" for file in article_details['files']}
-        file_download_link = {file['name']: file['download_url'] for file in article_details['files']}
+        
         yaml_data = {
             'article_link': article_link,
-            'file_url': file_url_links,
-            'file_download': file_download_link
+            'version': version,
         }
 
         with open('/tmp/figshare_latest.yml', 'w') as file:
             yaml.dump(yaml_data, file, default_flow_style=False)
-
+            
 
     article_id = create_or_get_article(title, project_id, article_id)
     all_files_uploaded = True
@@ -249,7 +256,7 @@ def write_figshare_details_to_yaml(article_id, project_id, title):
         print("Files uploaded successfully but not published.")
 
     if all_files_uploaded:
-        write_figshare_details_to_yaml(article_id, project_id,title)
+        write_figshare_details_to_yaml(article_id, project_id,title, version)
 
 def main():
     parser = argparse.ArgumentParser(description='Upload files to Figshare.')
@@ -259,9 +266,10 @@ def main():
     parser.add_argument('-p', '--publish', help='Publish the article', action='store_true')
     parser.add_argument('-j', '--project_id', help='Existing Figshare project ID', required=True)
     parser.add_argument('-a', '--article_id', help='Existing Figshare article ID', required=False, default=None)
+    parser.add_argument('-v', '--version', help='Latest Version', required=True)
     args = parser.parse_args()
 
-    upload_to_figshare(args.token, args.title, args.directory, args.project_id, args.publish, args.article_id)
+    upload_to_figshare(args.token, args.title, args.directory, args.project_id, args.publish, args.version, args.article_id)
 
 if __name__ == "__main__":
     main()