diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 1aee2a47..775406e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -8,15 +8,12 @@ on: type: choice options: - terra-base - - terra-jupyter-aou - terra-jupyter-base - terra-jupyter-bioconductor - terra-jupyter-gatk - terra-jupyter-hail - terra-jupyter-python - terra-jupyter-r - - terra-rstudio-aou - - wondershaper default: terra-rstudio-rstudio branch: description: 'Branch to run the workflow on' @@ -25,6 +22,9 @@ on: jobs: publish-job: runs-on: self-hosted + env: + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 steps: - name: Checkout current code uses: actions/checkout@v3 @@ -40,19 +40,26 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Explicitly auth Docker for GCR run: gcloud auth configure-docker --quiet + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and publish run: | gcloud auth configure-docker diff --git a/.github/workflows/publish_all.yml b/.github/workflows/publish_all.yml index 8d71f067..47c99025 100644 --- a/.github/workflows/publish_all.yml +++ b/.github/workflows/publish_all.yml @@ -13,6 +13,9 @@ on: jobs: publish-job: runs-on: self-hosted + env: + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 steps: - name: Checkout current code uses: actions/checkout@v3 @@ -23,19 +26,26 @@ jobs: python-version: '3.10' - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Explicitly auth Docker for GCR run: gcloud auth configure-docker --quiet + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and publish run: | gcloud auth configure-docker diff --git a/.github/workflows/test-terra-jupyter-aou.yml b/.github/workflows/test-terra-jupyter-aou.yml deleted file mode 100644 index f64e24f1..00000000 --- a/.github/workflows/test-terra-jupyter-aou.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Test terra-jupyter-aou -# Perform smoke tests on the terra-jupyter-aou Docker image to have some amount of confidence that -# Python package versions are compatible. -# -# To configure the minimal auth needed for these tests to be able to read public data from Google Cloud Platform: -# Step 1: Create a service account per these instructions: -# https://github.com/google-github-actions/setup-gcloud/blob/master/setup-gcloud/README.md -# Step 2: Give the service account the following permissions within the project: BigQuery User -# Step 3: Store its key and project id as GitHub repository secrets TD_GCP_SA_KEY and GCP_PROJECT_ID. -# https://docs.github.com/en/free-pro-team@latest/actions/reference/encrypted-secrets#creating-encrypted-secrets-for-a-repository - -on: - pull_request: - branches: [ master ] - paths: - - 'terra-jupyter-aou/**' - - '.github/workflows/test-terra-jupyter-aou.yml' - - push: - # Note: GitHub secrets are not passed to pull requests from forks. For community contributions from - # regular contributors, its a good idea for the contributor to configure the GitHub actions to run correctly - # in their fork as described above. - # - # For occasional contributors, the dev team will merge the PR fork branch to a branch in upstream named - # test-community-contribution- to run all the GitHub Action smoke tests. - branches: [ 'test-community-contribution*' ] - paths: - - 'terra-jupyter-aou/**' - - '.github/workflows/test-terra-jupyter-aou.yml' - - workflow_dispatch: - # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. - # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. - -env: - GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} - -jobs: - - test_docker_image: - runs-on: self-hosted - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Free up some disk space - run: sudo rm -rf /usr/share/dotnet - - - id: auth - uses: google-github-actions/auth@v2 - with: - credentials_json: ${{ secrets.TD_GCP_SA_KEY }} - create_credentials_file: true - - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 - with: - project_id: ${{ secrets.GCP_PROJECT_ID }} - - - name: Build Docker image and base images too, if needed - run: | - gcloud auth configure-docker - ./build_smoke_test_image.sh terra-jupyter-aou - - - name: Run Python code specific to notebooks with nbconvert - # Run all notebooks from start to finish, regardles of error, so that we can capture the - # result as a workflow artifact. - # See also https://github.com/marketplace/actions/run-notebook if a more complicated - # workflow for notebooks is needed in the future. - run: | - chmod a+w -R $GITHUB_WORKSPACE - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-aou:smoke-test \ - /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-aou/tests}/*ipynb ; do jupyter nbconvert --to html_embed --ExecutePreprocessor.allow_errors=True --execute "${nb}" ; done' - - - name: Upload workflow artifacts - uses: actions/upload-artifact@v4 - with: - name: notebook-execution-results - path: | - terra-jupyter-python/tests/*.html - terra-jupyter-aou/tests/*.html - retention-days: 30 - - - name: Test Python code with pytest - run: | - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-aou:smoke-test \ - /bin/bash -c 'pip3 install pytest ; pytest terra-jupyter-python/tests/ terra-jupyter-aou/tests/' - - - name: Test Python code specific to notebooks with nbconvert - # Simply 'Cell -> Run All` these notebooks and expect no errors in the case of a successful run of the test suite. - # If the tests throw any exceptions, execution of the notebooks will halt at that point. Look at the workflow - # artifacts to understand if there are more failures than just the one that caused this task to halt. - run: | - chmod a+r "${{ steps.auth.outputs.credentials_file_path }}" - docker run \ - --env GOOGLE_PROJECT \ - --volume "${{ steps.auth.outputs.credentials_file_path }}":/tmp/credentials.json:ro \ - --env GOOGLE_APPLICATION_CREDENTIALS="/tmp/credentials.json" \ - --volume $GITHUB_WORKSPACE:/tests \ - --workdir=/tests \ - --entrypoint="" \ - terra-jupyter-aou:smoke-test \ - /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-aou/tests}/*ipynb ; do jupyter nbconvert --to html_embed --execute "${nb}" ; done' diff --git a/.github/workflows/test-terra-jupyter-base.yml b/.github/workflows/test-terra-jupyter-base.yml index 295afc15..6afe3760 100644 --- a/.github/workflows/test-terra-jupyter-base.yml +++ b/.github/workflows/test-terra-jupyter-base.yml @@ -28,12 +28,18 @@ on: - 'terra-jupyter-base/**' - '.github/workflows/test-terra-jupyter-base.yml' + schedule: + # Nightly rebuild to catch rot, 30 minute cascade starting 2a ET + - cron: '0 6 * * *' + workflow_dispatch: # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. env: GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 jobs: @@ -53,16 +59,23 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and base images too, if needed run: | gcloud auth configure-docker diff --git a/.github/workflows/test-terra-jupyter-bioconductor.yml b/.github/workflows/test-terra-jupyter-bioconductor.yml index 669e6df1..5e8833f1 100644 --- a/.github/workflows/test-terra-jupyter-bioconductor.yml +++ b/.github/workflows/test-terra-jupyter-bioconductor.yml @@ -28,12 +28,17 @@ on: - 'terra-jupyter-bioconductor/**' - '.github/workflows/test-terra-jupyter-bioconductor.yml' + schedule: + - cron: '30 6 * * *' + workflow_dispatch: # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. env: GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 jobs: @@ -53,16 +58,23 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and base images too, if needed run: | gcloud auth configure-docker diff --git a/.github/workflows/test-terra-jupyter-gatk.yml b/.github/workflows/test-terra-jupyter-gatk.yml index ebb10bf8..7b55effe 100644 --- a/.github/workflows/test-terra-jupyter-gatk.yml +++ b/.github/workflows/test-terra-jupyter-gatk.yml @@ -28,12 +28,17 @@ on: - 'terra-jupyter-gatk/**' - '.github/workflows/test-terra-jupyter-gatk.yml' + schedule: + - cron: '0 7 * * *' + workflow_dispatch: # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. env: GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 jobs: @@ -53,16 +58,23 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and base images too, if needed run: | gcloud auth configure-docker @@ -123,4 +135,3 @@ jobs: --entrypoint="" \ terra-jupyter-gatk:smoke-test \ /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-gatk/tests}/*ipynb ; do jupyter nbconvert --to html_embed --execute "${nb}" ; done' - diff --git a/.github/workflows/test-terra-jupyter-hail.yml b/.github/workflows/test-terra-jupyter-hail.yml index 18c172ff..63493ae7 100644 --- a/.github/workflows/test-terra-jupyter-hail.yml +++ b/.github/workflows/test-terra-jupyter-hail.yml @@ -28,12 +28,17 @@ on: - 'terra-jupyter-hail/**' - '.github/workflows/test-terra-jupyter-hail.yml' + schedule: + - cron: '30 7 * * *' + workflow_dispatch: # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. env: GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 jobs: @@ -53,16 +58,23 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and base images too, if needed run: | gcloud auth configure-docker @@ -123,4 +135,3 @@ jobs: --entrypoint="" \ terra-jupyter-hail:smoke-test \ /bin/bash -c 'for nb in {terra-jupyter-python/tests,terra-jupyter-hail/tests}/*ipynb ; do jupyter nbconvert --to html_embed --execute "${nb}" ; done' - diff --git a/.github/workflows/test-terra-jupyter-python.yml b/.github/workflows/test-terra-jupyter-python.yml index 63fd0f14..f6f23f68 100644 --- a/.github/workflows/test-terra-jupyter-python.yml +++ b/.github/workflows/test-terra-jupyter-python.yml @@ -28,6 +28,9 @@ on: - 'terra-jupyter-python/**' - '.github/workflows/test-terra-jupyter-python.yml' + schedule: + - cron: '0 8 * * *' + workflow_dispatch: # Allows manually triggering of workflow on a selected branch via the GitHub Actions tab. # GitHub blog demo: https://github.blog/changelog/2020-07-06-github-actions-manual-triggers-with-workflow_dispatch/. @@ -36,6 +39,8 @@ env: GOOGLE_PROJECT: ${{ secrets.GCP_PROJECT_ID }} PIP_USER: "false" R_LIBS: "/home/jupyter/notebooks/packages" + # Create a plain image, not an attestation manifest list. + BUILDX_NO_DEFAULT_ATTESTATIONS: 1 jobs: @@ -55,16 +60,23 @@ jobs: run: sudo rm -rf /usr/share/dotnet - id: auth - uses: google-github-actions/auth@v1 + uses: google-github-actions/auth@v3 with: credentials_json: ${{ secrets.TD_GCP_SA_KEY }} create_credentials_file: true - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v0.3.0 + uses: google-github-actions/setup-gcloud@v3 with: project_id: ${{ secrets.GCP_PROJECT_ID }} + - name: Set up Docker Buildx + # driver: docker keeps build output in the local image store; the default + # docker-container driver would need --load and break the steps below. + uses: docker/setup-buildx-action@v4 + with: + driver: docker + - name: Build Docker image and base images too, if needed run: | gcloud auth configure-docker diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65f4b446..d901698d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,6 +1,6 @@ # Contributing -`terra-docker` consists of docker images that are compatible with the notebooks service in Terra called Leonardo. We follow the naming convention of `terra-[application]-[image_name]` (for example `terra-jupyter-aou`). +`terra-docker` consists of docker images that are compatible with the notebooks service in Terra called Leonardo. We follow the naming convention of `terra-[application]-[image_name]` (for example `terra-jupyter-bioconductor`). ## Opening a PR @@ -8,7 +8,7 @@ We do our work on a feature branch based on master. When you open a pull request This is also a good opportunity to verify that all the appropriate image versions were bumped. It is important to keep in mind which images extend from where ([Image Dependecies](https://github.com/DataBiosphere/terra-docker#image-dependencies)). Make sure to bump the version of the image you are changing and update its reference in all images that extend from it. -You can also run `amm ./updateVersions.sc ` to upgrade derived images. If you're updating `terra-jupyter-aou` image, please make a pull request similar to [this PR](https://github.com/DataBiosphere/leonardo/pull/1612) (less updating `terra_jupyter_aou_old`). +You can also run `amm ./updateVersions.sc ` to upgrade derived images. ## Legacy Image Versioning @@ -67,7 +67,7 @@ You can have a .json file with package versions and the tools that your image us ## Merging your PR -Make sure to get two thumbs before merging your PR. We often reach out to the AOU or Bioconductor team to also review changes we are making to their images (Bioconductor team may also want to review changes to `terra-jupyter-r`). +Make sure to get two thumbs before merging your PR. We often reach out to the Bioconductor team to also review changes we are making to their images (they may also want to review changes to `terra-jupyter-r`). Once you merge your PR, this will automatically kick off a Jenkins job (given title "terra-docker image hash update") to create a PR in Leonardo that bumps the image versions in [reference.conf](https://github.com/DataBiosphere/leonardo/blob/develop/automation/src/test/resources/reference.conf) under automation. This allows for our automation tests in Leonardo to run against your new docker image changes. diff --git a/build.sh b/build.sh index 1f1a87b6..9d3d67d0 100755 --- a/build.sh +++ b/build.sh @@ -50,10 +50,8 @@ docker kill $IMAGE_DIR || true docker rm -f $IMAGE_DIR || true docker run --rm -itd -u root -e PIP_USER=false --entrypoint='/bin/bash' --name $IMAGE_DIR $GCR_IMAGE_REPO/$IMAGE_DIR:$VERSION -if [[ $IMAGE_DIR != "terra-rstudio-aou" ]]; then - gcloud auth list - python scripts/generate_package_docs.py "$IMAGE_DIR" -fi +gcloud auth list +python scripts/generate_package_docs.py "$IMAGE_DIR" docker kill $IMAGE_DIR || true docker rm -f $IMAGE_DIR || true diff --git a/build_all.sh b/build_all.sh index 04912cc2..87e294d8 100755 --- a/build_all.sh +++ b/build_all.sh @@ -9,11 +9,8 @@ # 4- terra-jupyter-r # 5- terra-jupyter-gatk # 6- terra-jupyter-hail -# 7- terra-jupyter-aou -# 8- terra-jupyter-bioconductor -# 9- terra-rstudio-aou -# 10- wondershaper -images=("terra-base", "terra-jupyter-base" "terra-jupyter-python" "terra-jupyter-r" "terra-jupyter-gatk" "terra-jupyter-hail" "terra-jupyter-aou" "terra-jupyter-bioconductor" "terra-rstudio-aou" "wondershaper") +# 7- terra-jupyter-bioconductor +images=("terra-base" "terra-jupyter-base" "terra-jupyter-python" "terra-jupyter-r" "terra-jupyter-gatk" "terra-jupyter-hail" "terra-jupyter-bioconductor") # Loop over each image to build in the correct order for image in "${images[@]}"; do diff --git a/config/conf.json b/config/conf.json index 7758b4b8..11f76fb5 100644 --- a/config/conf.json +++ b/config/conf.json @@ -150,60 +150,6 @@ "include_in_custom_dataproc" : true, "include_in_custom_gce" : true } - }, - { - "name" : "terra-jupyter-aou", - "base_label" : "All of Us", - "tools" : [ - "python", - "r" - ], - "packages" : { - - }, - "version" : "2.2.16", - "automated_flags" : { - "include_in_ui" : false, - "generate_docs" : false, - "build" : true, - "include_in_custom_dataproc" : true, - "include_in_custom_gce" : true - } - }, - { - "name" : "terra-rstudio-aou", - "base_label" : "All of Us", - "tools" : [ - "r" - ], - "packages" : { - - }, - "version" : "0.1.1", - "automated_flags" : { - "include_in_ui" : false, - "generate_docs" : false, - "build" : true, - "include_in_custom_dataproc" : false, - "include_in_custom_gce" : false - } - }, - { - "name" : "wondershaper", - "base_label" : "wondershaper", - "tools" : [ - ], - "packages" : { - - }, - "version" : "0.0.1", - "automated_flags" : { - "include_in_ui" : false, - "generate_docs" : false, - "build" : false, - "include_in_custom_dataproc" : false, - "include_in_custom_gce" : false - } } ] } diff --git a/mind.puml b/mind.puml index 7d71527f..536de18b 100644 --- a/mind.puml +++ b/mind.puml @@ -2,7 +2,6 @@ * terra-jupyter-base ** terra-jupyter-python *** terra-jupyter-gatk -**** terra-jupyter-aou *** terra-jupyter-hail ** terra-jupyter-r *** terra-jupyter-gatk diff --git a/terra-jupyter-aou/CHANGELOG.md b/terra-jupyter-aou/CHANGELOG.md deleted file mode 100644 index e6391912..00000000 --- a/terra-jupyter-aou/CHANGELOG.md +++ /dev/null @@ -1,639 +0,0 @@ -## 1.1.14 - 2025-05-01 -- Update `hail` to `0.2.134` to be compatible with dataproc 2.2.x versions - - See https://github.com/hail-is/hail/releases/tag/0.2.134 for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.16` - -## 2.2.15 - 2025-04-24 - -- Update `terra-jupyter-gatk` to `2.3.9` - - Update `terra-jupyter-r` to `2.2.7` and `terra-jupyter-python` to `1.1.6` - - Update `terra-jupyter-base` to `1.1.4` - - Update base image to `gcr.io/deeplearning-platform-release/tf2-cu123.2-17.py310` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.15` - -## 2.2.14 - 2024-12-04T17:30:49.302274524Z - -- Update `terra-jupyter-r` to `2.2.6` - - Update for R 4.4.2 and Bioc 3.20 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.14` - -## 2.2.13 - 2024-08-12 -- Update `hail` to `0.2.130.post1` - - See https://github.com/hail-is/hail/releases/tag/0.2.130.post1 for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.13` - -## 2.2.12 - 2024-05-15T15:36:28.616710060Z - -- Update `terra-jupyter-r` to `2.2.5` - - Update for R 4.4.0 and Bioc 3.19 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.12` - -## 2.2.11 - 2024-04-10 -- Update `hail` to `0.2.130` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-130) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.11` - -## 2.2.10 - 2024-04-05 -- Update `hail` to `0.2.129` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-129) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.10` - -## 2.2.9 - 2024-01-12 -- Update `hail` to `0.2.128` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-128) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.9` - -## 2.2.8 - 2023-11-14T15:56:44.507726350Z - -- Update `terra-jupyter-gatk` to `2.3.6` - - Bioconductor 3.18 release - - Samtools 1.18 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.8` - -## 2.2.7 - 2023-11-01 -- Pin `gnomad` to `0.6.4` - - See https://github.com/broadinstitute/gnomad_methods/releases for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.7` - -## 2.2.6 - 2023-10-30 -- Update `hail` to `0.2.126` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-126 for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.6` - -## 2.2.5 - 2023-10-27 -- Update `hail` to `0.2.125` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-125 for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.5` - -## 2.2.4 - 2023-10-06 -- Update `terra-jupyter-gatk` to `2.3.4` - - Update `terra-jupyter-python` to `1.1.4` - - Downgrade pytz to `2023.3` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.4` - -## 2.2.3 - 2023-10-03 -- Update `terra-jupyter-gatk` to `2.3.3` - - Update `terra-jupyter-python` to `1.1.3` - - Update `terra-jupyter-base` to `1.1.3` - - Downgrade notebook to `6.5.4` and jupyterlab-server to `2.23.0` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.3` - -## 2.2.2 - 2023-07-31 -- Update `terra-jupyter-gatk` to `2.3.2` - - Update `terra-jupyter-python` to `1.1.2` - - Update `terra-jupyter-base` tp `1.1.2` - - Upgrade Terra Notebook Utils to `0.13.0` -- Update PanGenie to 3.0.1 and pinning version -- Update RMBlast to 2.14.0 -- Update RepeatMasker to 4.1.5 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.2` - -## 2.2.1 - 2023-07-28 - -- Fix bug introduced in https://github.com/DataBiosphere/terra-docker/commit/4a5b4c9212aedcafa2f41fbeb2b161089341c578 -- Remove outdated cromshell alias -- Remove Java 8 and update `hail` to `0.2.120` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-120 for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.1` - -## 2.2.0 - 2023-06-23 - -- Update python from 3.7 to 3.10 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.2.0` - -## 2.1.22 - 2023-06-01T17:49:47.721962740Z - -- Update `terra-jupyter-r` to `2.1.10` - - Bioconductor 3.17 release with R 4.3 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.22` - -## 2.1.21 - 2023-05-24 -- Rollback `hail` to `0.2.107` -- Fix `cromshell` alias typo - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.21` -- - -## 2.1.20 - 2023-04-11 -- Update `hail` to `0.2.113` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-113) for details -- Replace r-saige with SAIGEgds. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.20` -- - -## 2.1.19 - 2023-03-16 -- Update `hail` to `0.2.112` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-112) for details -- Add alias for `cromshell` -- Remove `workspace_cromwell.p` -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.19` - -## 2.1.18 - 2023-03-13T17:26:34.247154Z - -- Update `terra-jupyter-base` to `1.0.14` - - Include `jupyter lab build` step - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.18` - -## 2.1.17 - 2023-02-08 -- Update `hail` to `0.2.109` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-109) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.17` - -## 2.1.16 - 2023-02-09T14:18:49.188741Z - -- Update `terra-jupyter-base` to `1.0.12` - - Add cloud.google.com gpg key. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.16` - -## 2.1.15 - 2022-12-14 -- Update `hail` to `0.2.107` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-107) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.15` - -## 2.1.14 - 2022-12-12 - -- Add tools for long reads support - - jellyfish 2.3.0 - - pangenie - - minimap2 2.24 - - whatshap - - TRF - - HMMER 3.3 - - rmblast 2.11.0 - - RepeatMasker 4.1.4 - - Text::Soundex - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.14` - -## 2.1.13 - 2022-11-22T20:49:44.192223144Z - -- Update `terra-jupyter-r` to `2.1.7` - - Update Bioconductor for 3.16 release - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.13` - -## 2.1.12 - 2022-10-21 -- Update `hail` to `0.2.104` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-104) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.12` - -## 2.1.11 - 2022-10-04 -- Update `hail` to `0.2.101` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-101) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.11` - -## 2.1.10 - 2022-09-23 -- Update `hail` to `0.2.100` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-100) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.10` - -## 2.1.9 - 2022-09-02 - - - Add support for QIIME2 - - Update `hail` to `0.2.98` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-98) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.9` - -## 2.1.8 - 2022-07-12 - -- Revert attempted Java11 upgrade back to Java8 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.8` - -## 2.1.7 - 2022-06-30 -- Update `hail` to `0.2.96` - - See https://hail.is/docs/0.2/change_log.html#version-0-2-96) for details - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.7` - -## 2.1.6 - 2022-06-23T10:58:12.961300Z - -- Update `terra-jupyter-base` to `1.0.10` - - Fix leo_url variable in workspace_cromwell.py script for AoU projects - -## 2.1.5 - 2022-06-09 - -- Update `terra-jupyter-gatk` base to `2.2.5` - - Fix Python sites-packages to avoid issues on uninstall -- Bump Hail to 0.2.93 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.5` - -## 2.1.4 - 2022-06-03T18:35:22.582705Z - -- reverted, do not use - -## 2.1.3 - 2022-05-20T18:06:39.587654Z - -- Update `terra-jupyter-base` to `1.0.9` - - Fix adding workspace_cromwell.py script to manage Cromwell App - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.3` - -## 2.1.2 - 2022-05-17T17:14:41.365718Z - -- Update `terra-jupyter-base` to `1.0.8` - - Add script that manages Cromwell app - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.2` - -## 2.1.1 - 2022-05-10T22:08:26.056998Z - -- Update `terra-jupyter-base` to `1.0.7` - - Install Cromshell 2.0 (https://github.com/broadinstitute/cromshell) - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.1.1` - -## 2.1.0 - 2022-05-02 - -- Update `terra-jupyter-r` to `2.1.0` - - Update Bioconductor to 3.15.0 and R to 4.2.0 - -## 2.0.15 - 2022-05-02T14:33:24.031254Z - -- Update `terra-jupyter-base` to `1.0.6` - - use new nvidia key -- Update `terra-jupyter-gatk` to `2.1.0` - - Install Nextflow in GATK image - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.15` - -## 2.0.14 - 2022-04-17 - -- Update `terra-jupyter-gatk` to `2.0.9` - - Install gcloud alpha storage dependency -- Install SAIGE -- Add igv-jupyter extension by default - -## 2.0.13 - 2022-03-18 - -- Update `terra-jupyter-gatk` to `2.0.8` -- Update Hail installation to latest - -## 2.0.12 - 2022-02-02T20:47:42.403024Z - -- Update `terra-jupyter-base` to `1.0.4` - - update notebook_dir to $HOME; install scikit-learn-intelex, xgboost - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.12` - -## 2.0.11 - 2022-01-25 - -- Add additional genomics tools: gcta, PRsice, vcftools, plink2 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.11` - -## 2.0.10 - 2022-01-06T18:21:41.240337Z - -- Update `terra-jupyter-base` to `1.0.3` - - Bumping Google Deeplearning image to 2.7, removing gcc-6 and gcc-8 install due to package problems - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.10` - -## 2.0.9 - 2021-12-14T19:48:03.857067Z - -- Update `terra-jupyter-r` to `2.0.3` - - Update Terra Jupyter image to use latest Bioconductor 3.14 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.9` - -## 2.0.8 - 2021-12-16 - -- Bump Hail version to 0.2.74 - -## 2.0.7 - 2021-12-15T18:29:12.526374Z -- Update `terra-jupyter-gatk` base image to `2.0.5` - - Upgrade gatk image to 4.2.4.0 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.7` - -## 2.0.6 - 2021-12-13 - -- Update `terra-jupyter-gatk` base image to `2.0.4` - - Update AsyncMappingKernelManager https://github.com/jupyter/notebook/issues/6164 - -## 2.0.5 - 2021-12-02T21:50:37.297833 - -- Fix nextflow install path. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.5` - -## 2.0.4 - 2021-11-24T05:51:34.197137 - -- Install nextflow -- Update bolt lmm to 2.3.6 -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.4` - -## 2.0.3 - 2021-10-07T14:47:43.445699Z - -- Unpinning cwltool version and updating protobuf version to 3.18 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.3` - -## 2.0.2 - 2021-09-10T15:10:44.156815Z - -- Update `terra-jupyter-base` to `1.0.1` - - Update base image to gcr.io/deeplearning-platform-release/tf2-gpu.2-6 to support TensorFlow 2.6.0 - - Fix multipart Jupyter uploads - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.2` - -## 2.0.1 - 2021-07-20 - -- Upgrade `terra-jupyter-gatk` to `2.0.1` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:2.0.1` - -## 2.0.0 - 2021-06-28T19:39:56.024375Z - -- Update `terra-jupyter-base` to `1.0.0` - - use `gcr.io/deeplearning-platform-release/base-cu110:latest` as base - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-r:2.0.0` - -## 1.1.7 - 2021-06-30 - -- Add less -- Add papermill -- Install a global gitignore to ensure that by default only code files and documentation are commited to source control. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.7` - -## 1.1.6 - 2021-06-07T19:39:56.024375Z - -- Update `terra-jupyter-r` to `1.0.16` - - Update R to 4.1.0 and Bioconductor version to 3.13. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.6` - -## 1.1.5 - 2021-05-24 - -- Reduce unnecessary package upgrades. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.5` - -## 1.1.4 - 2021-05-24 - -- Add bcftools - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.4` - -## 1.1.3 - 2021-05-05T16:19:57.558388Z - -- Update `terra-jupyter-base` to `0.0.20` - - [IA-1644] Install crcmod in base image to support GCS composite objects - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.3` - -## 1.1.2 - 2021-04-21 - -- Refactor to terra-jupyter-gatk:1.1.1 as the base image - - GATK now included - - samtools now included -- Install bolt-lmm -- Install regenie -- Install Nirvana -- Install R packages: SAIGE, GENESIS -- Update Plink version - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.2` - -## 1.1.1 - 2021-04-05T17:29:05.447Z - -- Update `terra-jupyter-python` to `0.1.1` - - fix ggplot import bug - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.1` - -## 1.1.0 - 2021-03-20 - -- Update `terra-jupyter-python` to `0.1.0` -- Remove more Python package overrides which are no longer needed due to updates in the base Python image. -- Add nbstripout and enable it globally so that its in effect for any git repository cloned to the VM. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.1.0` - -## 1.0.23 - 2021-02-22 - -- Remove varstore scripts / GATK jar -- Upgrade Hail -- Remove some Python package / R lib overrides which are no longer needed - -## 1.0.22 - 2021-02-10T00:36:57.205Z - -- Update `terra-jupyter-r` to `1.0.13` - - [IA-2511] Install Seurat R package in terra-jupyter-r image - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.22` - -## 1.0.21 - 2021-01-20T16:00:48.370Z - -- Update `terra-jupyter-base` to `0.0.19` - - [IA-2472] Turn on debug-level logging when JUPYTER_DEBUG_LOGGING env var is true - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.21` - -## 1.0.20 - 2020-12-09T18:50:57.334Z - -- Update `terra-jupyter-python` to `0.0.22` - - [IA-2420] Update to a newer version of pymc3 -- Update hail version to `0.2.61` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.20` - -## 1.0.19 - 2020-12-02T17:59:36.458Z - -- Update `terra-jupyter-base` to `0.0.18` - - bump terra-notebook-utils version to 0.7.0 and move it to base image - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.19` - -## 1.0.18 - 2020-11-16T18:11:40.848741Z - -- Remove dependency on miniconda -- Update `terra-jupyter-base` to `0.0.17` - - remove miniconda - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.18` - -## 1.0.17 - 2020-11-10T13:26:19.148Z - -- Update `terra-jupyter-python` to `0.0.19` - - Remove Python package enum34. Fixes https://github.com/DataBiosphere/terra-docker/issues/175 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.17` - -## 1.0.16 - 2020-11-04T21:43:54.919Z - -- Update `terra-jupyter-r` to `1.0.9` - - Version upgrades: Bioconductor version 3.12 and R version 4.0.3 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.16` - -## 1.0.15 - 2020-10-26T20:13:57.770Z - -- Update `terra-jupyter-base` to `0.0.16` - - add keychain to base image - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.15` - -## 1.0.14 - 2020-10-22 - -- Don't use gsutil, it creates local credentials files which break downstream CLI usage - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.14` - -## 1.0.13 - 2020-10-21 - -- Add [variantstore](https://github.com/broadinstitute/variantstore) extraction tool -- Update gatk custom jar build to latest - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.13` - -## 1.0.12 - 2020-10-08T18:42:19.744Z - -- Update `terra-jupyter-base` to `0.0.15` - - include pip binary installation location in PATH - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.12` - -## 1.0.11 - 09/29/2020 - - - Just download the custom GATK jar instead of building it. - -## 1.0.10 - 09/18/2020 - - - Add Hail / Spark libs. - - Bump GATK jar to latest commit ah_var_store branch: - https://github.com/broadinstitute/gatk/tree/ah_var_store - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.10` - -## 1.0.9 - 09/17/2020 - - - Install plink binary - - Build/install ah_var_store branch of GATK jar: - https://github.com/broadinstitute/gatk/tree/ah_var_store - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.9` - -## 1.0.8 - 2020-09-02T15:12:19.917Z - -- Update `terra-jupyter-base` to `0.0.14` - - Terminal now opens to /notebooks directory where PD is mounted - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.8` - -## 1.0.7 - 2020-08-18T13:43:26.277Z - -- Update `terra-jupyter-base` to `0.0.13` - - update notebook to 6.1.1, tornado to 5.1.1 - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.7` - -## 1.0.6 - 8/18/2020 - -- Update to newer TensorFlow python package. -- Update numpy to match documented version. -- Reinstall AoU fork of bigrquery (new patch taken from ahead of the main package) - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.6` - -## 1.0.5 - 07/27/2020 - -- Update `terra-jupyter-python` to `0.0.14` -- Update `terra-jupyter-r` to `1.0.4` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.5` - -## 1.0.4 - 07/28/2020 - -- Drop fork install of bigrquery (patches have now been merged and released in the main bigrquery package) -- Install new AoU Python library -- Avoid installing pandas 1.0 for now; pin other packages - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.4` - -## 1.0.3 - 06/29/2020 - -- Only user package installations will persist (installed to `/home/jupyter-user/notebooks/packages`) not image installed packages -- Update `terra-jupyter-r` base image to `1.0.3` -- Update `terra-jupyter-aou` to version `1.0.3` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.3` - -## 1.0.2 - 06/26/2020 - -- Update `terra-jupyter-r` base image to `1.0.2` -- Update `terra-jupyter-aou` to version `1.0.2` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.2` - -## 1.0.1 - 06/15/2020 - -- Update `terra-jupyter-r` base image to `1.0.1` and `terra-jupyter-python` base image to `0.0.13` -- Update `terra-jupyter-aou` to version `1.0.1` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.1` - -## 1.0.0 - 05/21/2020 - -- Update `terra-jupyter-aou` to version `1.0.0` -- Update `terra-jupyter-r` base image to `1.0.0` - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:1.0.0` - -## 0.0.4 - 05/18/2020 - -- Update `terra-jupyter-python` base image to `0.0.12` and `terra-jupyter-r` base image to `0.0.14` - - Adds jupyter notebook extension collapsible headers and code-folding - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:0.0.4` - -## 0.0.3 - 04/28/2020 - -- Update `terra-jupyter-python` base image to `0.0.11` - - Add Google Cloud support to pysam - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:0.0.3` - -## 0.0.2 - 04/24/2020 - -- Update to `terra-jupyter-r:0.0.13`. -- Update `terra-jupyter-aou` image version to `0.0.2`. - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:0.0.2` - -## 0.0.1 - 02/28/2020 - -- Iniitalized image; based on terra-jupyter-gatk, combined with the original - [All of Us initialization script](https://github.com/all-of-us/workbench/blob/5ee08d7cd8f069963e5d35877b711c0643e1c8b5/api/cluster-resources/initialize_notebook_cluster.sh) - -Image URL: `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:0.0.1` diff --git a/terra-jupyter-aou/Dockerfile b/terra-jupyter-aou/Dockerfile deleted file mode 100644 index 8ad41b8c..00000000 --- a/terra-jupyter-aou/Dockerfile +++ /dev/null @@ -1,289 +0,0 @@ -FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-gatk:2.3.9 - -USER root - -# need to apt-get everything for python since we can only copy pip installed packages -RUN apt-get update && apt-get install -yq --no-install-recommends \ - jq \ - g++ \ - less \ - liblz4-dev \ - libmagick++-dev \ - iproute2 \ - # Nirvana/.NET Core dependencies \ - ca-certificates \ - libc6 \ - libgcc1 \ - libgssapi-krb5-2 \ - libicu70 \ - liblttng-ust1 \ - libssl3 \ - libstdc++6 \ - zlib1g \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -ENV PIP_USER=false - -RUN pip3 install --upgrade \ - "pandas_profiling==3.6.6" \ - "markupsafe==2.1.2" - -RUN pip3 install \ - nbstripout \ - papermill \ - dsub \ - "git+https://github.com/all-of-us/workbench-snippets.git#egg=terra_widgets&subdirectory=py" \ - 'PyYAML==6.0.1' - - -# 2.0.0 has breaking changes and we're not ready for them yet. -RUN pip3 install "igv-jupyter>=1.0.0,<2.0.0" -RUN jupyter nbextension enable --py igv --sys-prefix - -# Spark/Hail setup. -# Copied from terra-jupyter-hail; keep updated. -ENV PYTHONPATH $PYTHONPATH:/usr/lib/spark/python - -ENV PYSPARK_PYTHON=python3 - -ENV HAIL_VERSION=0.2.134 - -RUN find $JUPYTER_HOME/scripts -name '*.sh' -type f | xargs chmod +x \ - && $JUPYTER_HOME/scripts/kernel/kernelspec.sh $JUPYTER_HOME/scripts/kernel /opt/conda/share/jupyter/kernels \ - # Note Spark and Hadoop are mounted from the outside Dataproc VM. - # Make empty conf dirs for the update-alternatives commands. - && mkdir -p /etc/spark/conf.dist && mkdir -p /etc/hadoop/conf.empty && mkdir -p /etc/hive/conf.dist \ - && update-alternatives --install /etc/spark/conf spark-conf /etc/spark/conf.dist 100 \ - && update-alternatives --install /etc/hadoop/conf hadoop-conf /etc/hadoop/conf.empty 100 \ - && update-alternatives --install /etc/hive/conf hive-conf /etc/hive/conf.dist 100 \ - && apt-get update \ - # Note that starting on 2.1.x, dataproc VM also provides java 11, which is incompatible with 8. No need to install it in the container - # && apt install -yq --no-install-recommends openjdk-8-jdk \ - # g++ \ - # liblz4-dev \ - # && update-alternatives --set java /usr/lib/jvm/java-8-openjdk-amd64/jre/bin/java \ - && pip3 install pypandoc gnomad==0.6.4 \ - && pip3 install --no-dependencies hail==$HAIL_VERSION \ - && X=$(mktemp -d) \ - && requirements_file=$(mktemp) \ - && mkdir -p $X \ - && (cd $X && pip3 download hail==$HAIL_VERSION --no-dependencies && \ - unzip hail*.whl && \ - grep 'Requires-Dist: ' hail*dist-info/METADATA | sed 's/Requires-Dist: //' | sed 's/ (//' | sed 's/)//' | grep -v 'pyspark' >$requirements_file && \ - pip install -r $requirements_file) \ - && rm -rf $X \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Install Wondershaper from source, for client-side egress limiting. -RUN cd /usr/local/share && \ - git clone https://github.com/magnific0/wondershaper.git --depth 1 && \ - cd wondershaper && \ - make install && \ - cd $HOME - -# Plink install -ENV PLINK_VERSION 20210416 -RUN mkdir -p /tmp/plink && \ - cd /tmp/plink && \ - curl -fsSL -o plink.zip "http://s3.amazonaws.com/plink1-assets/plink_linux_x86_64_${PLINK_VERSION}.zip" && \ - unzip plink.zip && \ - mv plink /bin/plink && \ - cd $HOME && \ - rm -rf /tmp/plink - -RUN mkdir -p /tmp/plink2 && \ - cd /tmp/plink2 && \ - curl -fsSL -o plink2.zip "https://s3.amazonaws.com/plink2-assets/plink2_linux_x86_64_latest.zip" && \ - unzip plink2.zip && \ - mv plink2 /bin/plink2 && \ - cd $HOME && \ - rm -rf /tmp/plink2 - -RUN mkdir -p /tmp/prsice && \ - cd /tmp/prsice && \ - curl -fsSL -o prsice.zip https://github.com/choishingwan/PRSice/releases/download/2.3.5/PRSice_linux.zip && \ - unzip prsice.zip && \ - rm prsice.zip && \ - cd $HOME && \ - mv /tmp/prsice /opt/prsice && \ - ln -s "/opt/prsice/PRsice_linux" /bin/prsice - -# BOLT-LMM install -ENV BOLT_LMM_VERSION v2.4.1 -RUN mkdir -p /tmp/bolt-lmm && \ - cd /tmp/bolt-lmm && \ - curl -fsSL -o bolt-lmm.tar.gz "https://storage.googleapis.com/broad-alkesgroup-public/BOLT-LMM/downloads/BOLT-LMM_${BOLT_LMM_VERSION}.tar.gz" && \ - tar -xvzf bolt-lmm.tar.gz && \ - mv "BOLT-LMM_${BOLT_LMM_VERSION}" /opt/bolt-lmm && \ - ln -s "/opt/bolt-lmm/bolt" /bin/bolt && \ - cd $HOME && \ - rm -rf /tmp/bolt-lmm - -# REGENIE install -ENV REGENIE_VERSION v2.0.2 -RUN mkdir -p /tmp/regenie && \ - cd /tmp/regenie && \ - curl -fsSL -o regenie.zip "https://github.com/rgcgithub/regenie/releases/download/${REGENIE_VERSION}/regenie_${REGENIE_VERSION}.gz_x86_64_Linux.zip" && \ - unzip regenie.zip && \ - mv regenie_${REGENIE_VERSION}.gz_x86_64_Linux /bin/regenie && \ - cd $HOME && \ - rm -rf /tmp/regenie - -# Install .NET Core + Nirvana -ENV DOTNET_VERSION=2.1.26 -ENV NIRVANA_VERSION 3.14.0 -ENV NIRVANA_ZIP_URL="https://github.com/Illumina/Nirvana/releases/download/v${NIRVANA_VERSION}/Nirvana-${NIRVANA_VERSION}-dotnet-2.1.0.zip" - -# Based on https://github.com/MichaelStromberg-Illumina/DotNetMisc/blob/master/Dockerfiles/Nirvana/Dockerfile#L13 -RUN curl -SL --output dotnet.tar.gz "https://dotnetcli.azureedge.net/dotnet/Runtime/${DOTNET_VERSION}/dotnet-runtime-${DOTNET_VERSION}-linux-x64.tar.gz" \ - && dotnet_sha512='41cc13f14dd7721a079bdd0ab489de40e9d4f32787239a26e7d10fcb0020a8e78d446c3b430b4bf80a557a925b3ca87d7981bfda4bbf9495cc44b1d42d877c40' \ - && echo "${dotnet_sha512} dotnet.tar.gz" | sha512sum -c - \ - && mkdir -p /usr/share/dotnet \ - && tar -zxf dotnet.tar.gz -C /usr/share/dotnet \ - && rm dotnet.tar.gz \ - && ln -s /usr/share/dotnet/dotnet /usr/bin/dotnet \ - && mkdir -p /opt/nirvana \ - && curl -SL --output nirvana.zip $NIRVANA_ZIP_URL \ - && unzip -d /opt/nirvana nirvana.zip \ - && rm nirvana.zip - -ENV GCTA_VERSION=v1.94.0Beta -RUN mkdir -p /tmp/gcta && \ - cd /tmp/gcta && \ - curl -fsSL -o gcta.zip "https://yanglab.westlake.edu.cn/software/gcta/bin/gcta_${GCTA_VERSION}_linux_kernel_4_x86_64.zip" && \ - unzip gcta.zip && \ - mv "gcta_${GCTA_VERSION}_linux_kernel_4_x86_64/gcta_${GCTA_VERSION}_linux_kernel_4_x86_64_static" /bin/gcta && \ - cd $HOME && \ - rm -rf /tmp/gcta - -# Install bcftools -RUN mkdir -p /tmp/bcftools && \ - cd /tmp/bcftools && \ - curl -fsSL -o bcftools.tar.bz2 https://github.com/samtools/bcftools/releases/download/1.12/bcftools-1.12.tar.bz2 && \ - bzip2 -d bcftools.tar.bz2 && \ - tar xvf bcftools.tar && \ - cd bcftools-1.12 && \ - make && \ - make install && \ - cd $HOME && \ - rm -rf /tmp/bcftools - -# Install vcftools -RUN mkdir -p /tmp/vcftools && \ - cd /tmp/vcftools && \ - curl -fsSL -o vcftools.tar.gz https://github.com/vcftools/vcftools/releases/download/v0.1.16/vcftools-0.1.16.tar.gz && \ - tar xvzf vcftools.tar.gz && \ - cd vcftools-0.1.16 && \ - ./configure && \ - make && \ - make install && \ - cd $HOME && \ - rm -rf /tmp/vcftools - -RUN R -e 'BiocManager::install(c("GENESIS"))' - -RUN R -e 'BiocManager::install(c("SAIGEgds","SNPRelate"))' - -# Install hdf5 library and h5py -RUN apt-get update && \ - apt install -yq --no-install-recommends \ - libhdf5-serial-dev \ - cmake \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* -RUN pip install --upgrade pip && pip install --verbose h5py - -# QIIME 2 2024.5 Amplicon Distribution -RUN curl -fsSL -o qiime2-2024.5.yml https://data.qiime2.org/distro/amplicon/qiime2-amplicon-2024.5-py39-linux-conda.yml && \ - conda env create -n qiime2-2024.5 --file qiime2-2024.5.yml && \ - rm qiime2-2024.5.yml - -# Install jellyfish -RUN mkdir -p /tmp/jellyfish && \ - cd /tmp/jellyfish && \ - curl -fsSL -o jellyfish.tar.gz https://github.com/gmarcais/Jellyfish/releases/download/v2.3.0/jellyfish-2.3.0.tar.gz && \ - tar xvf jellyfish.tar.gz && \ - cd jellyfish-2.3.0 && \ - ./configure && \ - make && \ - make install && \ - cd $HOME && \ - rm -rf /tmp/jellyfish - -# Install Cereal headers (dependency of pangenie) -RUN mkdir -p /tmp/cereal && \ - cd /tmp/cereal && \ - curl -fsSL -o cereal.tar.gz https://github.com/USCiLab/cereal/archive/refs/tags/v1.3.2.tar.gz && \ - tar xvf cereal.tar.gz && \ - cd cereal-1.3.2/include && \ - mv cereal/ /usr/local/include/ && \ - cd $HOME && \ - rm -rf /tmp/cereal - -# Install pangenie -RUN mkdir -p /tmp/pangenie && \ - cd /tmp/pangenie && \ - curl -fsSL -o pangenie.tar.gz https://github.com/eblerjana/pangenie/archive/refs/tags/v4.1.0.tar.gz && \ - tar xvf pangenie.tar.gz && \ - cd pangenie-4.1.0 && \ - mkdir build; cd build; cmake .. ; make && cd src && \ - mv *.so /lib && \ - mv * /bin/ && \ - cd $HOME && \ - rm -rf /tmp/pangenie - -# Install minimap2 -RUN curl -fsSL https://github.com/lh3/minimap2/releases/download/v2.24/minimap2-2.24_x64-linux.tar.bz2 | tar -jxvf - && cd minimap2-2.24_x64-linux/ && cp minimap2 /bin/minimap2 && cd $HOME - -# Run whatshap (>1h install) -# RUN conda install -c bioconda whatshap -# Alternatively, not using conda (1 min install) -RUN pip install --verbose git+https://github.com/whatshap/whatshap - -# TRF -RUN mkdir -p /tmp/trf && cd /tmp/trf && \ - git clone https://github.com/Benson-Genomics-Lab/TRF.git && cd TRF && \ - mkdir build && cd build && ../configure && make && make install && \ - cd $HOME && \ - rm -rf /tmp/trf - -# HMMER -RUN mkdir -p /tmp/hmmer && cd /tmp/hmmer && \ - curl -fsSL -o hmmer-3.3.tar.gz http://eddylab.org/software/hmmer/hmmer-3.3.tar.gz && \ - tar -xzf hmmer-3.3.tar.gz && cd hmmer-3.3 && ./configure && \ - make && make install && cd .. && rm -r hmmer-3.3 hmmer-3.3.tar.gz && cd $HOME && \ - rm -rf /tmp/hmmer - -# rmblast (200M) -RUN mkdir -p /tmp/rmblast && cd /tmp/rmblast && \ - curl -fsSL -o rmblast-2.14.0+-x64-linux.tar.gz https://www.repeatmasker.org/rmblast/rmblast-2.14.0+-x64-linux.tar.gz && \ - tar -zxvf rmblast-2.14.0+-x64-linux.tar.gz && \ - cp rmblast-2.14.0/bin/* /usr/local/bin && \ - cd $HOME && \ - rm -rf /tmp/rmblast - -RUN PERL_MM_USE_DEFAULT=1 cpan Text::Soundex - -# Install RepeatMasker (900M) -RUN curl -fsSL http://www.repeatmasker.org/RepeatMasker/RepeatMasker-4.1.5.tar.gz \ - | (cd /usr/local; tar -xz) - -# Remove workspace_cromwell script as it may allow user bypass AoU to create Cromwell via Leo directly -RUN rm -f /usr/local/bin/workspace_cromwell.py - -# Downgrade Jinja2 as the newer version breaks the notebook -# See https://jinja.palletsprojects.com/en/3.1.x/changes/#version-3-1-0 -RUN pip install --force-reinstall -v "Jinja2==3.0.3" - -ENV PATH="$PATH:/usr/local/RepeatMasker" - -# Turn off beatrix-jupyterlab and bq_stats plugins, they don't activate cleanly in our version -# of jupyter and they're google-managed packages we don't have access to older versions of. -# Retain two plugins: google.cloud.bigquery and sql -RUN sed -i "s/^\( *'beatrix_jupyterlab',\)/#\1/g" /etc/ipython/ipython_kernel_config.py -RUN sed -i "s/^\( *'bq_stats',\)/#\1/g" /etc/ipython/ipython_kernel_config.py - -ENV PIP_USER=true -USER $USER diff --git a/terra-jupyter-aou/README.md b/terra-jupyter-aou/README.md deleted file mode 100644 index 35bdd9ba..00000000 --- a/terra-jupyter-aou/README.md +++ /dev/null @@ -1,20 +0,0 @@ -# terra-jupyter-aou image - -This repo contains the terra-jupyter-aou image that is compatible with notebook service in [Terra]("https://app.terra.bio/") called Leonardo. For example, use `us.gcr.io/broad-dsp-gcr-public/terra-jupyter-aou:{version}` in terra. - -This image is intended to support usage in the [All of Us Workbench](https://github.com/all-of-us/workbench), which depends on Leonardo. - -## Image contents - -The terra-jupyter-aou extends the [terra-jupyter-python](../terra-jupyter-python/README.md) and [terra-jupyter-r](../terra-jupyter-r/README.md) by including the following: - -- Open JDK -- Libraries to support All of Us anlaysis use cases - -To see the complete contents of this image please see the [Dockerfile](./Dockerfile). - -## Selecting prior versions of this image - -To select an older version this image, you can search the [CHANGELOG.md](./CHANGELOG.md) for a specific package version you need. - -Once you find an image version that you want, simply copy and paste the image url from the changelog into the corresponding custom docker field in the Terra notebook runtime widget. diff --git a/terra-jupyter-aou/gitignore_global b/terra-jupyter-aou/gitignore_global deleted file mode 100644 index e492c72e..00000000 --- a/terra-jupyter-aou/gitignore_global +++ /dev/null @@ -1,15 +0,0 @@ -# By default, all files should be ignored by git. -# We want to be sure to exclude files containing data such as CSVs and images such as PNGs. -*.* -# Now, allow the file types that we do want to track via source control. -!*.ipynb -!*.py -!*.r -!*.R -!*.wdl -!*.sh -# Allow documentation files. -!*.md -!*.rst -!LICENSE* - diff --git a/terra-jupyter-aou/tests/aou_r_test.ipynb b/terra-jupyter-aou/tests/aou_r_test.ipynb deleted file mode 100644 index f65ffee8..00000000 --- a/terra-jupyter-aou/tests/aou_r_test.ipynb +++ /dev/null @@ -1,68 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify pre-installed libraries are loadable." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "library(SAIGE)\n", - "library(GENESIS)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Verify we're able to install additional libraries" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "install.packages('skimr')\n", - "library(skimr)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "R", - "language": "R", - "name": "ir" - }, - "language_info": { - "codemirror_mode": "r", - "file_extension": ".r", - "mimetype": "text/x-r-source", - "name": "R", - "pygments_lexer": "r", - "version": "4.1.2" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": false - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/terra-jupyter-aou/tests/aou_smoke_test.ipynb b/terra-jupyter-aou/tests/aou_smoke_test.ipynb deleted file mode 100644 index 1bb59ad5..00000000 --- a/terra-jupyter-aou/tests/aou_smoke_test.ipynb +++ /dev/null @@ -1,170 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test cases requiring or benefiting from the context of a notebook\n", - "\n", - "If the notebook runs successfully from start to finish, the test is successful!\n", - "\n", - "TODO(all): Add additional tests and/or tests with particular assertions, as we encounter Python package version incompatibilities not currently detected by these tests.\n", - "\n", - "In general, only add test cases here that require the context of a notebook. This is because this notebook, as currently written, will abort at the **first** failure. Compare this to a proper test suite where all cases are run, giving much more information about the full extent of any problems encountered." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Package versions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip3 freeze" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip3 show plotnine pandas google-cloud-storage google-cloud-bigquery google-resumable-media" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test cases requiring the context of a notebook " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test Terra widgets\n", - "\n", - "TODO(deflaux) this test could be enabled if we shared a Terra test workspace with the service account used for the GitHub Action." - ] - }, - { - "cell_type": "raw", - "metadata": {}, - "source": [ - "from terra_widgets.html_snapshots import display_html_snapshots_widget\n", - "\n", - "# This will display a user interface to interact with HTML snapshots stored in the workspace bucket.\n", - "display_html_snapshots_widget()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Test cases benefiting from the context of a notebook " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test AoU workbench snippets setup\n", - "\n", - "We test these here since they don't test well via `pytest` because they all have the same module name." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "\n", - "git clone https://github.com/all-of-us/workbench-snippets.git /tmp/workbench-snippets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export WORKSPACE_CDR=placeholder_string\n", - "python3 /tmp/workbench-snippets/dataset-snippets/snippets_setup.py" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export WORKSPACE_CDR=placeholder_string\n", - "python3 /tmp/workbench-snippets/sql-snippets/snippets_setup.py" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%bash\n", - "\n", - "export WORKSPACE_CDR=placeholder_string\n", - "python3 /tmp/workbench-snippets/storage-snippets/snippets_setup.py" - ] - } - ], - "metadata": { - "environment": { - "name": "r-cpu.3-6.m56", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/r-cpu.3-6:m56" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - }, - "toc": { - "base_numbering": 1, - "nav_menu": {}, - "number_sections": true, - "sideBar": true, - "skip_h1_title": false, - "title_cell": "Table of Contents", - "title_sidebar": "Contents", - "toc_cell": false, - "toc_position": {}, - "toc_section_display": true, - "toc_window_display": true - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/terra-jupyter-aou/tests/aou_smoke_test.py b/terra-jupyter-aou/tests/aou_smoke_test.py deleted file mode 100644 index 694b3c9d..00000000 --- a/terra-jupyter-aou/tests/aou_smoke_test.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Brief smoke tests for terra-jupyter-python. - -Minimal tests simply import packages and run 'hello world' code to determine whether any exceptions -are thrown. - -TODO(all): Add additional tests and/or tests with particular assertions, as we encounter Python package version -incompatibilities not currently detected by these tests. -""" - -import os -import pytest - -def test_terra_widgets(): - import os - os.system("pip3 list") - from terra_widgets.html_snapshots import display_html_snapshots_widget diff --git a/terra-jupyter-hail/Dockerfile b/terra-jupyter-hail/Dockerfile index 5a18ec4e..084a8237 100644 --- a/terra-jupyter-hail/Dockerfile +++ b/terra-jupyter-hail/Dockerfile @@ -3,7 +3,6 @@ FROM us.gcr.io/broad-dsp-gcr-public/terra-jupyter-python:1.1.6 USER root -# If anything below this is updated, make the corresponding changes in the aou image ENV PIP_USER=false ENV PYTHONPATH=$PYTHONPATH:/usr/lib/spark/python ENV PYSPARK_PYTHON=python3 diff --git a/terra-jupyter-python/tests/smoke_test.py b/terra-jupyter-python/tests/smoke_test.py index 55ded191..35854432 100644 --- a/terra-jupyter-python/tests/smoke_test.py +++ b/terra-jupyter-python/tests/smoke_test.py @@ -43,8 +43,8 @@ def test_pandas(): def test_tensorflow_gfile(): - tf.io.gfile.copy(src='gs://genomics-public-data/1000-genomes/other/sample_info/sample_info.csv', - dst='/tmp/genomics-public-data-1000-genomes-sample_info.csv', + tf.io.gfile.copy(src='gs://gcp-public-data--broad-references/refdisk_manifest.json', + dst='/tmp/refdisk_manifest.json', overwrite=True) def test_tensorflow_hello_world(): @@ -84,4 +84,4 @@ def test_pysam(): # Call samtools to sort the file # This will fail if the file is not a valid BAM file pysam.sort("-o", "sorted.bam", file_name) - assert os.path.isfile("sorted.bam") \ No newline at end of file + assert os.path.isfile("sorted.bam") diff --git a/terra-rstudio-aou/CHANGELOG.md b/terra-rstudio-aou/CHANGELOG.md deleted file mode 100644 index 3a3b4bf3..00000000 --- a/terra-rstudio-aou/CHANGELOG.md +++ /dev/null @@ -1,9 +0,0 @@ -# Changelog - -## 0.1.1 - 2023-03-16 - -* Install Wondershaper - -## 3.16.0 - 2023-01-29 - -* First release diff --git a/terra-rstudio-aou/Dockerfile b/terra-rstudio-aou/Dockerfile deleted file mode 100644 index 925fadeb..00000000 --- a/terra-rstudio-aou/Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -# Base this image on the Terra AnVIL RStudio-Bioconductor image. -FROM us.gcr.io/broad-dsp-gcr-public/anvil-rstudio-bioconductor:3.16.0 - -ENV AOU_DOCKER_VERSION=0.1.1 -LABEL vendor="The All Of Us Research Program" \ - description="RStudio + Bioconductor docker image for use in the All of Us Researcher Workbench." \ - license="GPL-2.0-or-later" \ - version=$AOU_DOCKER_VERSION \ - url="https://github.com/DataBiosphere/terra-docker" - -# BUG: Update broken google-cloud GPG key preventing -# apt-get update to complete -RUN curl -O https://packages.cloud.google.com/apt/doc/apt-key.gpg \ - && apt-key add apt-key.gpg - -# DEBUG: install speedtest-cli to ascertain tc bandwidth throttling is working -RUN apt-get update \ - && apt-get install -yq --no-install-recommends \ - iproute2 \ - speedtest-cli \ - && echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections \ - && echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections \ - && apt-get -y install iptables-persistent \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Compile from source during image construction to prevent loading -# packages that have not been updated recently. -ENV BIOCONDUCTOR_USE_CONTAINER_REPOSITORY=FALSE - -# Install tidyverse -COPY rstudio/scripts/install_tidyverse.sh /rocker_scripts/install_tidyverse.sh -RUN chmod +x /rocker_scripts/install_tidyverse.sh && /rocker_scripts/install_tidyverse.sh - -# Install top-25 most common packages -# This step is very slow and we want to cache it as much as possible to prevent -# users from having a bad experience. -COPY rstudio/scripts/install_bioconductor.R /rocker_scripts/install_bioconductor.R -RUN R -f /rocker_scripts/install_bioconductor.R - -# Use binary repos to speed up package installation. -ENV BIOCONDUCTOR_USE_CONTAINER_REPOSITORY=TRUE - -# Add our rserver configuration -ADD rstudio/rserver.conf $RSTUDIO_HOME/rserver.conf - -# Install Wondershaper from source, for client-side egress limiting. -RUN cd /usr/local/share && \ - git clone https://github.com/magnific0/wondershaper.git --depth 1 && \ - cd wondershaper && \ - make install && \ - cd $HOME - -# Expose ports 8787 and disable authentication as this is handled -# internally. Root access/super powers are disabled. -ENV RSTUDIO_PORT 8787 -ENV DISABLE_AUTH=true -EXPOSE $RSTUDIO_PORT - -# Hide tc etc from users -RUN chmod 700 -R /usr/sbin/ - -COPY rstudio/entrypoint.sh /init_aou -RUN chmod +x /init_aou -ENTRYPOINT [ "/init_aou" ] diff --git a/terra-rstudio-aou/README.md b/terra-rstudio-aou/README.md deleted file mode 100644 index cc95ac25..00000000 --- a/terra-rstudio-aou/README.md +++ /dev/null @@ -1,19 +0,0 @@ -# terra-rstudio-aou image - -This repo contains the terra-rstudio-aou image that is compatible with the notebook service in [Terra]("https://app.terra.bio/") called Leonardo. For example, use `us.gcr.io/broad-dsp-gcr-public/terra-rstudio-aou:{version}` in Terra. - -This image is intended to support usage in the [All of Us Workbench](https://github.com/all-of-us/workbench). - -## Image contents - -The terra-rstudio-aou extends the [AnVIL Bioconductor image](https://github.com/anvilproject/anvil-docker) by including the following: - -- Configuring `wondershaper` to limit network bandwidth. - -To see the complete contents of this image please see the [Dockerfile](./Dockerfile). - -## Selecting prior versions of this image - -To select an older version this image, you can search the [CHANGELOG.md](./CHANGELOG.md) for a specific package version you need. - -Once you find an image version that you want, copy and paste the image url from the changelog into the corresponding custom Docker field in the Terra notebook runtime widget. diff --git a/terra-rstudio-aou/VERSION b/terra-rstudio-aou/VERSION deleted file mode 100644 index 6da28dde..00000000 --- a/terra-rstudio-aou/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.1.1 \ No newline at end of file diff --git a/terra-rstudio-aou/rstudio/entrypoint.sh b/terra-rstudio-aou/rstudio/entrypoint.sh deleted file mode 100644 index 499829d7..00000000 --- a/terra-rstudio-aou/rstudio/entrypoint.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -set -e # catches errors - -pushd /usr/local/share/wondershaper -wondershaper -a "eth0" -u 16384 # kilobits; 16Mib/s (2MiB/s) - -popd - -# Init must be run as root and will internally handle mapping the end-user -# to UID and GID mapping using S6-overlay -# https://github.com/just-containers/s6-overlay -# using S6 -# https://skarnet.org/software/s6/overview.html -exec /init diff --git a/terra-rstudio-aou/rstudio/rserver.conf b/terra-rstudio-aou/rstudio/rserver.conf deleted file mode 100644 index c7d9860e..00000000 --- a/terra-rstudio-aou/rstudio/rserver.conf +++ /dev/null @@ -1,7 +0,0 @@ -# Server Configuration File -# See https://support.rstudio.com/hc/en-us/articles/200552316-Configuring-the-Server - -rsession-which-r=/usr/local/bin/R -auth-none=1 -www-port=8787 -www-address=0.0.0.0 diff --git a/terra-rstudio-aou/rstudio/scripts/install_bioconductor.R b/terra-rstudio-aou/rstudio/scripts/install_bioconductor.R deleted file mode 100644 index e850a2e0..00000000 --- a/terra-rstudio-aou/rstudio/scripts/install_bioconductor.R +++ /dev/null @@ -1,50 +0,0 @@ -options(Ncpus = parallel::detectCores()) - -BiocManager::install(c( - 'AnVIL', - 'DataBiosphere/Ronaldo', - 'shiny', - 'bigrquery', - 'googleCloudStorageR', - 'tidyverse', - 'Seurat', - "SingleCellExperiment", - "GenomicFeatures", - "GenomicAlignments", - "ShortRead", - "DESeq2", - "AnnotationHub", - "ExperimentHub", - "ensembldb", - "scRNAseq", - "scran", - "Rtsne", - 'BiocGenerics', - 'S4Vectors', - 'BiocVersion', - 'GenomeInfoDb', - 'IRanges', - 'Biobase', - 'zlibbioc', - 'Biostrings', - 'BiocParallel', - 'GenomicRanges', - 'DelayedArray', - 'SummarizedExperiment', - 'MatrixGenerics', - 'limma', - 'AnnotationDbi', - 'KEGGREST', - 'annotate', - 'BiocFileCache', - 'genefilter', - 'Rhtslib', - 'Rsamtools', - 'edgeR', - 'biomaRt', - 'geneplotter', - 'ggtree', - 'rhdf5filters', - 'fgsea', - 'Rgraphviz' - )) diff --git a/terra-rstudio-aou/rstudio/scripts/install_tidyverse.sh b/terra-rstudio-aou/rstudio/scripts/install_tidyverse.sh deleted file mode 100644 index 4281290f..00000000 --- a/terra-rstudio-aou/rstudio/scripts/install_tidyverse.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -set -e - -## build ARGs -NCPUS=${NCPUS:--1} - -# a function to install apt packages only if they are not installed -function apt_install() { - if ! dpkg -s "$@" >/dev/null 2>&1; then - if [ "$(find /var/lib/apt/lists/* | wc -l)" = "0" ]; then - apt-get update - fi - apt-get install -y --no-install-recommends "$@" - fi -} - -apt_install \ - libxml2-dev \ - libcairo2-dev \ - libgit2-dev \ - default-libmysqlclient-dev \ - libpq-dev \ - libsasl2-dev \ - libsqlite3-dev \ - libssh2-1-dev \ - libxtst6 \ - libcurl4-openssl-dev \ - libharfbuzz-dev \ - libfribidi-dev \ - libfreetype6-dev \ - libpng-dev \ - libtiff5-dev \ - libjpeg-dev \ - unixodbc-dev - -install2.r --error --skipinstalled -n "$NCPUS" \ - tidyverse \ - devtools \ - rmarkdown \ - BiocManager \ - vroom \ - gert - -## dplyr database backends -install2.r --error --skipmissing --skipinstalled -n "$NCPUS" \ - arrow \ - dbplyr \ - DBI \ - dtplyr \ - duckdb \ - nycflights13 \ - Lahman \ - RMariaDB \ - RPostgres \ - RSQLite \ - fst - -## a bridge to far? -- brings in another 60 packages -# install2.r --error --skipinstalled -n "$NCPUS" tidymodels - -# Clean up -rm -rf /var/lib/apt/lists/* -rm -rf /tmp/downloaded_packages - -## Strip binary installed lybraries from RSPM -## https://github.com/rocker-org/rocker-versioned2/issues/340 -strip /usr/local/lib/R/site-library/*/libs/*.so - -# Check the tidyverse core packages' version -echo -e "Check the tidyverse package...\n" - -R -q -e "library(tidyverse)" - -echo -e "\nInstall tidyverse package, done!" diff --git a/terra-rstudio-aou/rstudio/tests/smoke_test.R b/terra-rstudio-aou/rstudio/tests/smoke_test.R deleted file mode 100755 index 8790327e..00000000 --- a/terra-rstudio-aou/rstudio/tests/smoke_test.R +++ /dev/null @@ -1,73 +0,0 @@ -#! /usr/bin/env Rscript - -BiocManager::install("assertthat") -require(assertthat) -BiocManager::version() -BiocManager::install("rsbml") -library(rsbml) -BiocManager::install("RCurl") -library(RCurl) -library("ShortRead") -library("GenomicAlignments") -library("GenomicFeatures") -find.package("ShortRead") -BiocManager::install("XML") -library(XML) -BiocManager::install("EBImage") -library(EBImage) -BiocManager::install("RMySQL") -library(RMySQL) -BiocManager::install("DirichletMultinomial") -library(DirichletMultinomial) -BiocManager::install("rjags") -library("rjags") -BiocManager::install("testit") -BiocManager::install("protolite") -library(protolite) - -assert_that("AnVIL" %in% installed.packages() == "TRUE") -assert_that("Ronaldo" %in% installed.packages() == "TRUE") -assert_that("shiny" %in% installed.packages() == "TRUE") -assert_that("bigrquery" %in% installed.packages() == "TRUE") -assert_that("googleCloudStorageR" %in% installed.packages() == "TRUE") -assert_that("tidyverse" %in% installed.packages() == "TRUE") -assert_that("Seurat" %in% installed.packages() == "TRUE") -assert_that("SingleCellExperiment" %in% installed.packages() == "TRUE") -assert_that("GenomicFeatures" %in% installed.packages() == "TRUE") -assert_that("GenomicAlignments" %in% installed.packages() == "TRUE") -assert_that("ShortRead" %in% installed.packages() == "TRUE") -assert_that("DESeq2" %in% installed.packages() == "TRUE") -assert_that("AnnotationHub" %in% installed.packages() == "TRUE") -assert_that("ExperimentHub" %in% installed.packages() == "TRUE") -assert_that("ensembldb" %in% installed.packages() == "TRUE") -assert_that("scRNAseq" %in% installed.packages() == "TRUE") -assert_that("scran" %in% installed.packages() == "TRUE") -assert_that("Rtsne" %in% installed.packages() == "TRUE") -assert_that("BiocGenerics" %in% installed.packages() == "TRUE") -assert_that("S4Vectors" %in% installed.packages() == "TRUE") -assert_that("BiocVersion" %in% installed.packages() == "TRUE") -assert_that("GenomeInfoDb" %in% installed.packages() == "TRUE") -assert_that("IRanges" %in% installed.packages() == "TRUE") -assert_that("Biobase" %in% installed.packages() == "TRUE") -assert_that("zlibbioc" %in% installed.packages() == "TRUE") -assert_that("Biostrings" %in% installed.packages() == "TRUE") -assert_that("BiocParallel" %in% installed.packages() == "TRUE") -assert_that("GenomicRanges" %in% installed.packages() == "TRUE") -assert_that("DelayedArray" %in% installed.packages() == "TRUE") -assert_that("SummarizedExperiment" %in% installed.packages() == "TRUE") -assert_that("MatrixGenerics" %in% installed.packages() == "TRUE") -assert_that("limma" %in% installed.packages() == "TRUE") -assert_that("AnnotationDbi" %in% installed.packages() == "TRUE") -assert_that("KEGGREST" %in% installed.packages() == "TRUE") -assert_that("annotate" %in% installed.packages() == "TRUE") -assert_that("BiocFileCache" %in% installed.packages() == "TRUE") -assert_that("genefilter" %in% installed.packages() == "TRUE") -assert_that("Rhtslib" %in% installed.packages() == "TRUE") -assert_that("Rsamtools" %in% installed.packages() == "TRUE") -assert_that("edgeR" %in% installed.packages() == "TRUE") -assert_that("biomaRt" %in% installed.packages() == "TRUE") -assert_that("geneplotter" %in% installed.packages() == "TRUE") -assert_that("ggtree" %in% installed.packages() == "TRUE") -assert_that("rhdf5filters" %in% installed.packages() == "TRUE") -assert_that("fgsea" %in% installed.packages() == "TRUE") -assert_that("Rgraphviz" %in% installed.packages() == "TRUE") diff --git a/updateVersions.sc b/updateVersions.sc index ef0db3da..0d4a7d73 100755 --- a/updateVersions.sc +++ b/updateVersions.sc @@ -76,19 +76,16 @@ def main(updatedImage: String, updatedImageReleaseNote: String, bumpMajorVersion "terra-jupyter-python", "terra-jupyter-r", "terra-jupyter-gatk", - "terra-jupyter-aou", ) case "terra-jupyter-r" => List( "terra-jupyter-r", "terra-jupyter-bioconductor", "terra-jupyter-gatk", - "terra-jupyter-aou", ) case "terra-jupyter-python" => List( "terra-jupyter-python", "terra-jupyter-hail", "terra-jupyter-gatk", - "terra-jupyter-aou", ) case "terra-jupyter-hail" => List( "terra-jupyter-hail" @@ -96,9 +93,6 @@ def main(updatedImage: String, updatedImageReleaseNote: String, bumpMajorVersion case "terra-jupyter-gatk" => List( "terra-jupyter-gatk" ) - case "terra-jupyter-aou" => List( - "terra-jupyter-aou" - ) case "terra-base" => List( "terra-base" ) diff --git a/wondershaper/Dockerfile b/wondershaper/Dockerfile deleted file mode 100644 index 30b01c33..00000000 --- a/wondershaper/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:latest - -# Install dependencies -RUN apt-get update && apt-get install -y \ - iproute2 \ - ethtool \ - wget \ - unzip \ - build-essential - -# Install Wondershaper -RUN apt-get install -y wget \ - && wget https://github.com/magnific0/wondershaper/archive/master.zip \ - && unzip master.zip \ - && rm master.zip \ - && cd wondershaper-master \ - && make install - -# Set up entrypoint -COPY entrypoint.sh /entrypoint.sh -RUN chmod +x /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/wondershaper/README.md b/wondershaper/README.md deleted file mode 100644 index f3ab6e5f..00000000 --- a/wondershaper/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Dockerfile and Entrypoint for Wondershaper -This Dockerfile and associated entrypoint script provide a way to set up and run Wondershaper, a network traffic shaping tool, within a Docker container. The Docker image is already published and available at us.gcr.io/wondershaper/wondershaper:latest. It is not necessary to republish the image unless modifications are required. - -## Dockerfile -The Dockerfile sets up an Ubuntu-based environment, installs the required dependencies, and installs Wondershaper from the official GitHub repository. It also includes an entrypoint script that configures the network interface and applies bandwidth limits using Wondershaper. - -To build the Docker image locally, use the following command: - -``` -docker build -t wondershaper-local-image . -``` - -To run the local image, use: -``` -docker run --privileged --network=host wondershaper-local-image -``` - -## Entrypoint Script (entrypoint.sh) -The entrypoint script (entrypoint.sh) is copied into the Docker image. It performs the following tasks: - -Sets the network interface (eth0) and the desired upload bandwidth limit (in kilobits per second). -Applies the bandwidth limits to the network interface using Wondershaper. -Keeps the container running by executing the tail -f /dev/null command. - -## Running the Docker Image -To run the Wondershaper container using the published image (us.gcr.io/wondershaper/wondershaper:latest), execute the following command: - -``` -docker run --name wondershaper-container us.gcr.io/broad-dsp-gcr-public/wondershaper:latest -``` - -Please note that the image is already published, so you don't need to perform any additional steps unless you want to modify the Dockerfile or the entrypoint script. - - -Remember to replace wondershaper-container with your preferred container name. - - - diff --git a/wondershaper/entrypoint.sh b/wondershaper/entrypoint.sh deleted file mode 100644 index 8c035aa4..00000000 --- a/wondershaper/entrypoint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -# Network interface and bandwidth values -INTERFACE="eth0" -UPLOAD_BANDWIDTH="16384" # kilobits; 16Mib/s (2MiB/s) - -# Apply bandwidth limits using Wondershaper -wondershaper -a $INTERFACE -u $UPLOAD_BANDWIDTH - -# Keep the container running -tail -f /dev/null - -#!/bin/bash