From 9c303572af6f41fc772ddd5db44000ccf2ecc622 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 13 Aug 2026 10:34:35 -0500 Subject: [PATCH 1/7] chore: uv support --- .github/dependabot.yml | 41 ++ .github/workflows/ci.yaml | 43 +- .github/workflows/pr.yml | 24 +- .github/workflows/promote.yml | 9 +- .github/workflows/promotion-checker.yml | 2 +- .markdownlint-cli2.jsonc | 8 - .pre-commit-config.yaml | 30 +- CODEOWNERS | 6 +- README.md | 2 +- pyproject.toml | 53 +++ requirements.in | 6 - requirements.txt | 123 ----- scripts/generate_mdx.py | 31 +- scripts/promote_collection.py | 16 +- scripts/promote_dataset.py | 34 +- scripts/requirements.txt | 1 - .../Transfer-MCP-veda-data-store.ipynb | 52 ++- .../generate-cmip6-kerchunk-historical.ipynb | 43 +- .../publish-cmip6-kerchunk-stac.ipynb | 10 +- .../collection-and-item-ingest.ipynb | 26 +- .../collection-metadata-reconciliation.ipynb | 44 +- .../create-stac-for-cmr-collections.ipynb | 24 +- .../fwi-stations/FWI_Transformation.ipynb | 8 +- .../generate-rasi-cogs-from-netcdf3.ipynb | 26 +- .../publish-hls-provided-item-metadata.ipynb | 27 +- .../landsat-lakes-discovery/aral-sea.ipynb | 33 +- .../lake-balaton.ipynb | 43 +- .../landsat-lakes-discovery/lake-biwa.ipynb | 45 +- .../mirror-landsat-demos.ipynb | 43 +- .../landsat-lakes-discovery/tonle-sap.ipynb | 45 +- .../landsat-lakes-discovery/vanern.ipynb | 43 +- .../cmip6-generate-organize-cogs.ipynb | 128 +++--- .../cmip6-generate-stac.ipynb | 6 +- .../cmip6_file_organization.py | 7 +- .../soil_moisture_transformation_cog.ipynb | 15 +- transformation-scripts/update-hrefs.ipynb | 10 +- uv.lock | 419 ++++++++++++++++++ 37 files changed, 1001 insertions(+), 525 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .markdownlint-cli2.jsonc create mode 100644 pyproject.toml delete mode 100644 requirements.in delete mode 100644 requirements.txt delete mode 100644 scripts/requirements.txt create mode 100644 uv.lock diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..415eedec --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,41 @@ +version: 2 +updates: + - package-ecosystem: uv + directory: / + cooldown: + default-days: 7 + semver-major-days: 30 + schedule: + interval: weekly + groups: + python-dependencies: + patterns: + - "*" + commit-message: + prefix: "chore(deps):" + + - package-ecosystem: github-actions + directory: / + cooldown: + default-days: 7 + schedule: + interval: weekly + groups: + github-actions: + patterns: + - "*" + commit-message: + prefix: "ci(deps):" + + - package-ecosystem: pre-commit + directory: / + cooldown: + default-days: 7 + schedule: + interval: weekly + groups: + pre-commit: + patterns: + - "*" + commit-message: + prefix: "ci(deps):" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8522d212..119a9a77 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -7,31 +7,24 @@ on: pull_request: jobs: - test: + ci: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - name: test + run: uv run --group dev pytest + - name: lint + run: uv run --group dev pre-commit run --all-files steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Setup Python - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 - with: - python-version: "3.11" - cache: "pip" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Test collections - run: pytest - lint: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - - name: Setup Python - uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0 + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - python-version: "3.11" - cache: "pip" - - name: Install dependencies - run: pip install -r requirements.txt - - name: Run pre-commit hooks - run: pre-commit run --all-files + enable-cache: true + + - name: ${{ matrix.name }} + run: ${{ matrix.run }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 174da7f6..3f3034cf 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -35,7 +35,7 @@ jobs: comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }} has_new_files_to_promote: ${{ steps.changed-files.outputs.all_changed_files_count != 0 }} steps: - - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 @@ -116,7 +116,7 @@ jobs: STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }} DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }} run: | - pip install -r ./scripts/requirements.txt + uv sync --group scripts # Track successful publications all_failed=true @@ -180,14 +180,10 @@ jobs: $status_message" gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY" - - name: Set up Python - uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 with: - python-version: '3.9' - - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2 - with: - path: ${{ env.pythonLocation }} - key: ${{ env.pythonLocation }}-pip-${{ hashFiles('requirements.txt') }} + enable-cache: true # If the workflow fails at any point, the PR comment will be updated - name: Update PR comment on overall workflow failure @@ -233,7 +229,7 @@ jobs: run: ls -lah /tmp/upload-artifacts - name: Upload published collections files as directory artifact - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 env: ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }} with: @@ -247,7 +243,7 @@ jobs: echo "${{ github.run_id }}" > run_id.txt - name: Upload Workflow Run ID - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 env: ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }} with: @@ -272,7 +268,7 @@ jobs: skip_mdx_pr_step: ${{ steps.skip_mdx_pr_step_setting.outputs.skip_mdx_pr_step }} steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Set SKIP_MDX_PR_STEP condition id: skip_mdx_pr_step_setting @@ -289,7 +285,7 @@ jobs: needs: ['mdx-pr-flag-evaluator', 'publish-new-datasets'] steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Use output from publish-new-datasets run: | @@ -302,7 +298,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES collection_ids="" - pip install -r ./scripts/requirements.txt + uv sync --group scripts for file in ${PUBLISHED_COLLECTION_FILES}; do collection_id=$(python3 ./scripts/generate_mdx.py "$file") collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//') diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 7bd3641e..af9aadc3 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -45,7 +45,12 @@ jobs: environment: production steps: - name: Checkout code - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: true - name: Determine artifact name run: echo "ARTIFACT_NAME=collections-to-promote-from-${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV @@ -67,7 +72,7 @@ jobs: PROMOTION_DAG: ${{ vars.PROMOTION_DAG_NAME }} run: | - pip install -r ./scripts/requirements.txt + uv sync --group scripts for file in downloaded-files/*.json; do echo "Processing $file" # Check if collection key exists to determine if it's a dataset or collection diff --git a/.github/workflows/promotion-checker.yml b/.github/workflows/promotion-checker.yml index 1da3c698..bdcf5a89 100644 --- a/.github/workflows/promotion-checker.yml +++ b/.github/workflows/promotion-checker.yml @@ -53,7 +53,7 @@ jobs: needs: check-conditions steps: - name: Checkout Repository - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - name: Get Workflow Run ID Artifact run: | diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc deleted file mode 100644 index e5a00e2c..00000000 --- a/.markdownlint-cli2.jsonc +++ /dev/null @@ -1,8 +0,0 @@ -{ - "config": { - "MD013": false, // disable line length checks - "MD033": { - "allowed_elements": [ "a", "b", "details", "summary" ] - } - } -} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9f3482e7..284a8e63 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,22 +1,26 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v6.0.0 hooks: - id: check-yaml - - id: file-contents-sorter - files: ^requirements.in$ - id: trailing-whitespace exclude: ^ingestion-data/.*$ + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.16.1 + hooks: + - id: ruff-check + args: [--fix] + - id: ruff-format + types_or: [python, pyi, jupyter, pyproject] + - repo: https://github.com/DavidAnson/markdownlint-cli2 - rev: v0.10.0 + rev: v0.23.2 hooks: - id: markdownlint-cli2 - - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.3.0 - hooks: - - id: black - - id: black-jupyter - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.0.292 - hooks: - - id: ruff + args: + - "--fix" + - "--config" + - "pyproject.toml" + - "--configPointer" + - "/tool/markdownlint-cli2" diff --git a/CODEOWNERS b/CODEOWNERS index 296e5839..ebb5e4f4 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,5 +1 @@ -* @anayeaye -* @gadomski -* @ividito -* @slesaad -* @smohiudd +* @NASA-IMPACT/veda-data-services diff --git a/README.md b/README.md index 3c6cb4ce..1cfba3a4 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ To add data to VEDA you will: ### Step 1: Stage your files -Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request access [here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html) +Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request [access here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html) ### Step 2: Generate STAC metadata in the staging catalog diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..5ea46ef2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,53 @@ +[project] +name = "veda-data" +version = "0.1.0" +description = "Config data used to load datasets into the VEDA catalog" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [] + +[dependency-groups] +dev = [ + "pre-commit==4.6.1", + "pystac[validation]==1.8.4", + "pytest==7.4.2", +] +scripts = [ + "pyyaml==6.0.1", +] + +[tool.uv] +default-groups = ["dev"] + +[tool.ruff] +line-length = 88 + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # Pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "PTH", # flake8-use-pathlib + "RET", # flake8-return + "TID", # flake8-tidy-imports + "TC", # flake8-type-checking + "ISC", # flake8-implicit-str-concat + "ICN", # flake8-import-conventions + "BLE", # flake8-blind-except +] +ignore = [ + "BLE001", # Blind Exceptions +] + +[tool.ruff.format] +docstring-code-format = true +docstring-code-line-length = "dynamic" + +[tool.markdownlint-cli2.config] +MD013 = false +MD033 = { allowed_elements = ["a", "b", "details", "summary"] } diff --git a/requirements.in b/requirements.in deleted file mode 100644 index 8f9b6370..00000000 --- a/requirements.in +++ /dev/null @@ -1,6 +0,0 @@ -black[jupyter] -pip-tools -pre-commit -pystac[validation] -pytest -ruff diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 64c91392..00000000 --- a/requirements.txt +++ /dev/null @@ -1,123 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.11 -# by the following command: -# -# pip-compile -# -asttokens==2.4.0 - # via stack-data -attrs==23.1.0 - # via - # jsonschema - # referencing -backcall==0.2.0 - # via ipython -black[jupyter]==23.10.0 - # via -r requirements.in -build==1.0.3 - # via pip-tools -cfgv==3.4.0 - # via pre-commit -click==8.1.7 - # via - # black - # pip-tools -decorator==5.1.1 - # via ipython -distlib==0.3.7 - # via virtualenv -executing==2.0.0 - # via stack-data -filelock==3.12.4 - # via virtualenv -identify==2.5.30 - # via pre-commit -iniconfig==2.0.0 - # via pytest -ipython==8.16.1 - # via black -jedi==0.19.1 - # via ipython -jsonschema==4.19.1 - # via pystac -jsonschema-specifications==2023.7.1 - # via jsonschema -matplotlib-inline==0.1.6 - # via ipython -mypy-extensions==1.0.0 - # via black -nodeenv==1.8.0 - # via pre-commit -packaging==23.2 - # via - # black - # build - # pytest -parso==0.8.3 - # via jedi -pathspec==0.11.2 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -pip-tools==7.3.0 - # via -r requirements.in -platformdirs==3.11.0 - # via - # black - # virtualenv -pluggy==1.3.0 - # via pytest -pre-commit==3.5.0 - # via -r requirements.in -prompt-toolkit==3.0.39 - # via ipython -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.2 - # via stack-data -pygments==2.16.1 - # via ipython -pyproject-hooks==1.0.0 - # via build -pystac[validation]==1.8.4 - # via -r requirements.in -pytest==7.4.2 - # via -r requirements.in -python-dateutil==2.8.2 - # via pystac -pyyaml==6.0.1 - # via pre-commit -referencing==0.30.2 - # via - # jsonschema - # jsonschema-specifications -rpds-py==0.10.6 - # via - # jsonschema - # referencing -ruff==0.1.0 - # via -r requirements.in -six==1.16.0 - # via - # asttokens - # python-dateutil -stack-data==0.6.3 - # via ipython -tokenize-rt==5.2.0 - # via black -traitlets==5.11.2 - # via - # ipython - # matplotlib-inline -virtualenv==20.26.6 - # via pre-commit -wcwidth==0.2.8 - # via prompt-toolkit -wheel==0.41.2 - # via pip-tools - -# The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools diff --git a/scripts/generate_mdx.py b/scripts/generate_mdx.py index e8726d53..4db953bf 100644 --- a/scripts/generate_mdx.py +++ b/scripts/generate_mdx.py @@ -5,10 +5,11 @@ Dependency: `dataset.mdx` file """ -import yaml -import os import json import sys +from pathlib import Path + +import yaml def create_frontmatter(input_data): @@ -91,39 +92,37 @@ def create_frontmatter(input_data): json_data["layers"].append(layer) # Convert json to yaml for frontmatter - yaml_data = yaml.dump(json_data, sort_keys=False) - - return yaml_data + return yaml.dump(json_data, sort_keys=False) def safe_open_w(path): """Open "path" for writing, creating any parent directories as needed.""" - os.makedirs(os.path.dirname(path), exist_ok=True) - return open(path, "w") + Path(path).parent.mkdir(parents=True, exist_ok=True) + return Path(path).open("w") if __name__ == "__main__": - input_data = json.load(open(sys.argv[1])) + with Path(sys.argv[1]).open() as f: + input_data = json.load(f) dataset_config = create_frontmatter(input_data) front_matter = f"---\n{dataset_config}---\n" # Path to the existing file - curr_directory = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(curr_directory, "dataset.mdx") + curr_directory = Path(__file__).parent + file_path = curr_directory / "dataset.mdx" # Read the existing content of the file - with open(file_path, "r") as file: - existing_content = file.read() + existing_content = file_path.read_text() # Combine front matter and existing content new_content = front_matter + existing_content # Write the combined content back to the file - output_filepath = os.path.join( - curr_directory, - f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx", + output_filepath = ( + Path(curr_directory) + / f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx" ) - with safe_open_w(output_filepath) as ofile: + with safe_open_w(str(output_filepath)) as ofile: ofile.write(new_content) collection_id = input_data["collection"] diff --git a/scripts/promote_collection.py b/scripts/promote_collection.py index abc26e69..2cf7ebfb 100644 --- a/scripts/promote_collection.py +++ b/scripts/promote_collection.py @@ -1,14 +1,14 @@ -from typing import Dict, Any - import http.client import json -import sys import os +import sys import uuid from base64 import b64encode +from pathlib import Path +from typing import Any -def trigger_collection_dag(payload: Dict[str, Any], stage: str): +def trigger_collection_dag(payload: dict[str, Any], stage: str): """ Triggers the veda_collection_pipeline DAG in either staging or production SM2A. """ @@ -43,7 +43,7 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): headers = { "Content-Type": "application/json", - "Authorization": "Basic " + api_token, + "Authorization": f"Basic {api_token}", } body = { @@ -66,7 +66,7 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): if __name__ == "__main__": try: - with open(sys.argv[1], "r") as file: + with Path(sys.argv[1]).open() as file: input_data = json.load(file) stage = sys.argv[2] dag_payload = {"conf": input_data} @@ -76,5 +76,5 @@ def trigger_collection_dag(payload: Dict[str, Any], stage: str): print("Usage: promote_collection.py ") except FileNotFoundError: print(f"Error: File '{sys.argv[1]}' not found.") - except json.JSONDecodeError: - raise ValueError(f"Invalid JSON content in file {sys.argv[1]}") + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON content in file {sys.argv[1]}") from e diff --git a/scripts/promote_dataset.py b/scripts/promote_dataset.py index e2ea38fd..2714d526 100644 --- a/scripts/promote_dataset.py +++ b/scripts/promote_dataset.py @@ -1,18 +1,18 @@ -from typing import Dict, Any - import http.client import json -import sys import os +import sys import uuid from base64 import b64encode +from pathlib import Path +from typing import Any class MissingFieldError(Exception): pass -def validate_discovery_item_config(item: Dict[str, Any]) -> Dict[str, Any]: +def validate_discovery_item_config(item: dict[str, Any]) -> dict[str, Any]: if "bucket" not in item: raise MissingFieldError( "Missing required field 'bucket' in discovery item: {item}" @@ -43,13 +43,13 @@ def publish_to_staging(payload): if not base_api_url or not api_token: raise ValueError( "STAGING_SM2A_API_URL or STAGING_SM2A_ADMIN_USERNAME" - + " or STAGING_SM2A_ADMIN_PASSWORD is not" - + " set in the environment variables." + " or STAGING_SM2A_ADMIN_PASSWORD is not" + " set in the environment variables." ) headers = { "Content-Type": "application/json", - "Authorization": "Basic " + api_token, + "Authorization": f"Basic {api_token}", } body = { @@ -83,13 +83,13 @@ def promote_to_production(payload): if not base_api_url or not api_token: raise ValueError( - "SM2A_API_URL or SM2A_ADMIN_USERNAME or SM2A_ADMIN_PASSWORD is not" - + " set in the environment variables." + "SM2A_API_URL or SM2A_ADMIN_USERNAME or SM2A_ADMIN_PASSWORD" + " is not set in the environment variables." ) headers = { "Content-Type": "application/json", - "Authorization": "Basic " + api_token, + "Authorization": f"Basic {api_token}", } payload["conf"]["transfer"] = True @@ -113,22 +113,22 @@ def promote_to_production(payload): if __name__ == "__main__": try: - with open(sys.argv[1], "r") as file: - input = json.load(file) + with Path(sys.argv[1]).open() as file: + _input = json.load(file) stage = sys.argv[2] - discovery_items = input.get("discovery_items") + discovery_items = _input.get("discovery_items") validated_discovery_items = [ validate_discovery_item_config(item) for item in discovery_items ] - dag_payload = {"conf": input} + dag_payload = {"conf": _input} if stage == "production": promote_to_production(dag_payload) elif stage == "staging": publish_to_staging(dag_payload) except IndexError: - print("Usage: promote_collection.py ") + print("Usage: promote_dataset.py ") except FileNotFoundError: print(f"Error: File '{sys.argv[1]}' not found.") - except json.JSONDecodeError: - raise ValueError(f"Invalid JSON content in file {sys.argv[1]}") + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON content in file {sys.argv[1]}") from e diff --git a/scripts/requirements.txt b/scripts/requirements.txt deleted file mode 100644 index 4818cc54..00000000 --- a/scripts/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -pyyaml \ No newline at end of file diff --git a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb index 0c5cbdbb..34bdf207 100644 --- a/transformation-scripts/Transfer-MCP-veda-data-store.ipynb +++ b/transformation-scripts/Transfer-MCP-veda-data-store.ipynb @@ -2,29 +2,34 @@ "cells": [ { "cell_type": "code", - "execution_count": 105, + "execution_count": null, "id": "0a619c84-a55d-4f93-99e0-c31a42d52932", "metadata": {}, "outputs": [], "source": [ - "from pystac_client import Client\n", + "import base64\n", "import json\n", - "import glob\n", - "import requests\n", + "from pathlib import Path\n", + "\n", "import boto3\n", - "import base64\n", - "import time" + "import requests\n", + "from pystac_client import Client" ] }, { "cell_type": "code", - "execution_count": 34, + "execution_count": null, "id": "e1a873f2-7b72-4245-a3d0-a1a3d38bde97", "metadata": {}, "outputs": [], "source": [ - "discover_items_file_paths = glob.glob(\"../ingestion-data/discovery-items/*.json\")\n", - "# discover_items_ids = [i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths]" + "discover_items_file_paths = list(\n", + " Path(\"../ingestion-data/discovery-items\").glob(\"*.json\")\n", + ")\n", + "\n", + "# discover_items_ids = [\n", + "# i.split(\"/\")[-1].split(\".\")[0] for i in discover_items_file_paths\n", + "# ]" ] }, { @@ -49,7 +54,7 @@ }, { "cell_type": "code", - "execution_count": 119, + "execution_count": null, "id": "a3457a89-e5e3-4873-a524-51a914488b88", "metadata": {}, "outputs": [ @@ -302,7 +307,7 @@ ], "source": [ "for filename in discover_items_file_paths:\n", - " with open(filename, \"r\") as f:\n", + " with Path(filename).open() as f:\n", " dataset = json.load(f)\n", " if not isinstance(dataset, list):\n", " dataset = [dataset]\n", @@ -313,24 +318,24 @@ " if (\n", " i[\"collection\"] in staging_collections\n", " ): # only create config for collections in staging\n", - " config = dict()\n", + " config = {}\n", " config[\"origin_bucket\"] = i.get(\"bucket\", \"\")\n", " config[\"origin_prefix\"] = i.get(\"prefix\", \"\")\n", " config[\"target_bucket\"] = \"veda-data-store\"\n", " config[\"collection\"] = i.get(\"collection\")\n", "\n", " if \"assets\" in i: # conditional for multi asset discovery items\n", - " for k, v in i[\"assets\"].items():\n", + " for _k, v in i[\"assets\"].items():\n", " config[\"filename_regex\"] = v.get(\"regex\")\n", " transfer_config.append(config.copy())\n", " else:\n", " config[\"filename_regex\"] = i.get(\"filename_regex\", \"\")\n", " transfer_config.append(config)\n", "\n", - " with open(\n", - " f\"../ingestion-data/transfer-config/{id}.json\", \"w\", encoding=\"utf-8\"\n", + " with Path(f\"../ingestion-data/transfer-config/{id}.json\").open(\n", + " \"w\", encoding=\"utf-8\"\n", " ) as file:\n", - " print(f\"writing file: ingestion_data/transfer-config/{id}\")\n", + " print(f\"writing file: ../ingestion-data/transfer-config/{id}\")\n", " json.dump(transfer_config, file, ensure_ascii=False, indent=4)" ] }, @@ -376,7 +381,7 @@ }, { "cell_type": "code", - "execution_count": 118, + "execution_count": null, "id": "5a774091-a239-409a-9360-03dec3839ec7", "metadata": {}, "outputs": [ @@ -1448,11 +1453,13 @@ } ], "source": [ - "transfer_items_file_paths = glob.glob(\"../ingestion-data/transfer-config/*.json\")\n", + "transfer_items_file_paths = list(\n", + " Path(\"../ingestion-data/transfer-config\").glob(\"*.json\")\n", + ")\n", "\n", "for transfer_item_file in transfer_items_file_paths:\n", " print(f\"Transferring using config: {transfer_item_file}\")\n", - " with open(transfer_item_file, \"r\") as f:\n", + " with Path(transfer_item_file).open() as f:\n", " transfer_configs = json.load(f)\n", " for transfer_config in transfer_configs:\n", " mwaa_cli_token = get_mwaa_cli_token(\n", @@ -1469,7 +1476,10 @@ " pass\n", " else:\n", " if not dry_run:\n", - " raw_data = f\"dags trigger veda_transfer --conf '{json.dumps(transfer_config)}'\"\n", + " raw_data = (\n", + " \"dags trigger veda_transfer \"\n", + " f\"--conf '{json.dumps(transfer_config)}'\"\n", + " )\n", " mwaa_response = requests.post(\n", " f\"https://{mwaa_cli_token['WebServerHostname']}/aws_mwaa/cli\",\n", " headers={\n", @@ -1488,7 +1498,7 @@ " print(mwaa_response.status_code)\n", " print(f\"stderr: {mwaa_std_err_message}\")\n", " print(f\"stdout: {mwaa_std_out_message}\")\n", - " except:\n", + " except Exception:\n", " print(mwaa_response)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb index 9efd475b..07d65fe8 100644 --- a/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb +++ b/transformation-scripts/cmip6-kerchunk/generate-cmip6-kerchunk-historical.ipynb @@ -16,23 +16,22 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "9f7250a6", "metadata": {}, "outputs": [], "source": [ + "from pathlib import Path\n", "from tempfile import TemporaryDirectory\n", + "\n", "import boto3\n", - "from dask_gateway import GatewayCluster, Gateway\n", "import dask.bag as db\n", "import fsspec\n", - "import json\n", - "import os\n", "import ujson\n", "import xarray as xr\n", + "from dask_gateway import Gateway, GatewayCluster\n", "from kerchunk.combine import MultiZarrToZarr\n", "from kerchunk.hdf import SingleHdf5ToZarr\n", - "from typing import Dict\n", "\n", "# Specify the CMIP model and variable to use.\n", "# Here we are using near-surface air temperature from the GISS-E2-1-G GCM\n", @@ -41,7 +40,8 @@ "# If this code were re-used for a protected bucket, anon should be False.\n", "anon = True\n", "# Note: We are only using the historical data in this example.\n", - "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs) in the s3://nex-gddp-cmip6 bucket.\n", + "# More years of data are available from multiple Shared Socio-Economic Pathways (SSPs)\n", + "# in the s3://nex-gddp-cmip6 bucket.\n", "s3_path = f\"s3://nex-gddp-cmip6/NEX-GDDP-CMIP6/{model}/historical/r1i1p1*/{variable}/*\"" ] }, @@ -119,15 +119,21 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "b046c17c", "metadata": {}, "outputs": [], "source": [ - "so = dict(mode=\"rb\", anon=anon, default_fill_cache=False, default_cache_type=\"first\")\n", + "so = {\n", + " \"mode\": \"rb\",\n", + " \"anon\": anon,\n", + " \"default_fill_cache\": False,\n", + " \"default_cache_type\": \"first\",\n", + "}\n", "\n", "\n", - "# Use Kerchunk's `SingleHdf5ToZarr` method to create a `Kerchunk` index from a NetCDF file.\n", + "# Use Kerchunk's `SingleHdf5ToZarr` method\n", + "# to create a `Kerchunk` index from a NetCDF file.\n", "def generate_json_reference(u):\n", " with fs_read.open(u, **so) as infile:\n", " fname = u.split(\"/\")[-1].strip(\".nc\")\n", @@ -136,8 +142,8 @@ "\n", "\n", "def write_json(fname, reference_json, temp_dir):\n", - " outf = os.path.join(temp_dir, f\"{fname}.json\")\n", - " with open(outf, \"wb\") as f:\n", + " outf = Path(temp_dir) / f\"{fname}.json\"\n", + " with outf.open(\"wb\") as f:\n", " f.write(reference_json)\n", " return outf" ] @@ -188,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "id": "794be824-0833-4229-a8b4-10b071eced26", "metadata": { "tags": [] @@ -258,7 +264,8 @@ "gateway = Gateway()\n", "clusters = gateway.list_clusters()\n", "\n", - "# connect to an existing cluster - this is useful when the kernel shutdown in the middle of an interactive session\n", + "# connect to an existing cluster\n", + "# this is useful when the kernel shutdown in the middle of an interactive session\n", "if clusters:\n", " cluster = gateway.connect(clusters[0].name)\n", "else:\n", @@ -355,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "id": "55ced4a6-0c38-43c8-8c6e-a4fff8a9f7bb", "metadata": {}, "outputs": [ @@ -369,8 +376,8 @@ ], "source": [ "output_fname = f\"combined_CMIP6_daily_{model}_{variable}_kerchunk.json\"\n", - "output_location = os.path.join(temp_dir, output_fname)\n", - "with open(f\"{output_location}\", \"wb\") as f:\n", + "output_location = Path(temp_dir) / output_fname\n", + "with output_location.open(\"wb\") as f:\n", " print(f\"Writing combined kerchunk reference file {output_location}\")\n", " f.write(ujson.dumps(multi_kerchunk).encode())" ] @@ -391,7 +398,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "ac76c0c0", "metadata": {}, "outputs": [ @@ -841,7 +848,7 @@ ], "source": [ "# Check the data\n", - "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs=dict(consolidated=False))\n", + "ds = xr.open_dataset(m, engine=\"zarr\", backend_kwargs={\"consolidated\": False})\n", "display(ds)" ] }, diff --git a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb index 1bd30f3f..4e9e9a0a 100644 --- a/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb +++ b/transformation-scripts/cmip6-kerchunk/publish-cmip6-kerchunk-stac.ipynb @@ -24,7 +24,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "3c3e536f-f61f-4ad7-a0b4-414387d634d1", "metadata": { "tags": [] @@ -32,11 +32,11 @@ "outputs": [], "source": [ "#!pip install xstac\n", - "import pystac\n", - "import s3fs\n", - "import xstac\n", "import fsspec\n", - "import xarray as xr" + "import pystac\n", + "import requests\n", + "import xarray as xr\n", + "import xstac" ] }, { diff --git a/transformation-scripts/collection-and-item-ingest.ipynb b/transformation-scripts/collection-and-item-ingest.ipynb index 6adfb0ac..1800e13d 100644 --- a/transformation-scripts/collection-and-item-ingest.ipynb +++ b/transformation-scripts/collection-and-item-ingest.ipynb @@ -31,8 +31,9 @@ "metadata": {}, "outputs": [], "source": [ - "import glob\n", "import json\n", + "from pathlib import Path\n", + "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -64,7 +65,7 @@ " \"combined_CMIP6_daily_GISS-E2-1-G_tas_kerchunk_DEMO\",\n", "]\n", "\n", - "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", + "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", "filtered_list = [\n", " item\n", " for item in json_file_paths\n", @@ -74,10 +75,16 @@ " )\n", "]\n", "\n", + "\n", + "def load_json(file_path: str | Path) -> dict:\n", + " with Path(file_path).open() as f:\n", + " return json.load(f)\n", + "\n", + "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in filtered_list\n", - " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", + " if \"id\" in (data := load_json(file_path))\n", "]" ] }, @@ -182,15 +189,18 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " f\"Request was successful. Find the updated collection at {collection_url}\"\n", + " \"Request was successful. \"\n", + " f\"Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -241,10 +251,10 @@ " file_path = item[\"filePath\"]\n", "\n", " try:\n", - " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " with Path(file_path).open(encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", - " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", + " # Publish updated collection to the target ingestion `api/collections` endpoint\n", " post_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/collection-metadata-reconciliation.ipynb b/transformation-scripts/collection-metadata-reconciliation.ipynb index 54c5d3d1..d5337495 100644 --- a/transformation-scripts/collection-metadata-reconciliation.ipynb +++ b/transformation-scripts/collection-metadata-reconciliation.ipynb @@ -16,12 +16,13 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "import glob\n", "import json\n", + "from pathlib import Path\n", + "\n", "import requests\n", "from cognito_client import CognitoClient" ] @@ -35,16 +36,22 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "json_file_paths = glob.glob(\"../ingestion-data/collections/*.json\")\n", + "json_file_paths = Path(\"../ingestion-data/collections\").glob(\"*.json\")\n", + "\n", + "\n", + "def load_json(file_path: str | Path) -> dict:\n", + " with Path(file_path).open() as f:\n", + " return json.load(f)\n", + "\n", "\n", "file_paths_and_collection_ids = [\n", " {\"filePath\": file_path, \"collectionId\": data[\"id\"]}\n", " for file_path in json_file_paths\n", - " if \"id\" in (data := json.load(open(file_path, \"r\")))\n", + " if \"id\" in (data := load_json(file_path))\n", "]" ] }, @@ -136,7 +143,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -149,15 +156,18 @@ " response.raise_for_status()\n", " if response.status_code == 201:\n", " print(\n", - " f\"Request was successful. Find the updated collection at {collection_url}\"\n", + " \"Request was successful. \"\n", + " f\"Find the updated collection at {collection_url}\"\n", " )\n", " else:\n", " print(\n", - " f\"Updating {collection_id} failed. Request failed with status code: {response.status_code}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"Request failed with status code: {response.status_code}\"\n", " )\n", " except requests.RequestException as e:\n", " print(\n", - " f\"Updating {collection_id} failed. An error occurred during the request: {e}\"\n", + " f\"Updating {collection_id} failed. \"\n", + " f\"An error occurred during the request: {e}\"\n", " )\n", " except Exception as e:\n", " print(\n", @@ -176,15 +186,14 @@ "\n", "\n", "def retain_external_links(existing_links, retrieved_links):\n", - " unique_hrefs = set(link.get(\"href\") for link in existing_links)\n", + " unique_hrefs = {link.get(\"href\") for link in existing_links}\n", " additional_external_links = [\n", " link\n", " for link in retrieved_links\n", " if link.get(\"rel\") == \"external\" and link.get(\"href\") not in unique_hrefs\n", " ]\n", "\n", - " retained_links = existing_links + additional_external_links\n", - " return retained_links" + " return existing_links + additional_external_links" ] }, { @@ -196,7 +205,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -361,9 +370,9 @@ " collection_id = item[\"collectionId\"]\n", " file_path = item[\"filePath\"]\n", "\n", - " if VEDA_STAC_API == dev_endpoint:\n", + " if dev_endpoint == VEDA_STAC_API:\n", " url = f\"{VEDA_STAC_API}api/stac/collections/{collection_id}\"\n", - " elif VEDA_STAC_API == staging_endpoint:\n", + " elif staging_endpoint == VEDA_STAC_API:\n", " url = f\"{VEDA_STAC_API}collections/{collection_id}\"\n", "\n", " try:\n", @@ -374,7 +383,7 @@ " retrieved_summaries = json_response.get(\"summaries\", {})\n", " retrieved_links = json_response.get(\"links\", {})\n", "\n", - " with open(file_path, \"r\", encoding=\"utf-8\") as file:\n", + " with Path(file_path).open(encoding=\"utf-8\") as file:\n", " collection = json.load(file)\n", "\n", " existing_summaries = collection.get(\"summaries\", {})\n", @@ -385,7 +394,8 @@ " )\n", " collection[\"links\"] = retain_external_links(existing_links, retrieved_links)\n", "\n", - " # Publish the updated collection to the target ingestion `api/collections` endpoint\n", + " # Publish the updated collection to the target ingestion\n", + " # `api/collections` endpoint\n", " post_reconciled_collection(collection, collection_id)\n", "\n", " except requests.RequestException as e:\n", diff --git a/transformation-scripts/create-stac-for-cmr-collections.ipynb b/transformation-scripts/create-stac-for-cmr-collections.ipynb index 30d7ff17..7cf71bcf 100644 --- a/transformation-scripts/create-stac-for-cmr-collections.ipynb +++ b/transformation-scripts/create-stac-for-cmr-collections.ipynb @@ -63,22 +63,22 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "id": "67e37cc9-6022-46c2-aa20-4bb87dd502f9", "metadata": { "tags": [] }, "outputs": [], "source": [ - "import earthaccess\n", "import json\n", - "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "import earthaccess\n", "import pandas as pd\n", "import pystac\n", - "import requests\n", "import s3fs\n", - "import xstac\n", - "import xarray as xr" + "import xarray as xr\n", + "import xstac" ] }, { @@ -98,7 +98,7 @@ }, "outputs": [ { - "name": "stdin", + "name": "stdout", "output_type": "stream", "text": [ "Enter your Earthdata Login username: aimeeb\n", @@ -450,7 +450,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "ae799bbb-06f8-499b-be53-c9ab86762ca7", "metadata": { "tags": [] @@ -458,7 +458,7 @@ "outputs": [], "source": [ "cube_variables = collection.to_dict()[\"cube:variables\"]\n", - "for variable in cube_variables.keys():\n", + "for variable in cube_variables:\n", " cube_variables[variable][\"shape\"][0] = None" ] }, @@ -520,12 +520,14 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "id": "42d3023b-b551-43c1-9984-22553eb41fb1", "metadata": {}, "outputs": [], "source": [ - "with open(f\"../ingestion-data/staging/collections/{collection.id}.json\", \"w+\") as f:\n", + "with Path(f\"../ingestion-data/staging/collections/{collection.id}.json\").open(\n", + " \"w+\"\n", + ") as f:\n", " f.write(json.dumps(collection.to_dict(), indent=2))" ] } diff --git a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb index 0ca3196e..aaf3a1b3 100644 --- a/transformation-scripts/fwi-stations/FWI_Transformation.ipynb +++ b/transformation-scripts/fwi-stations/FWI_Transformation.ipynb @@ -7,12 +7,12 @@ "metadata": {}, "outputs": [], "source": [ + "import re\n", + "from io import StringIO\n", + "\n", "import boto3\n", - "import s3fs\n", "import pandas as pd\n", - "import re\n", - "import os\n", - "from io import StringIO" + "import s3fs" ] }, { diff --git a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb index f2795dcf..bdfe5f8d 100644 --- a/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb +++ b/transformation-scripts/generate-rasi-cogs-from-netcdf3.ipynb @@ -42,14 +42,15 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "aa94a74d-58ff-4527-b397-13016133ac71", "metadata": {}, "outputs": [], "source": [ - "import os\n", - "import rio_cogeo.cogeo\n", + "from pathlib import Path\n", + "\n", "import pandas as pd\n", + "import rio_cogeo.cogeo\n", "import rioxarray # noqa\n", "import s3fs\n", "import xarray as xr" @@ -682,7 +683,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "6c4e69b1-cb7f-4845-b656-8767a246832b", "metadata": {}, "outputs": [ @@ -699,7 +700,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -721,7 +723,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "08c31207-1bce-4573-9edd-adc441e476a8", "metadata": {}, "outputs": [ @@ -738,7 +740,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -811,7 +814,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "3ad681ca-1617-46d9-a282-2b60c3c749ab", "metadata": {}, "outputs": [ @@ -828,7 +831,8 @@ "last_value = da.sel(percentile=10).isel(lon=0, lat=151).item()\n", "\n", "print(\n", - " f\"First value in the array is {first_value}, last value along the lat dimension is {last_value}\"\n", + " f\"First value in the array is {first_value}, \"\n", + " f\"last value along the lat dimension is {last_value}\"\n", ")" ] }, @@ -1545,7 +1549,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "13fa9162-46d8-4466-801b-2e2e49458154", "metadata": {}, "outputs": [], @@ -1558,7 +1562,7 @@ "dates = pd.date_range(\"2014-01-01\", \"2014-03-01\", freq=\"MS\")\n", "\n", "# make sure the dir exists\n", - "os.makedirs(\"COGS\", exist_ok=True)\n", + "Path(\"COGS\").mkdir(parents=True, exist_ok=True)\n", "\n", "for date in dates:\n", " datestr = date.strftime(\"%Y%m\")\n", diff --git a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb index b8389b80..254c2d5c 100644 --- a/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb +++ b/transformation-scripts/hls-reprocessed-ej-demo/publish-hls-provided-item-metadata.ipynb @@ -30,8 +30,9 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", "import json\n", + "\n", + "import boto3\n", "import requests\n", "from pystac import Item\n", "\n", @@ -145,7 +146,8 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\" and touch up metadata to meet 1.0.0 spec\n", + " # Update the stac version for these items from \"stac_version\": \"1.0.0-beta.2\"\n", + " # and touch up metadata to meet 1.0.0 spec\n", " item_dict[\"stac_version\"] = \"1.0.0\"\n", "\n", " # Add full extension hrefs https://github.com/radiantearth/stac-spec/blob/master/item-spec/item-spec.md#stac_extensions\n", @@ -164,7 +166,7 @@ " item_assets = item_dict[\"assets\"]\n", " # Previous location did not have data version number\n", " old_prefix = collection_id.replace(\"-002-\", \"-\")\n", - " for asset_key in item_assets.keys():\n", + " for asset_key in item_assets:\n", " new_href = item_assets[asset_key][\"href\"].replace(old_prefix, collection_id)\n", " item_assets[asset_key][\"href\"] = new_href\n", "\n", @@ -172,7 +174,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " print(f\"Invalid {collection_id=} {item.id=}\")\n", "\n", " # Replace the s3 object with the updated metadata for stac version 1.0.0\n", @@ -225,7 +227,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " print(f\"invalid {collection_id=} {item.id=}\")\n", "\n", " # Publish to target STAC catalog\n", @@ -236,17 +238,16 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if not publish_response.reason == \"Created\":\n", + " if publish_response.reason != \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " else:\n", - " if verbose:\n", - " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", - " )" + " elif verbose:\n", + " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb index df65234a..ab3ca36c 100644 --- a/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/aral-sea.ipynb @@ -132,8 +132,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -178,7 +178,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item \"\n", + " \"records are not interchangeable across instruments for purposes \"\n", + " \"other than visualization and that the assets available are not the \"\n", + " \"same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -259,11 +267,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -316,9 +325,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -327,7 +334,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -362,7 +369,9 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { @@ -372,7 +381,9 @@ "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb index a86da37a..359fd8eb 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-balaton.ipynb @@ -146,8 +146,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -201,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": null, "id": "dff29fbf-0001-42c0-9058-7fb93ab0396d", "metadata": { "tags": [] @@ -223,7 +223,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item records \"\n", + " \"are not interchangeable across instruments for purposes other than \"\n", + " \"visualization and that the assets available are not the same for each \"\n", + " \"instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -296,7 +304,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": null, "id": "2bbb4330-787a-4ceb-99a4-a79a12c2f3a2", "metadata": { "tags": [] @@ -304,11 +312,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -347,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 105, + "execution_count": null, "id": "ef5c5f88", "metadata": {}, "outputs": [], @@ -361,9 +370,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -372,7 +379,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -410,22 +417,26 @@ }, { "cell_type": "code", - "execution_count": 69, + "execution_count": null, "id": "bf75d1ca", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 107, + "execution_count": null, "id": "d80b42eb", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb index 5fa41682..af4e83d6 100644 --- a/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/lake-biwa.ipynb @@ -134,8 +134,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -160,14 +160,14 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": f\"{collection_id}\",\n", + " \"id\": collection_id,\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -180,7 +180,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection \"\n", + " \"for the purpose of visualizing RGB images. Note that the full item \"\n", + " \"records are not interchangeable across instruments for purposes \"\n", + " \"other than visualization and that the assets available are not \"\n", + " \"the same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -255,7 +263,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -263,11 +271,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -306,7 +315,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "id": "34e88a7c", "metadata": {}, "outputs": [], @@ -320,9 +329,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -331,7 +338,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -369,22 +376,26 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "id": "5e419e70", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "id": "23c57e6c", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb index 714b7c73..e96a35ac 100644 --- a/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/mirror-landsat-demos.ipynb @@ -35,10 +35,11 @@ ], "source": [ "import json\n", - "import requests\n", "from pathlib import Path\n", - "from pystac_client import Client\n", + "\n", + "import requests\n", "from pystac import Collection, Item\n", + "from pystac_client import Client\n", "\n", "# Source records to correct and publish to target\n", "SRC_STAC_API_URL = \"https://staging-stac.delta-backend.com\"\n", @@ -98,7 +99,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "af3c723e-df35-4aed-abb1-f9c1dc78909b", "metadata": {}, "outputs": [ @@ -131,12 +132,14 @@ " # Start the new collection we will publish\n", " collection_dict = src_collection.to_dict(include_self_link=False)\n", "\n", - " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections:\n", + " # cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = collection_dict.get(\"item_assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " collection_dict[\"item_assets\"] = item_assets\n", "\n", - " # Make sure the summaries object from the staged collection is included in the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", + " # Make sure the summaries object from the staged collection is included in\n", + " # the veda-data record (TODO check why this isn't passed through pystac to_dict)\n", " collection_dict[\"summaries\"] = src_collection.summaries._summaries\n", "\n", " # Validate\n", @@ -172,7 +175,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "ac9df7f0-308b-4cc9-9268-55a8a5812047", "metadata": {}, "outputs": [ @@ -358,12 +361,14 @@ " )\n", " item_dict[\"links\"] = links\n", "\n", - " # Special case for these landsat collections: cog_default assets were mistakenly added to item_assets so fix it\n", + " # Special case for these landsat collections:\n", + " # cog_default assets were mistakenly added to item_assets so fix it\n", " item_assets = item_dict.get(\"assets\")\n", " item_assets.pop(\"cog_default\", None)\n", " item_dict[\"assets\"] = item_assets\n", "\n", - " # Remove classification extension (many items do not have valid classification metadata)\n", + " # Remove classification extension\n", + " # (many items do not have valid classification metadata)\n", " extensions = item_dict[\"stac_extensions\"]\n", " item_dict[\"stac_extensions\"] = [\n", " i\n", @@ -376,7 +381,7 @@ " item = Item.from_dict(item_dict)\n", " try:\n", " item.validate()\n", - " except Exception as e:\n", + " except Exception:\n", " invalid_items.append(item.id)\n", " print(f\"invalid {item.id=} {len(invalid_items)=}\")\n", "\n", @@ -388,18 +393,17 @@ " )\n", " if verbose:\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", - " if not publish_response.reason == \"Created\":\n", + " if publish_response.reason != \"Created\":\n", " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {publish_response.reason=}\"\n", + " f\"POST {publish_url} {collection_id=}\\n\"\n", + " f\"{item_dict['id']=} {publish_response.reason=}\"\n", " )\n", " failed_ingestions.append(item.id)\n", - " else:\n", - " if verbose:\n", - " print(\n", - " f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\"\n", - " )\n", + " elif verbose:\n", + " print(f\"POST {publish_url} {collection_id=}\\n{item_dict['id']=} {dry_run=}\")\n", "\n", "\n", "if len(invalid_items):\n", @@ -419,7 +423,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "0d922785-dc2c-4bea-8205-5f69a47b68c7", "metadata": {}, "outputs": [ @@ -460,7 +464,8 @@ " target_item_count = len(target_item_collection)\n", "\n", " print(\n", - " f\"\\n{collection_id} {src_item_count=} {target_item_count=} OK={src_item_count==target_item_count}\"\n", + " f\"\\n{collection_id} {src_item_count=} {target_item_count=}\"\n", + " f\" OK={src_item_count == target_item_count}\"\n", " )\n", "\n", "print(\"\\nfin.\")" diff --git a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb index 6d132129..1956a590 100644 --- a/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/tonle-sap.ipynb @@ -144,8 +144,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -170,14 +170,14 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "c58bc898-4172-4e59-95dc-19b04273f019", "metadata": {}, "outputs": [], "source": [ "collection_id = f\"landsat-c2l2-sr-lakes-{LAKE_NAME_SLUG}\"\n", "collection = {\n", - " \"id\": f\"{collection_id}\",\n", + " \"id\": collection_id,\n", " \"type\": \"Collection\",\n", " \"title\": f\"Selected Landsat 7 through 9 Surface Reflectance Scenes for {LAKE_NAME}\",\n", " \"extent\": {\n", @@ -190,7 +190,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 \"\n", + " f\"Surface Reflectance scenes covering {LAKE_NAME}. Products from \"\n", + " \"multiple missions and instruments are combined in this single \"\n", + " \"collection for the purpose of visualizing RGB images. Note that \"\n", + " \"the full item records are not interchangeable across instruments \"\n", + " \"for purposes other than visualization and that the assets \"\n", + " \"available are not the same for each instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -263,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "f4dbf0d3-ccef-4651-b279-f881f7f9d84f", "metadata": { "tags": [] @@ -271,11 +279,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -314,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "48e9732a", "metadata": {}, "outputs": [], @@ -328,9 +337,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -339,7 +346,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -377,22 +384,26 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "e6b719d6", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "c080e51e", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb index a974a74b..c5ec97ce 100644 --- a/transformation-scripts/landsat-lakes-discovery/vanern.ipynb +++ b/transformation-scripts/landsat-lakes-discovery/vanern.ipynb @@ -142,8 +142,8 @@ "xmin = ymin = 1000\n", "xmax = ymax = -1000\n", "\n", - "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.timezone.utc)\n", - "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.timezone.utc)\n", + "mindate = datetime.datetime(2100, 1, 1, tzinfo=datetime.UTC)\n", + "maxdate = datetime.datetime(1900, 1, 1, tzinfo=datetime.UTC)\n", "\n", "for item in search.get_items():\n", " xmin = min(xmin, item.bbox[0])\n", @@ -168,7 +168,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "id": "d1558909-6bd8-4161-a710-4ce0f0502fc3", "metadata": {}, "outputs": [], @@ -188,7 +188,15 @@ " },\n", " \"license\": \"https://d9-wret.s3.us-west-2.amazonaws.com/assets/palladium/production/s3fs-public/atoms/files/Landsat_Data_Policy.pdf\",\n", " \"summaries\": {\"datetime\": [], \"cog_default\": {\"max\": None, \"min\": None}},\n", - " \"description\": f\"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface Reflectance scenes covering {LAKE_NAME}. Products from multiple missions and instruments are combined in this single collection for the purpose of visualizing RGB images. Note that the full item records are not interchangeable across instruments for purposes other than visualization and that the assets available are not the same for each instrument.\",\n", + " \"description\": (\n", + " \"Selected low-cloud-cover Landsat Collection-2 Level-2 Surface \"\n", + " f\"Reflectance scenes covering {LAKE_NAME}. Products from multiple \"\n", + " \"missions and instruments are combined in this single collection for \"\n", + " \"the purpose of visualizing RGB images. Note that the full item records \"\n", + " \"are not interchangeable across instruments for purposes other than \"\n", + " \"visualization and that the assets available are not the same for each \"\n", + " \"instrument.\"\n", + " ),\n", " \"item_assets\": {\n", " \"red\": {\n", " \"type\": \"image/vnd.stac.geotiff; cloud-optimized=true\",\n", @@ -263,7 +271,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "c8297cb0-b600-4b96-a3bd-1b2b8b1d05d1", "metadata": { "tags": [] @@ -271,11 +279,12 @@ "outputs": [], "source": [ "import json\n", + "from pathlib import Path\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-scenes.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-scenes.json\").open(\"w\") as fobj:\n", " json.dump(items, fobj, indent=2)\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-collection.json\", \"w\") as fobj:\n", + "with Path(f\"{LAKE_NAME_SLUG}-collection.json\").open(\"w\") as fobj:\n", " json.dump(collection, fobj, indent=2)" ] }, @@ -314,7 +323,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "0f66e4e7", "metadata": {}, "outputs": [], @@ -328,9 +337,7 @@ " )\n", "\n", " replacement_prefix = \"s3://usgs-landsat/collection02/\"\n", - " replaced_string = input_string.replace(expected_prefix, replacement_prefix)\n", - "\n", - " return replaced_string\n", + " return input_string.replace(expected_prefix, replacement_prefix)\n", "\n", "\n", "def remove_keys(dictionary, keys_to_keep):\n", @@ -339,7 +346,7 @@ "\n", "bands = [\"red\", \"green\", \"blue\"]\n", "\n", - "with open(f\"{LAKE_NAME_SLUG}-items.json\", \"w\") as file:\n", + "with Path(f\"{LAKE_NAME_SLUG}-items.json\").open(\"w\") as file:\n", " for item in search.items_as_dicts():\n", " item.update(item_links)\n", " item[\"collection\"] = collection_id\n", @@ -377,22 +384,26 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "id": "90eef7ab", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load collections {LAKE_NAME_SLUG}-collection.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "id": "f140d9c1", "metadata": {}, "outputs": [], "source": [ - "!pypgstac load items {LAKE_NAME_SLUG}-items.json --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis --method upsert" + "!pypgstac load items {LAKE_NAME_SLUG}-items.json \\\n", + " --dsn postgresql://$DB_USERNAME:$DB_PASSWORD@$DB_HOSTNAME:5432/postgis \\\n", + " --method upsert" ] } ], diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb index 63d77074..27296971 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-organize-cogs.ipynb @@ -87,10 +87,8 @@ "outputs": [], "source": [ "import boto3\n", - "import s3fs\n", - "import rioxarray\n", - "import rasterio\n", "import rio_cogeo.cogeo\n", + "import s3fs\n", "import xarray as xr\n", "from rasterio.io import MemoryFile\n", "\n", @@ -139,15 +137,19 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "f5b3ff3f-1060-4223-a127-9780e7bb6c9d", "metadata": {}, "outputs": [], "source": [ - "# COG configuration, deflate compression is slightly smaller than the LZW default for the cog driver\n", - "# and predictor=2 further reduces file size\n", + "# COG configuration, deflate compression is slightly smaller than\n", + "# the LZW default for the cog driver and predictor=2 further reduces file size\n", + "\n", "# https://kokoalberti.com/articles/geotiff-compression-optimization-guide/\n", - "# \"Predictors work especially well when there is some spatial correlation in the data, and pixels have values which are similar to their neighbours. As the name suggests, the floating point predictor can only be used on floating point data.\"\n", + "\n", + "# \"Predictors work especially well when there is some spatial correlation in the data,\n", + "# and pixels have values which are similar to their neighbours. As the name suggests,\n", + "# the floating point predictor can only be used on floating point data.\"\n", "\n", "driver = \"COG\"\n", "compress = \"DEFLATE\"\n", @@ -3951,71 +3953,75 @@ " print(f\"{src_key=}\")\n", "\n", " # Get object and read open as xarray dataset\n", - " with fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj:\n", - " with xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds:\n", - " # Now loop through the individual variables and generate a COG for each of these\n", - " for metric in METRICS:\n", - " # Generate output path\n", - " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", - " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", - " )\n", + " with (\n", + " fs.open(f\"{STAGING_BUCKET}/{src_key}\") as fileobj,\n", + " xr.open_dataset(fileobj, engine=\"h5netcdf\") as ds,\n", + " ):\n", + " # Now loop through the individual variables and generate a COG for each\n", + " for metric in METRICS:\n", + " # Generate output path\n", + " cog_key = cmip6_file_organization.generate_yearly_fwi_metrics_key(\n", + " src_key, \"v0\", pub_type=\"cog\", metric=metric\n", + " )\n", "\n", - " if verbose or extra_dryrun:\n", - " print(f\"{cog_key=}\")\n", - " if extra_dryrun:\n", - " continue\n", + " if verbose or extra_dryrun:\n", + " print(f\"{cog_key=}\")\n", + " if extra_dryrun:\n", + " continue\n", "\n", - " # Read individual metric into data array (only one time in yearly NetCDFs)\n", - " da = ds[metric]\n", + " # Read individual metric into data array\n", + " # (only one time in yearly NetCDFs)\n", + " da = ds[metric]\n", "\n", - " # Realign the x dimension to -180 origin for dataset\n", - " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\n", - " \"lon\"\n", - " )\n", + " # Realign the x dimension to -180 origin for dataset\n", + " da = da.assign_coords(lon=(((da.lon + 180) % 360) - 180)).sortby(\"lon\")\n", "\n", - " # Reverse the DataArray's y dimension. It appears that the source NetCDF's y dimension is inverted.\n", - " da = da.reindex(lat=list(reversed(da.lat)))\n", + " # Reverse the DataArray's y dimension\n", + " # It appears that the source NetCDF's y dimension is inverted\n", + " da = da.reindex(lat=list(reversed(da.lat)))\n", "\n", - " # Convert to COG via rioxarray\n", - " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", - " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", + " # Convert to COG via rioxarray\n", + " da.rio.set_spatial_dims(\"lon\", \"lat\")\n", + " da.rio.write_crs(\"epsg:4326\", inplace=True)\n", "\n", - " # Here we need a memory file to write the output raster to\n", - " with MemoryFile() as memfile:\n", - " da.rio.to_raster(\n", - " memfile.name,\n", - " driver=\"COG\",\n", - " compress=compress,\n", - " predictor=predictor,\n", - " )\n", + " # Here we need a memory file to write the output raster to\n", + " with MemoryFile() as memfile:\n", + " da.rio.to_raster(\n", + " memfile.name,\n", + " driver=\"COG\",\n", + " compress=compress,\n", + " predictor=predictor,\n", + " )\n", "\n", - " # Validate and upload COG\n", - " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", - " if dryrun:\n", - " print(\n", - " f\"Generation of valid COG {cog_key=} successful. Skipping upload.\"\n", - " )\n", - " continue\n", + " # Validate and upload COG\n", + " if rio_cogeo.cogeo.cog_validate(memfile.name)[0]:\n", + " if dryrun:\n", + " print(\n", + " f\"Generation of valid COG {cog_key=} successful. \"\n", + " \"Skipping upload.\"\n", + " )\n", + " continue\n", "\n", - " if verbose:\n", - " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", + " if verbose:\n", + " print(f\"START to upload {cog_key=} to {DST_BUCKET=}\")\n", "\n", - " # Upload memory file and confirm success\n", - " if not dry_run:\n", - " r = client.put_object(\n", - " Body=memfile,\n", - " Bucket=DST_BUCKET,\n", - " Key=cog_key,\n", - " )\n", - " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", - " error_responses.append(r)\n", - " if verbose:\n", - " print(f\"WARNING unable to upload {cog_key=}\")\n", - " else:\n", - " invalid_cogs.append(cog_key)\n", + " # Upload memory file and confirm success\n", + " if not dryrun:\n", + " r = client.put_object(\n", + " Body=memfile,\n", + " Bucket=DST_BUCKET,\n", + " Key=cog_key,\n", + " )\n", + " if r[\"ResponseMetadata\"][\"HTTPStatusCode\"] != 200:\n", + " error_responses.append(r)\n", + " if verbose:\n", + " print(f\"WARNING unable to upload {cog_key=}\")\n", + " else:\n", + " invalid_cogs.append(cog_key)\n", "\n", "print(\n", - " f\"\\nCOMPLETED with {len(error_responses)} errors and {len(invalid_cogs)} invalid cogs\"\n", + " f\"\\nCOMPLETED with {len(error_responses)} errors \"\n", + " f\"and {len(invalid_cogs)} invalid cogs\"\n", ")\n", "print(error_responses)\n", "print(invalid_cogs)" diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb index c3c91052..13c9656e 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6-generate-stac.ipynb @@ -31,7 +31,6 @@ "metadata": {}, "outputs": [], "source": [ - "from datetime import datetime, timezone\n", "import pystac" ] }, @@ -53,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "id": "dcf5702f-ed7c-4844-9fcd-a2e693f5ec05", "metadata": { "tags": [] @@ -64,7 +63,8 @@ "TITLE = \"Fire Weather Index Yearly SSP245\"\n", "DESCRIPTION = (\n", " \"FWI is a general index of fire danger and the data are monthly and annual \"\n", - " \"multi-model ensemble’s of the 50th percentile of mean Fire Weather Index from the 33 GCMs\"\n", + " \"multi-model ensemble's of the 50th percentile of mean Fire Weather Index \"\n", + " \"from the 33 GCMs\"\n", ")\n", "DASHBOARD__IS_PERIODIC = True\n", "DASHBOARD__TIME_DENSITY = \"year\"\n", diff --git a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py index 38496c83..f84dcde4 100644 --- a/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py +++ b/transformation-scripts/nex-gddp-cmip6-fwi/cmip6_file_organization.py @@ -1,5 +1,4 @@ -from os.path import basename -from typing import Optional +from pathlib import Path from urllib.parse import urlparse @@ -8,7 +7,7 @@ def generate_yearly_fwi_metrics_key( dst_version: str, ensemble: str = "mme", pub_type: str = "netcdf", - metric: Optional[str] = None, + metric: str | None = None, verbose: bool = False, ): """_summary_ @@ -23,7 +22,7 @@ def generate_yearly_fwi_metrics_key( cadence = "yearly" parsed = urlparse(src_url, allow_fragments=False) - nc_base = basename(parsed.path) + nc_base = Path(parsed.path).name # Assume pattern ensemble-stat_experiment_fwi_metrics_yearly_yyyy # MME50_ssp245_fwi_metrics_yearly_2100 diff --git a/transformation-scripts/soil_moisture_transformation_cog.ipynb b/transformation-scripts/soil_moisture_transformation_cog.ipynb index 1f15bf0a..0698e62e 100644 --- a/transformation-scripts/soil_moisture_transformation_cog.ipynb +++ b/transformation-scripts/soil_moisture_transformation_cog.ipynb @@ -17,17 +17,14 @@ }, "outputs": [], "source": [ - "import s3fs\n", - "import rioxarray\n", - "import rasterio\n", - "import rio_cogeo.cogeo\n", - "import xarray as xr\n", + "import json\n", "import re\n", - "from datetime import datetime\n", - "import pandas as pd\n", - "import boto3\n", "import tempfile\n", - "import json" + "\n", + "import boto3\n", + "import pandas as pd\n", + "import s3fs\n", + "import xarray as xr" ] }, { diff --git a/transformation-scripts/update-hrefs.ipynb b/transformation-scripts/update-hrefs.ipynb index b3d57b47..c388c7cc 100644 --- a/transformation-scripts/update-hrefs.ipynb +++ b/transformation-scripts/update-hrefs.ipynb @@ -13,8 +13,9 @@ "metadata": {}, "outputs": [], "source": [ - "import boto3\n", - "import json" + "import json\n", + "\n", + "import boto3" ] }, { @@ -51,7 +52,7 @@ }, { "cell_type": "code", - "execution_count": 109, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -91,7 +92,8 @@ " # Serialize updated JSON\n", " updated_json = json.dumps(data)\n", "\n", - " # Upload updated JSON file back to S3, commented out the line below so it doesn't actually upload\n", + " # Upload updated JSON file back to S3\n", + " # commented out the line below so it doesn't actually upload\n", " s3.put_object(Bucket=bucket_name, Key=key, Body=updated_json)\n", " print(f\"Updated {key}\")" ] diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..bc082ea3 --- /dev/null +++ b/uv.lock @@ -0,0 +1,419 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "cfgv" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "distlib" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c9/02/bd72be9134d25ed783ecbbc38a539ffaefbf90c78418c7fb7229600dbac7/distlib-0.4.3.tar.gz", hash = "sha256:f152097224a0ae24be5a0f6bae1b9359af82133bce63f98a95f86cae1aede9ed", size = 615141, upload-time = "2026-06-12T08:04:52.847Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/08/9c41fb51ab5b43eb21674aff13df270e8ba6c4b29c8624e328dc7a9482af/distlib-0.4.3-py2.py3-none-any.whl", hash = "sha256:4b0ce306c966eb73bc3a7b6abad017c556dadd92c44701562cd528ac7fde4d5b", size = 470628, upload-time = "2026-06-12T08:04:50.506Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/57/3ba6e6cb097f85b855b00163d169f35365f44277df044dcf96d55b8f62a3/filelock-3.32.2.tar.gz", hash = "sha256:c33351e1f49cae33414acbc6d56784e6ecee82514ec90795da1161fc4836b5b8", size = 217172, upload-time = "2026-07-29T22:46:04.895Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c1/e8/72f8cef9fdfeffe06213fe8508039396ee48daa0e3259457ed766173bfd6/filelock-3.32.2-py3-none-any.whl", hash = "sha256:87dd94cf281e586d135fa51132b8e3d9a598b316e90377a288663c9321036c82", size = 98830, upload-time = "2026-07-29T22:46:03.52Z" }, +] + +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.11.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/98/0bf930c4f97d0266b58a89e36c015f56232c52b5d2f207215d48cca9e8f7/platformdirs-4.11.2.tar.gz", hash = "sha256:3a2ae5fca3520a01ab1be8b45613537f52ddf5b5f6f53d88233892dfbf0cd82d", size = 32716, upload-time = "2026-08-10T15:48:06.092Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/e2/4e6eee633809c376c024821b91ade709cbfd040ec53939ffbcc292aa7eee/platformdirs-4.11.2-py3-none-any.whl", hash = "sha256:7f89089b6ea71bda7962953edcf784b2e2d9d285b40ad88be2bb75c6e9d82ab4", size = 23361, upload-time = "2026-08-10T15:48:04.855Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pre-commit" +version = "4.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cfgv" }, + { name = "identify" }, + { name = "nodeenv" }, + { name = "pyyaml" }, + { name = "virtualenv" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, +] + +[[package]] +name = "pystac" +version = "1.8.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2f/02/4a4bf2aceacd8e8c293434a454a42cab75de07df3f3d15e3dbd09e901500/pystac-1.8.4.tar.gz", hash = "sha256:ab9b93d16c4cca80e3c225cd5815dbaec610ee9320e2119857db17481b9bd2e5", size = 135011, upload-time = "2023-09-25T19:11:45.094Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/33/528355f091d6522e5f11e02b1309cb4713e21ef8f772dd9b38e8276da011/pystac-1.8.4-py3-none-any.whl", hash = "sha256:0d9917bf3abc71fca3edfb2dd5f66be44b22a63774cda520c81484a126fa780e", size = 176247, upload-time = "2023-09-25T19:11:42.828Z" }, +] + +[package.optional-dependencies] +validation = [ + { name = "jsonschema" }, +] + +[[package]] +name = "pytest" +version = "7.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e5/d0/18209bb95db8ee693a9a04fe056ab0663c6d6b1baf67dd50819dd9cd4bd7/pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069", size = 1354640, upload-time = "2023-09-07T18:45:54.667Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/d0/e192c4275aecabf74faa1aacd75ef700091913236ec78b1a98f62a2412ee/pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002", size = 324497, upload-time = "2023-09-07T18:45:52.001Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-discovery" +version = "1.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "filelock" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/38/b7/ac44da2cf0e53ada0e419033c2d058219c95dc1403126f163304c9e814b1/python_discovery-1.5.2.tar.gz", hash = "sha256:45fd4f20a4e3f9b7bf2e0817870bc8e3b320a19658da177af800768c82dbf354", size = 82350, upload-time = "2026-08-12T14:05:26.419Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/45/689603d04b3bb8d7faa00f25c24acef993aab7813b3dbbfc472a459ab0b5/python_discovery-1.5.2-py3-none-any.whl", hash = "sha256:3e338c2d0f15dfaeea57493f4c2c6caebe0e998ea815c30ae8bf8ee21f1112d3", size = 38350, upload-time = "2026-08-12T14:05:25.113Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/e5/af35f7ea75cf72f2cd079c95ee16797de7cd71f29ea7c68ae5ce7be1eda0/PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", size = 125201, upload-time = "2023-07-18T00:00:23.308Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/0d/26fb23e8863e0aeaac0c64e03fd27367ad2ae3f3cccf3798ee98ce160368/PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", size = 187867, upload-time = "2023-07-17T23:57:34.35Z" }, + { url = "https://files.pythonhosted.org/packages/28/09/55f715ddbf95a054b764b547f617e22f1d5e45d83905660e9a088078fe67/PyYAML-6.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", size = 167530, upload-time = "2023-07-17T23:57:36.975Z" }, + { url = "https://files.pythonhosted.org/packages/5e/94/7d5ee059dfb92ca9e62f4057dcdec9ac08a9e42679644854dc01177f8145/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", size = 732244, upload-time = "2023-07-17T23:57:43.774Z" }, + { url = "https://files.pythonhosted.org/packages/06/92/e0224aa6ebf9dc54a06a4609da37da40bb08d126f5535d81bff6b417b2ae/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", size = 752871, upload-time = "2023-07-17T23:57:51.921Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5e/efd033ab7199a0b2044dab3b9f7a4f6670e6a52c089de572e928d2873b06/PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", size = 757729, upload-time = "2023-07-17T23:57:59.865Z" }, + { url = "https://files.pythonhosted.org/packages/03/5c/c4671451b2f1d76ebe352c0945d4cd13500adb5d05f5a51ee296d80152f7/PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b", size = 748528, upload-time = "2023-08-28T18:43:23.207Z" }, + { url = "https://files.pythonhosted.org/packages/73/9c/766e78d1efc0d1fca637a6b62cea1b4510a7fb93617eb805223294fef681/PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", size = 130286, upload-time = "2023-07-17T23:58:02.964Z" }, + { url = "https://files.pythonhosted.org/packages/b3/34/65bb4b2d7908044963ebf614fe0fdb080773fc7030d7e39c8d3eddcd4257/PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", size = 144699, upload-time = "2023-07-17T23:58:05.586Z" }, + { url = "https://files.pythonhosted.org/packages/bc/06/1b305bf6aa704343be85444c9d011f626c763abb40c0edc1cad13bfd7f86/PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28", size = 178692, upload-time = "2023-08-28T18:43:24.924Z" }, + { url = "https://files.pythonhosted.org/packages/84/02/404de95ced348b73dd84f70e15a41843d817ff8c1744516bf78358f2ffd2/PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9", size = 165622, upload-time = "2023-08-28T18:43:26.54Z" }, + { url = "https://files.pythonhosted.org/packages/c7/4c/4a2908632fc980da6d918b9de9c1d9d7d7e70b2672b1ad5166ed27841ef7/PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef", size = 696937, upload-time = "2024-01-18T20:40:22.92Z" }, + { url = "https://files.pythonhosted.org/packages/b4/33/720548182ffa8344418126017aa1d4ab4aeec9a2275f04ce3f3573d8ace8/PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0", size = 724969, upload-time = "2023-08-28T18:43:28.56Z" }, + { url = "https://files.pythonhosted.org/packages/4f/78/77b40157b6cb5f2d3d31a3d9b2efd1ba3505371f76730d267e8b32cf4b7f/PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4", size = 712604, upload-time = "2023-08-28T18:43:30.206Z" }, + { url = "https://files.pythonhosted.org/packages/2e/97/3e0e089ee85e840f4b15bfa00e4e63d84a3691ababbfea92d6f820ea6f21/PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54", size = 126098, upload-time = "2023-08-28T18:43:31.835Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9f/fbade56564ad486809c27b322d0f7e6a89c01f6b4fe208402e90d4443a99/PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df", size = 138675, upload-time = "2023-08-28T18:43:33.613Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "veda-data" +version = "0.1.0" +source = { virtual = "." } + +[package.dev-dependencies] +dev = [ + { name = "pre-commit" }, + { name = "pystac", extra = ["validation"] }, + { name = "pytest" }, +] +scripts = [ + { name = "pyyaml" }, +] + +[package.metadata] + +[package.metadata.requires-dev] +dev = [ + { name = "pre-commit", specifier = "==4.6.1" }, + { name = "pystac", extras = ["validation"], specifier = "==1.8.4" }, + { name = "pytest", specifier = "==7.4.2" }, +] +scripts = [{ name = "pyyaml", specifier = "==6.0.1" }] + +[[package]] +name = "virtualenv" +version = "21.7.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "distlib" }, + { name = "filelock" }, + { name = "platformdirs" }, + { name = "python-discovery" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2d/dc/a6eb1ddfa7f1e390fa599b078453c97edb3f6f846b34fb4eac3e8ea16401/virtualenv-21.7.4.tar.gz", hash = "sha256:c9d960c95fa458171e58222a5ccab7465298e4b6559977865e627c4719f1e825", size = 5345511, upload-time = "2026-08-10T22:54:33.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/4c/eb2f52aeeaf30dbd073d315a251a63ae2b8263171ec4428c135140cb0802/virtualenv-21.7.4-py3-none-any.whl", hash = "sha256:376ec93cd6aab3044fa395d7db226db38043b7b5748948044b2a87168525e843", size = 5324444, upload-time = "2026-08-10T22:54:31.515Z" }, +] From acea77673354cd4b30c7c045461f6a56281071bd Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 13 Aug 2026 11:13:44 -0500 Subject: [PATCH 2/7] chore: add locked flag --- .github/dependabot.yml | 2 +- .github/workflows/ci.yaml | 4 ++-- .github/workflows/pr.yml | 4 ++-- .github/workflows/promote.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 415eedec..12515289 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -38,4 +38,4 @@ updates: patterns: - "*" commit-message: - prefix: "ci(deps):" + prefix: "chore(deps):" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 119a9a77..84078d76 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,9 +14,9 @@ jobs: matrix: include: - name: test - run: uv run --group dev pytest + run: uv run --locked --group dev pytest - name: lint - run: uv run --group dev pre-commit run --all-files + run: uv run --locked --group dev pre-commit run --all-files steps: - name: Checkout code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3f3034cf..a8a853e7 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -116,7 +116,7 @@ jobs: STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }} DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }} run: | - uv sync --group scripts + uv sync --locked--group scripts # Track successful publications all_failed=true @@ -298,7 +298,7 @@ jobs: run: | echo $PUBLISHED_COLLECTION_FILES collection_ids="" - uv sync --group scripts + uv sync --locked --group scripts for file in ${PUBLISHED_COLLECTION_FILES}; do collection_id=$(python3 ./scripts/generate_mdx.py "$file") collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//') diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index af9aadc3..4d9894f5 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -72,7 +72,7 @@ jobs: PROMOTION_DAG: ${{ vars.PROMOTION_DAG_NAME }} run: | - uv sync --group scripts + uv sync --locked --group scripts for file in downloaded-files/*.json; do echo "Processing $file" # Check if collection key exists to determine if it's a dataset or collection From c910acabc2827dc3f66eb4f5ca566209e61119c3 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Thu, 13 Aug 2026 13:12:07 -0500 Subject: [PATCH 3/7] chore: remove redundant path --- scripts/generate_mdx.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/generate_mdx.py b/scripts/generate_mdx.py index 4db953bf..a5f3ddcf 100644 --- a/scripts/generate_mdx.py +++ b/scripts/generate_mdx.py @@ -119,7 +119,7 @@ def safe_open_w(path): # Write the combined content back to the file output_filepath = ( - Path(curr_directory) + curr_directory / f"../ingestion-data/dataset-mdx/{input_data['collection']}.data.mdx" ) with safe_open_w(str(output_filepath)) as ofile: From d8b51558fa521f66ea441fc894acd019e9422539 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sat, 15 Aug 2026 23:39:04 -0500 Subject: [PATCH 4/7] chore: missing space --- .github/workflows/pr.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a8a853e7..b205f4b0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -116,7 +116,7 @@ jobs: STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }} DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }} run: | - uv sync --locked--group scripts + uv sync --locked --group scripts # Track successful publications all_failed=true From 659eef0988651142e4063ca7f7f818e1243ef0fa Mon Sep 17 00:00:00 2001 From: Jamison French Date: Wed, 19 Aug 2026 12:07:23 -0500 Subject: [PATCH 5/7] chore: update pre-commit --- .pre-commit-config.yaml | 5 +++++ pyproject.toml | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 284a8e63..17149243 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,3 +24,8 @@ repos: - "pyproject.toml" - "--configPointer" - "/tool/markdownlint-cli2" + + - repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.12.5 + hooks: + - id: uv-lock diff --git a/pyproject.toml b/pyproject.toml index 5ea46ef2..e2c6b1bf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,7 +38,6 @@ select = [ "TC", # flake8-type-checking "ISC", # flake8-implicit-str-concat "ICN", # flake8-import-conventions - "BLE", # flake8-blind-except ] ignore = [ "BLE001", # Blind Exceptions From 9c1a7ea605da5af499c3ed8d089ce94a3d8d9790 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sat, 22 Aug 2026 01:24:47 -0500 Subject: [PATCH 6/7] chore: group imports --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e2c6b1bf..381840f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,3 +50,6 @@ docstring-code-line-length = "dynamic" [tool.markdownlint-cli2.config] MD013 = false MD033 = { allowed_elements = ["a", "b", "details", "summary"] } + +[tool.ruff.lint.isort] +combine-as-imports = true From 137f1326524a296aada7e7d0a4808c3837aa5151 Mon Sep 17 00:00:00 2001 From: Jamison French Date: Sat, 22 Aug 2026 01:58:37 -0500 Subject: [PATCH 7/7] chore: upgrade pytest --- pyproject.toml | 2 +- uv.lock | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 381840f8..30f96aa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [] dev = [ "pre-commit==4.6.1", "pystac[validation]==1.8.4", - "pytest==7.4.2", + "pytest==9.0.3", ] scripts = [ "pyyaml==6.0.1", diff --git a/uv.lock b/uv.lock index bc082ea3..1564f9ab 100644 --- a/uv.lock +++ b/uv.lock @@ -144,6 +144,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, ] +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + [[package]] name = "pystac" version = "1.8.4" @@ -163,17 +172,18 @@ validation = [ [[package]] name = "pytest" -version = "7.4.2" +version = "9.0.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, { name = "iniconfig" }, { name = "packaging" }, { name = "pluggy" }, + { name = "pygments" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e5/d0/18209bb95db8ee693a9a04fe056ab0663c6d6b1baf67dd50819dd9cd4bd7/pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069", size = 1354640, upload-time = "2023-09-07T18:45:54.667Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/0d/549bd94f1a0a402dc8cf64563a117c0f3765662e2e668477624baeec44d5/pytest-9.0.3.tar.gz", hash = "sha256:b86ada508af81d19edeb213c681b1d48246c1a91d304c6c81a427674c17eb91c", size = 1572165, upload-time = "2026-04-07T17:16:18.027Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/df/d0/e192c4275aecabf74faa1aacd75ef700091913236ec78b1a98f62a2412ee/pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002", size = 324497, upload-time = "2023-09-07T18:45:52.001Z" }, + { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] [[package]] @@ -399,7 +409,7 @@ scripts = [ dev = [ { name = "pre-commit", specifier = "==4.6.1" }, { name = "pystac", extras = ["validation"], specifier = "==1.8.4" }, - { name = "pytest", specifier = "==7.4.2" }, + { name = "pytest", specifier = "==9.0.3" }, ] scripts = [{ name = "pyyaml", specifier = "==6.0.1" }]