chore: uv support

.github/dependabot.yml

@@ -0,0 +1,41 @@
+version: 2
+updates:
+  - package-ecosystem: uv
+    directory: /
+    cooldown:
+      default-days: 7
+      semver-major-days: 30
+    schedule:
+      interval: weekly
+    groups:
+      python-dependencies:
+        patterns:
+        - "*"
+    commit-message:
+      prefix: "chore(deps):"
+
+  - package-ecosystem: github-actions
+    directory: /
+    cooldown:
+      default-days: 7
+    schedule:
+      interval: weekly
+    groups:
+      github-actions:
+        patterns:
+        - "*"
+    commit-message:
+      prefix: "ci(deps):"
+
+  - package-ecosystem: pre-commit
+    directory: /
+    cooldown:
+      default-days: 7
+    schedule:
+      interval: weekly
+    groups:
+      pre-commit:
+        patterns:
+        - "*"
+    commit-message:
+      prefix: "chore(deps):"

.github/workflows/ci.yaml

@@ -7,31 +7,24 @@ on:
   pull_request:
 
 jobs:
-  test:
+  ci:
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: test
+            run: uv run --locked --group dev pytest
+          - name: lint
+            run: uv run --locked --group dev pre-commit run --all-files
     steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Setup Python
-        uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
-        with:
-          python-version: "3.11"
-          cache: "pip"
-      - name: Install dependencies
-        run: pip install -r requirements.txt
-      - name: Test collections
-        run: pytest
-  lint:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Setup Python
-        uses: actions/setup-python@b64ffcaf5b410884ad320a9cfac8866006a109aa # v4.8.0
+      - name: Checkout code
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
-          python-version: "3.11"
-          cache: "pip"
-      - name: Install dependencies
-        run: pip install -r requirements.txt
-      - name: Run pre-commit hooks
-        run: pre-commit run --all-files
+          enable-cache: true
+
+      - name: ${{ matrix.name }}
+        run: ${{ matrix.run }}

.github/workflows/pr.yml

@@ -35,7 +35,7 @@ jobs:
       comment_id: ${{ steps.init-comment.outputs.COMMENT_ID }}
       has_new_files_to_promote: ${{ steps.changed-files.outputs.all_changed_files_count != 0 }}
     steps:
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
         with:
           fetch-depth: 0
 
@@ -116,7 +116,7 @@ jobs:
           STAGING_SM2A_API_URL: ${{ vars.STAGING_SM2A_API_URL }}
           DATASET_DAG_NAME: ${{ vars.DATASET_DAG_NAME }}
         run: |
-          pip install -r ./scripts/requirements.txt
+          uv sync --locked --group scripts
 
           # Track successful publications
           all_failed=true
@@ -180,14 +180,10 @@ jobs:
           $status_message"
           gh api -X PATCH -H "Authorization: token $GITHUB_TOKEN" /repos/${{ github.repository }}/issues/comments/$COMMENT_ID -f body="$UPDATED_BODY"
 
-      - name: Set up Python
-        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
         with:
-          python-version: '3.9'
-      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
-        with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.pythonLocation }}-pip-${{ hashFiles('requirements.txt') }}
+          enable-cache: true
 
       # If the workflow fails at any point, the PR comment will be updated
       - name: Update PR comment on overall workflow failure
@@ -233,7 +229,7 @@ jobs:
         run: ls -lah /tmp/upload-artifacts
 
       - name: Upload published collections files as directory artifact
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         env:
           ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }}
         with:
@@ -247,7 +243,7 @@ jobs:
           echo "${{ github.run_id }}" > run_id.txt
 
       - name: Upload Workflow Run ID
-        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         env:
           ARTIFACT_RETENTION_DAYS: ${{ vars.ARTIFACT_RETENTION_DAYS }}
         with:
@@ -272,7 +268,7 @@ jobs:
       skip_mdx_pr_step: ${{ steps.skip_mdx_pr_step_setting.outputs.skip_mdx_pr_step }}
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Set SKIP_MDX_PR_STEP condition
         id: skip_mdx_pr_step_setting
@@ -289,7 +285,7 @@ jobs:
     needs: ['mdx-pr-flag-evaluator', 'publish-new-datasets']
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Use output from publish-new-datasets
         run: |
@@ -302,7 +298,7 @@ jobs:
         run: |
           echo $PUBLISHED_COLLECTION_FILES
           collection_ids=""
-          pip install -r ./scripts/requirements.txt
+          uv sync --locked --group scripts
           for file in ${PUBLISHED_COLLECTION_FILES}; do
             collection_id=$(python3 ./scripts/generate_mdx.py "$file")
             collection_id=$(echo "$collection_id" | sed 's/^["\s]*//;s/["\s]*$//')

.github/workflows/promote.yml

@@ -45,7 +45,12 @@ jobs:
     environment: production
     steps:
       - name: Checkout code
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0
+        with:
+          enable-cache: true
 
       - name: Determine artifact name
         run: echo "ARTIFACT_NAME=collections-to-promote-from-${{ github.event.inputs.pr_number }}" >> $GITHUB_ENV
@@ -67,7 +72,7 @@ jobs:
           PROMOTION_DAG: ${{ vars.PROMOTION_DAG_NAME }}
 
         run: |
-          pip install -r ./scripts/requirements.txt
+          uv sync --locked --group scripts
           for file in downloaded-files/*.json; do
             echo "Processing $file"
             # Check if collection key exists to determine if it's a dataset or collection

.github/workflows/promotion-checker.yml

@@ -53,7 +53,7 @@ jobs:
     needs: check-conditions
     steps:
       - name: Checkout Repository
-        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
 
       - name: Get Workflow Run ID Artifact
         run: |

.pre-commit-config.yaml

@@ -1,22 +1,31 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v6.0.0
     hooks:
       - id: check-yaml
-      - id: file-contents-sorter
-        files: ^requirements.in$
       - id: trailing-whitespace
         exclude: ^ingestion-data/.*$
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.16.1
+    hooks:
+      - id: ruff-check
+        args: [--fix]
+      - id: ruff-format
+        types_or: [python, pyi, jupyter, pyproject]
+
   - repo: https://github.com/DavidAnson/markdownlint-cli2
-    rev: v0.10.0
+    rev: v0.23.2
     hooks:
       - id: markdownlint-cli2
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.3.0
-    hooks:
-      - id: black
-      - id: black-jupyter
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.0.292
+        args:
+        - "--fix"
+        - "--config"
+        - "pyproject.toml"
+        - "--configPointer"
+        - "/tool/markdownlint-cli2"
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.12.5
     hooks:
-      - id: ruff
+      - id: uv-lock

CODEOWNERS

@@ -1,5 +1 @@
-* @anayeaye
-* @gadomski
-* @ividito
-* @slesaad
-* @smohiudd
+* @NASA-IMPACT/veda-data-services

README.md

@@ -20,7 +20,7 @@ To add data to VEDA you will:
 
 ### Step 1: Stage your files
 
-Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request access [here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html)
+Upload files to the staging bucket `s3://veda-data-store-staging` (which you can do with a VEDA JupyterHub account--request [access here](https://docs.openveda.cloud/user-guide/scientific-computing/getting-access.html)) or a self-hosted bucket in s3 has shared read access to VEDA service. [See docs.openveda.cloud for additional details on preparing files.](https://docs.openveda.cloud/user-guide/content-curation/dataset-ingestion/file-preparation.html)
 
 ### Step 2: Generate STAC metadata in the staging catalog
 

pyproject.toml

@@ -0,0 +1,55 @@
+[project]
+name = "veda-data"
+version = "0.1.0"
+description = "Config data used to load datasets into the VEDA catalog"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = []
+
+[dependency-groups]
+dev = [
+  "pre-commit==4.6.1",
+  "pystac[validation]==1.8.4",
+  "pytest==9.0.3",
+]
+scripts = [
+  "pyyaml==6.0.1",
+]
+
+[tool.uv]
+default-groups = ["dev"]
+
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+select = [
+    "E",    # pycodestyle errors
+    "W",    # pycodestyle warnings
+    "F",    # Pyflakes
+    "I",    # isort
+    "UP",   # pyupgrade
+    "B",    # flake8-bugbear
+    "C4",   # flake8-comprehensions
+    "SIM",  # flake8-simplify
+    "PTH",  # flake8-use-pathlib
+    "RET",  # flake8-return
+    "TID",  # flake8-tidy-imports
+    "TC",   # flake8-type-checking
+    "ISC",  # flake8-implicit-str-concat
+    "ICN",  # flake8-import-conventions
+]
+ignore = [
+  "BLE001",  # Blind Exceptions
+]
+
+[tool.ruff.format]
+docstring-code-format = true
+docstring-code-line-length = "dynamic"
+
+[tool.markdownlint-cli2.config]
+MD013 = false
+MD033 = { allowed_elements = ["a", "b", "details", "summary"] }
+
+[tool.ruff.lint.isort]
+combine-as-imports = true