Migrate from a google earth engine data pipeline to an xarray one

.env.example

@@ -0,0 +1,5 @@
+INGEST_BUCKET_NAME = 'cornerstone-ingest'
+GCP_PROJECT = 'cornerstone-data'
+NUMBER_OF_DASK_WORKERS = 12
+SCRATCH_BUCKET_NAME = 'cornerstone-scratch'
+USDA_NASS_API_KEY = 'Replace me with value from https://quickstats.nass.usda.gov/api'

.github/workflows/ci.yml

@@ -29,4 +29,4 @@ jobs:
       - name: Install dependencies
         run: uv sync
       - name: pytest (unit only)
-        run: uv run pytest jdluc/__tests__/ -m "not integration"
+        run: uv run pytest jdluc/__tests__/

.gitignore

@@ -10,3 +10,6 @@ __pycache__/
 # Notebook scratch — version-controlled notebooks live under specs/analyses/ only.
 *.ipynb
 .ipynb_checkpoints/
+
+# Env
+.env

.pre-commit-config.yaml

@@ -1,26 +1,17 @@
 exclude: |
   (?x)^(specs/analyses/)
 repos:
-  # mypyc-compiled black is ~2x faster than the upstream wheel.
-  - repo: https://github.com/psf/black-pre-commit-mirror
-    rev: 24.4.1
-    hooks:
-      - id: black
-        language_version: python3.14
-        args: [--skip-string-normalization]
-
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.15.12
+    rev: v0.15.13
     hooks:
-      - id: ruff
-        args: [--fix]
-
+      - id: ruff-check
+        args: [--fix, "--select=F,I,UP"]
+      - id: ruff-format
   - repo: https://github.com/seddonym/import-linter
     rev: v2.11
     hooks:
       - id: import-linter
         language: python
-
   # Run mypy within the venv so that libraries are visible
   - repo: local
     hooks:
@@ -29,3 +20,9 @@ repos:
         language: system
         entry: uv run mypy jdluc
         pass_filenames: false
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v6.0.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: trailing-whitespace

ATTRIBUTION.md

@@ -2,4 +2,4 @@ Attribute all emissions factors or other data built on this pipeline as: "Corner
 
 You must prominently display this attribution in public-facing models, tools, datasets or any other applications that are generated from this code. This includes reimplementations of the code that are built primarily by passing the methodology and/or technical specs to a coding agent.
 
-If the code has been modified in a way that changes the published data products, that should be clearly stated.
+If the code has been modified in a way that changes the published data products, that should be clearly stated.

LICENSE.txt

@@ -1,16 +1,16 @@
-Cornerstone Data Pipeline Attribution License 1.0 
+Cornerstone Data Pipeline Attribution License 1.0
 
 Acceptance
-In order to receive this license, you must agree to its rules. The rules of this license are both obligations under that agreement and conditions to your license. 
+In order to receive this license, you must agree to its rules. The rules of this license are both obligations under that agreement and conditions to your license.
 
 License
-The licensor licenses you to do everything with this software that would otherwise infringe the licensor’s copyright in it, or any patent claims the licensor can license or becomes able to license. If you make any written claim that the software infringes or contributes to infringement of any patent, your license ends immediately. 
+The licensor licenses you to do everything with this software that would otherwise infringe the licensor’s copyright in it, or any patent claims the licensor can license or becomes able to license. If you make any written claim that the software infringes or contributes to infringement of any patent, your license ends immediately.
 
-Notices on Software 
+Notices on Software
 If you provide a copy of the software to anyone else, or provide or display Produced Data to anyone else, you must provide a copy of this license or a link to https://github.com/cornerstone-data/jdluc/blob/main/LICENSE.txt.
 
 Notices on Produced Data
-If you calculate data (including summaries or reports of data) using this software, you must deliver an attribution, in a form to be specified by the licensor, in a reasonable and conspicuous manner to anyone to whom you deliver or display the data.  
+If you calculate data (including summaries or reports of data) using this software, you must deliver an attribution, in a form to be specified by the licensor, in a reasonable and conspicuous manner to anyone to whom you deliver or display the data.
 
 Modifications
 If you modify the Software, you may apply other license terms to your modifications, but you can only provide the Software to others under the terms at least as protective of the rights of the licensor as the terms of this license.

README.md

@@ -16,17 +16,20 @@ The methodology and technical decisions in this repo are intended as a starting
 
 ## Data access
 
-Emissions factors and summary statistics:
+Harmonized datasets and emissions layers:
 
-```shell
-curl -O https://storage.googleapis.com/cornerstone-luc/cornerstone-data/jdluc/conus/tables/crops.csv
-curl -O https://storage.googleapis.com/cornerstone-luc/cornerstone-data/jdluc/conus/tables/transitions.csv
+```python
+>>> import xarray
+>>> harmonized = xarray.open_zarr("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/harmonize.zarr", consolidated=False)
+>>> emissions = xarray.open_zarr("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/emissions.zarr", consolidated=False)
 ```
 
-Simple raster visualizations:
+Emissions factors:
 
-- [Land use transitions, crop, and peat masks](https://cornerstone-data.projects.earthengine.app/view/jdluc-landuse-conus-20260513) 
-- [Pixel-level emissions estimates](https://cornerstone-data.projects.earthengine.app/view/jdluc-emissions-conus-20260513)
+```python
+>>> import pandas
+>>> emission_factors = pandas.read_parquet("gs://cornerstone-luc/cornerstone-data/jdluc/conus-v2/emissions-factors.parquet")
+```
 
 The data is licensed [CC-BY 4.0](https://creativecommons.org/licenses/by/4.0/). Please follow the latest attribution guidance in ATTRIBUTION.md.
 
@@ -41,55 +44,44 @@ Once you're ready to look under the hood:
 
 ### Getting set up
 
-You'll need access to a GCP project with the Earth Engine and BigQuery APIs enabled. You'll also need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager.
+You'll need [uv](https://docs.astral.sh/uv/getting-started/installation/) installed as the Python env manager, plus a GCP project with GCS access and a USDA NASS QuickStats API key.
 
 ```bash
-# Set GCP_PROJECT to your GCP project ID (the same value you set in
-# jdluc/utils/constants.py).
-export GCP_PROJECT=cornerstone-data
+# Copy the example env file and fill in your values.
+cp .env.example .env
+# Set INGEST_BUCKET_NAME, GCP_PROJECT, SCRATCH_BUCKET_NAME, and USDA_NASS_API_KEY in .env.
 
 # Sync Python dependencies into the project venv.
 uv sync
 
-# Authenticate gcloud application-default credentials.
+# Authenticate gcloud application-default credentials (for GCS access).
 gcloud auth application-default login --project "${GCP_PROJECT}"
-
-# Authenticate Earth Engine and point it at the same project.
-uv run earthengine authenticate
-uv run earthengine set_project "${GCP_PROJECT}"
 ```
 
-### Running the pipeline
-
-First, set GCP project info by editing the deployment configuration block at the top of `jdluc/utils/constants.py`.
+Obtain a NASS QuickStats API key at https://quickstats.nass.usda.gov/api and set it as `USDA_NASS_API_KEY` in `.env`.
 
-Second, create the project folder in Google Earth Engine:
-
-```shell
-uv run earthengine --project=${GCP_PROJECT} create folder projects/${GCP_PROJECT}/assets/cornerstone-luc
-```
+### Running the pipeline
 
-Finally:
+The pipeline runs as a sequence of per-stage entry points, parameterized by a tile set (`DELAWARE`, `CONUS`, `BAY_AREA`, `GFW`, or `WHOLE_WORLD`):
 
 ```bash
-# Default: Delaware (single state, smallest test region)
-uv run python jdluc/cli.py -v
+# 1. Ingest each source dataset (positional args; --concurrency / --overwrite optional).
+uv run python -m jdluc.ingest <TILE_SET> <DATASET>
 
-# Multi-state (Iowa + Nebraska + South Dakota)
-uv run python jdluc/cli.py --region great_plains_test -v
+# 2. Harmonize ingested tiles onto the common grid.
+uv run python jdluc/harmonize.py <TILE_SET>
 
-# Full CONUS (longer — full 48 states + DC)
-uv run python jdluc/cli.py --region conus -v
+# 3. Compute per-pixel land-conversion emissions (writes zarr).
+uv run python jdluc/emissions.py <TILE_SET>
 
-# Force re-export even when an asset already exists at the target version
-uv run python jdluc/cli.py --force -v
+# 4. Build the emissions-factor table (Delaware example; --skip-glad-crop-filter optional).
+uv run python jdluc/emissions_factors.py --admin-id USA008
 ```
 
 ### Tests
 
 ```bash
-uv run pytest jdluc -m "not integration"   # unit tests; offline
-uv run pytest jdluc -m integration         # GEE / BQ integration
+uv run pytest jdluc
 ```
 
 ### Linting

jdluc/__tests__/test_attribution.py

@@ -0,0 +1,140 @@
+import collections.abc
+
+import numpy
+import pytest
+import xarray
+
+from jdluc.attribution import Crop, JurisdictionalCropEmission
+from jdluc.datasets.worldbank_jurisdictions import AdminLevel
+
+
+def get_darray_for_data(
+    data: collections.abc.Sequence[collections.abc.Sequence[float]],
+) -> xarray.DataArray:
+    arr = numpy.array(data)
+    y, x = arr.shape
+    return xarray.DataArray(
+        coords={"y": range(y), "x": range(x)}, data=arr, dims=("y", "x")
+    )
+
+
+def test_from_xarray() -> None:
+    dset = xarray.Dataset(
+        {
+            # 99s sit on non-crop pixels — they must be excluded by the mask.
+            "emissions-per-hectare:tco2e-per-ha": get_darray_for_data(
+                [[1, 2, 99], [99, 4, 99]]
+            ),
+            "hectares-per-pixel:ha": get_darray_for_data([[10, 10, 10], [10, 10, 10]]),
+            "gfw:global-peatlands:is-peatland": get_darray_for_data(
+                [[1, 0, 0], [0, 1, 0]]
+            ),
+            "land-class:2020": get_darray_for_data([[2] * 2] * 2),
+            "peatland-occupation:tco2e-per-ha": get_darray_for_data(
+                [[5, 5, 5], [5, 5, 5]]
+            ),
+            # crop pixels: (0,0), (0,1), (1,1); of those, peatland at (0,0) and (1,1).
+            "usda-nass:cdl:crop-class": get_darray_for_data([[1, 1, 0], [0, 1, 0]]),
+        }
+    )
+    result = JurisdictionalCropEmission.from_dset(
+        admin_id="USA008",
+        admin_level=AdminLevel.PROVINCIAL,
+        crop=Crop.CORN,
+        dset=dset,
+        jurisdiction_name="Delaware",
+        skip_glad_crop_filter=False,
+    )
+    assert result == JurisdictionalCropEmission(
+        admin_id="USA008",
+        admin_level="PROVINCIAL",
+        crop_hectares=30.0,  # 3 crop pixels × 10 ha
+        crop_name="CORN",
+        jurisdiction_name="Delaware",
+        peatland_crop_hectares=20.0,  # only (0,0) and (1,1) are crop AND peatland
+        peatland_occupation_emissions=150.0,  # 3 crop pixels × (5 × 10)
+        total_emissions=70.0,  # (1+2+4) × 10; non-crop 99s excluded
+    )
+
+
+def test_from_xarray_masked_out_by_glad() -> None:
+    dset = xarray.Dataset(
+        {
+            "emissions-per-hectare:tco2e-per-ha": get_darray_for_data([[1, 2], [3, 4]]),
+            "hectares-per-pixel:ha": get_darray_for_data([[10, 10], [10, 10]]),
+            "gfw:global-peatlands:is-peatland": get_darray_for_data([[1, 1], [1, 1]]),
+            "land-class:2020": get_darray_for_data([[0] * 2] * 2),
+            "peatland-occupation:tco2e-per-ha": get_darray_for_data([[5, 5], [5, 5]]),
+            "usda-nass:cdl:crop-class": get_darray_for_data([[1, 1], [1, 1]]),
+        }
+    )
+    result = JurisdictionalCropEmission.from_dset(
+        admin_id="USA008",
+        admin_level=AdminLevel.PROVINCIAL,
+        crop=Crop.CORN,
+        dset=dset,
+        jurisdiction_name="Delaware",
+        skip_glad_crop_filter=False,
+    )
+    assert result.crop_hectares == 0.0
+    assert result.peatland_crop_hectares == 0.0
+    assert result.peatland_occupation_emissions == 0.0
+    assert result.total_emissions == 0.0
+
+
+@pytest.mark.parametrize("glad_value", (0, 1, 2))
+@pytest.mark.parametrize("skip_glad_crop_filter", (False, True))
+def test_from_xarray_with_no_crop_pixels(
+    glad_value: int, skip_glad_crop_filter: bool
+) -> None:
+    dset = xarray.Dataset(
+        {
+            "emissions-per-hectare:tco2e-per-ha": get_darray_for_data([[1, 2], [3, 4]]),
+            "hectares-per-pixel:ha": get_darray_for_data([[10, 10], [10, 10]]),
+            "gfw:global-peatlands:is-peatland": get_darray_for_data([[1, 1], [1, 1]]),
+            "land-class:2020": get_darray_for_data([[glad_value] * 2] * 2),
+            "peatland-occupation:tco2e-per-ha": get_darray_for_data([[5, 5], [5, 5]]),
+            "usda-nass:cdl:crop-class": get_darray_for_data([[0, 0], [0, 0]]),
+        }
+    )
+    result = JurisdictionalCropEmission.from_dset(
+        admin_id="USA008",
+        admin_level=AdminLevel.PROVINCIAL,
+        crop=Crop.CORN,
+        dset=dset,
+        jurisdiction_name="Delaware",
+        skip_glad_crop_filter=skip_glad_crop_filter,
+    )
+    assert result.crop_hectares == 0.0
+    assert result.peatland_crop_hectares == 0.0
+    assert result.peatland_occupation_emissions == 0.0
+    assert result.total_emissions == 0.0
+
+
+def test_from_constituents() -> None:
+    constituents = [
+        JurisdictionalCropEmission(
+            admin_id="",
+            admin_level="",
+            crop_hectares=value,
+            crop_name="CROP",
+            jurisdiction_name="",
+            peatland_crop_hectares=value,
+            peatland_occupation_emissions=value,
+            total_emissions=value,
+        )
+        for value in (1, 2, 3)
+    ]
+    result = JurisdictionalCropEmission.from_constituents(
+        admin_id="ADMIN_ID",
+        admin_level=AdminLevel.NATIONAL,
+        constituents=constituents,
+        jurisdiction_name="JURISDICTION_NAME",
+    )
+    assert result.admin_id == "ADMIN_ID"
+    assert result.admin_level == AdminLevel.NATIONAL.name
+    assert result.crop_hectares == 6
+    assert result.jurisdiction_name == "JURISDICTION_NAME"
+    assert result.peatland_crop_hectares == 6
+    assert result.peatland_occupation_emissions == 6
+    assert result.total_emissions == 6

jdluc/__tests__/test_datasets.py

@@ -0,0 +1,37 @@
+import pytest
+
+from jdluc.datasets.glad_glcluc import flatten_ranges
+from jdluc.datasets.worldbank_jurisdictions import get_ten_degree_tile_ids_for_admin_id
+
+
+def test_flatten_ranges() -> None:
+    assert flatten_ranges(range(0, 5), range(5, 10), range(10, 15)) == list(range(15))
+
+
+@pytest.mark.integration
+def test_get_ten_degree_tile_ids_for_country() -> None:
+    iso_a3 = "BRA"  # Brazil
+    expected = (
+        "00N_040W",
+        "00N_050W",
+        "00N_060W",
+        "00N_070W",
+        "00N_080W",
+        "10N_050W",
+        "10N_060W",
+        "10N_070W",
+        "10N_080W",
+        "10S_040W",
+        "10S_050W",
+        "10S_060W",
+        "10S_070W",
+        "10S_080W",
+        "20S_030W",
+        "20S_050W",
+        "20S_060W",
+        "30S_060W",
+    )
+    assert (
+        tuple(get_ten_degree_tile_ids_for_admin_id(admin_id=iso_a3, admin_level=0))
+        == expected
+    )