diff --git a/source/conf.py b/source/conf.py index b3bf17c..e25fa87 100644 --- a/source/conf.py +++ b/source/conf.py @@ -60,6 +60,7 @@ myst_enable_extensions = ["attrs_block", "substitution"] myst_substitutions = { + "stac_api_url": f"{BASE_URL}/api/stac/v1", "datastore_host": os.getenv("DATASTORE_HOST", "api.earthdatahub.destine.eu"), "datastore_internal_host": os.getenv( "DATASTORE_INTERNAL_HOST", "data.earthdatahub.destine.eu" diff --git a/source/data-access.md b/source/data-access.md index c8044dc..c30afb5 100644 --- a/source/data-access.md +++ b/source/data-access.md @@ -1,7 +1,7 @@ --- myst: html_meta: - description lang=en: This page explains how to access Earth Data Hub datasets using open standards such as Xarray and Zarr, including environment setup, API key authentication, protected datasets, quotas, and local caching. + description lang=en: This page explains how to access Earth Data Hub datasets using open standards such as Xarray and Zarr, including environment setup, API key authentication, protected datasets, quotas, and local caching, plus the STAC catalogue API and direct Zarr access over HTTPS. --- # {edh-octicon}`download` Data Access @@ -230,3 +230,145 @@ ds = xr.open_dataset( Behind the scenes, caching is handled by the **fsspec** library. The `simplecache` backend is a great default for most workflows, while other caching backends provide additional features such as customisable cache expiration and size limits. To learn more about available caching options, see the [fsspec documentation](https://filesystem-spec.readthedocs.io/en/latest/features.html#caching-files-locally). + +## API reference + +Under the hood, Earth Data Hub is just two open interfaces: a **STAC catalogue** to find data, and **Zarr over HTTPS** to read it. No SDK required. If your tool can make a web request, it can use Earth Data Hub. + +### STAC catalogue + +[STAC](https://stacspec.org/) (SpatioTemporal Asset Catalog) is a standard, searchable catalogue of Earth data. Every collection and dataset you see in the Earth Data Hub catalogue is also available as STAC metadata, starting from a single root URL: {{ stac_api_url }} + +```{admonition} One URL to discover everything. +--- +class: note +--- +The STAC catalogue is **public**: browsing it requires no API key. You only need your key when you read the data itself. +``` + +Explore the catalogue straight from your terminal: + +```{code-block} bash +--- +substitutions: +--- +# Root catalogue +curl |stac_api_url|/ + +# All collections (e.g. era5, cmip6, climate-dt-2) +curl |stac_api_url|/collections + +# All datasets in a collection +curl |stac_api_url|/collections/era5/items + +# Search across the catalogue +curl "|stac_api_url|/search?collections=era5&limit=5" +``` + +For the full list of endpoints and query parameters, see the {edh_url}`interactive STAC API documentation `. + +In STAC, each dataset is an **item**. Its `data` asset holds the **dataset URL**, along with the exact arguments needed to open it with Xarray: + +```{code-block} json +--- +emphasize-lines: 3 +--- +"assets": { + "data": { + "href": "https://data.earthdatahub.destine.eu/era5/era5-land-monthly-v0.zarr", + "roles": ["data"], + "xarray:open_kwargs": {"chunks": {}, "engine": "zarr"}, + "xarray:storage_options": {"client_kwargs": {"trust_env": true}} + } +} +``` + +#### From catalogue to dataset in Python + +Install [pystac-client](https://pystac-client.readthedocs.io/) alongside the packages from [](environment-setup): + +````{tab-set} +--- +class: outline +--- +~~~{tab-item} {iconify}`vscode-icons:file-type-pypi` pip +```bash +pip install pystac-client xarray "zarr>3" dask aiohttp +``` +~~~ + +~~~{tab-item} {iconify}`vscode-icons:file-type-uv` uv +```bash +uv add pystac-client xarray "zarr>3" dask aiohttp +``` +~~~ + +~~~{tab-item} {iconify}`vscode-icons:file-type-conda` conda +```bash +conda install -c conda-forge pystac-client xarray "zarr>3" dask aiohttp +``` +~~~ +```` + +Then find a dataset in the catalogue and open it, with no URLs to copy and paste. This example uses your `.netrc` file, see [](protected-datasets): + +```{code-block} python +--- +substitutions: +--- +import pystac_client +import xarray as xr + +catalog = pystac_client.Client.open("|stac_api_url|") +item = catalog.get_collection("era5").get_item("era5-land") +asset = item.assets["data"] + +xr.open_dataset( + asset.href, + storage_options=asset.extra_fields.get("xarray:storage_options"), + **asset.extra_fields.get("xarray:open_kwargs", {}), +) + +``` + +### Zarr over HTTPS + +A Zarr dataset is simply a **folder of small files** on the web: a few JSON files describing the data, plus the data itself split into chunks. Each of those files has its own URL, so a plain HTTPS request is all it takes to read one. + +Try it with the public test dataset: + +```{code-block} bash +--- +substitutions: +--- +# Dataset metadata: variables, dimensions and attributes +curl https://|datastore_internal_host|/public/test-dataset-v1.zarr/zarr.json + +# Variable metadata: shape, chunk shape, data type and compression +curl https://|datastore_internal_host|/public/test-dataset-v1.zarr/demographic_totals/zarr.json + +# A single chunk of raw data +curl -o chunk.bin https://|datastore_internal_host|/public/test-dataset-v1.zarr/year/c/0 +``` + +File names follow the standard [Zarr conventions](https://zarr-specs.readthedocs.io/en/latest/v3/core/index.html). To learn more about how chunks are organised, see [](chunking-schemes). + +For protected datasets, add your API key to the request: + +```{code-block} bash +--- +substitutions: +--- +# Using your API key directly +curl -u edh: https://|datastore_internal_host||private_test_dataset_path|/zarr.json + +# Using your API key (with bearer auth) +curl https://|datastore_internal_host||private_test_dataset_path|/zarr.json -H "Authorization: Bearer " + +# Using your .netrc file +curl --netrc https://|datastore_internal_host||private_test_dataset_path|/zarr.json +``` + +```{note} +Every request counts towards your monthly quota, whichever tool you use. See [](quotas). +```