feat: DPS STAC user catalogs and STAC Browser deployment (#133)

Author: hrodmn

.github/workflows/deploy.yml

@@ -40,6 +40,8 @@ jobs:
       STAC_BROWSER_CERTIFICATE_ARN: ${{ vars.STAC_BROWSER_CERTIFICATE_ARN }}
       STAC_BROWSER_CUSTOM_DOMAIN_NAME: ${{ vars.STAC_BROWSER_CUSTOM_DOMAIN_NAME }}
       STAC_BROWSER_REPO_TAG: ${{ vars.STAC_BROWSER_REPO_TAG }}
+      USER_STAC_BROWSER_CERTIFICATE_ARN: ${{ vars.USER_STAC_BROWSER_CERTIFICATE_ARN }}
+      USER_STAC_BROWSER_CUSTOM_DOMAIN_NAME: ${{ vars.USER_STAC_BROWSER_CUSTOM_DOMAIN_NAME }}
       STAGE: ${{ github.event.inputs.deployment_environment || 'synthtest' }}
       TITILER_DATA_ACCESS_ROLE_ARN: ${{ vars.TITILER_DATA_ACCESS_ROLE_ARN }}
       TITILER_PGSTAC_API_CUSTOM_DOMAIN_NAME: ${{ vars.TITILER_PGSTAC_API_CUSTOM_DOMAIN_NAME }}

.gitignore

@@ -19,3 +19,4 @@ stac-browser/
 .pgdata/
 
 dev-docs/plans/
+issue-drafts/

Dockerfile.stac-browser

@@ -0,0 +1,29 @@
+FROM node:lts-alpine AS build
+ARG STAC_BROWSER_REPO_TAG=v5.1.0
+ARG DYNAMIC_CONFIG=true
+
+RUN apk add --no-cache git
+WORKDIR /app
+RUN git clone --depth 1 --branch "$STAC_BROWSER_REPO_TAG" https://github.com/radiantearth/stac-browser.git .
+COPY cdk/stac_browser_user_config.js ./stac_browser_user_config.js
+RUN npm install
+ENV DYNAMIC_CONFIG="$DYNAMIC_CONFIG" \
+    SB_CONFIG=./stac_browser_user_config.js
+RUN [ "$DYNAMIC_CONFIG" = "true" ] && sed -i 's/<!--RC//;s/RC-->//' index.html
+RUN npm run build
+
+FROM nginxinc/nginx-unprivileged:1-alpine
+
+USER root
+RUN apk add --no-cache jq pcre-tools
+COPY --from=build /app/config.schema.json /etc/nginx/conf.d/config.schema.json
+COPY --from=build /app/dist /usr/share/nginx/html
+COPY --from=build /app/docker/default.conf /etc/nginx/conf.d/default.conf.template
+COPY --from=build /app/docker/docker-entrypoint.sh /docker-entrypoint.d/40-stac-browser-entrypoint.sh
+RUN rm -f /etc/nginx/conf.d/default.conf && \
+    chown -R nginx:nginx /usr/share/nginx/html && \
+    chmod +x /docker-entrypoint.d/40-stac-browser-entrypoint.sh
+
+ENV SB_pathPrefix=/
+EXPOSE 8080
+USER nginx

README.md

@@ -32,6 +32,37 @@ asset containing the source `.met.json` file. The generator also overwrites the
 STAC Common Metadata `created` property with the UTC publication time shared by
 all Items generated from that catalog.
 
+For each job with generated items, the same SNS stream also receives a plain
+STAC 1.1.0 user Catalog and one Collection. If the input catalog contains one
+source Collection for those generated items, its useful metadata and resolved
+asset links are reused with the deterministic collection ID and the URL-safe
+user catalog ID as its only `parent_ids` value. With no source Collection, the
+Collection uses whole-world, open-ended extents. Multiple source Collections
+for one job fail rather than being merged. Authorized named collections remain
+Item-only.
+
+Repeated upserts overwrite manual curation on generated Catalog and Collection
+records. The deployed loader still has `CREATE_COLLECTIONS_IF_MISSING=TRUE`,
+and pgSTAC is configured to maintain collection extents from ingested Items, so
+a source extent is initial metadata and may be updated asynchronously from
+Items. Named collections are untouched.
+
+To add hierarchy records for historical generated collections, preview this
+conservative, restartable backfill before applying it:
+
+```bash
+uv run --script scripts/backfill_dps_user_catalogs.py --dry-run
+uv run --script scripts/backfill_dps_user_catalogs.py --apply
+```
+
+The backfill uses hydrated item metadata and actual collection IDs. It recognizes
+both current three-part and legacy tag-specific four-part generated IDs, and
+skips named, authorized, mixed, incomplete, and ambiguous collections. Historical
+authorization cannot always be proven when its registry is incomplete, so review
+the dry-run report. Existing Collection metadata is preserved; apply only adds
+the parent relationship and creates a missing user Catalog. It does not rewrite
+or rename Items.
+
 To merge legacy tag-specific DPS collections into these tag-free IDs, preview
 then apply the database migration:
 
@@ -42,10 +73,10 @@ then apply the database migration:
 
 It recognizes four-part IDs (`username__algorithm__version__tag`), merges their
 items into the corresponding three-part ID, and adds the DPS metadata fields
-from the legacy ID. Collections containing an item-ID collision after merging
-are reported and left unchanged. For a deployed database, follow the
-[RDS connection guide](#connect-to-rds-through-an-ssm-tunnel) below and the
-RDS usage instructions in the migration script's docstring.
+from the legacy ID. Collections containing an item-ID collision retain their
+legacy ID, but their Items still receive those metadata fields. For a deployed
+database, follow the [RDS connection guide](#connect-to-rds-through-an-ssm-tunnel)
+below and the RDS usage instructions in the migration script's docstring.
 
 Collection-only STAC transactions can still be enabled with:
 
@@ -74,6 +105,8 @@ The script is standalone and uses an inline `uv` execution header, so it install
 - `DPS Team Catalogs` as a root catalog, containing the shared `maap-demo-team` catalog
 - two synthetic DPS-output collections per user
 
+Open <http://127.0.0.1:8080> to test the user STAC Browser configuration used by the deployment. Its landing page shows only root catalogs; opening a catalog shows its scoped collections.
+
 Useful options:
 
 ```bash
@@ -93,6 +126,16 @@ For a catalogs-enabled deployment, verify:
 - `GET /` includes `rel="child"` links for listed catalogs so STAC Browser can discover catalog roots.
 - catalog write routes are absent unless `USER_STAC_CATALOG_TRANSACTIONS_AUTH_MODE=basic` is configured.
 
+The deployment includes a public STAC Browser and, when
+`USER_STAC_BROWSER_CUSTOM_DOMAIN_NAME` and `USER_STAC_BROWSER_CERTIFICATE_ARN` are
+set, a separate user-STAC Browser. Set `STAC_BROWSER_REPO_TAG` to `v5.1.0` (or a
+compatible STAC Browser v5 release) for the user browser configuration. On the
+user-STAC Browser landing page, verify that catalog links are shown and the
+landing page's broad `rel="data"` link is not. Open a child catalog and verify
+that its scoped `rel="data"` link still lists collections, then open a
+collection and verify that `rel="items"` lists its items. The customization
+uses the browser path for root detection, not an API landing-page ID.
+
 For a transaction-enabled internal deployment, verify:
 
 - `GET /conformance` includes `https://api.stacspec.org/v1.0.0/collections/extensions/transaction` when collection transactions are enabled.
@@ -166,7 +209,7 @@ STAC HTTP basic-auth secret. CloudFormation gives you the secret's identifier; r
 from Secrets Manager. In the same terminal:
 
 ```bash
-SECRET_ID='<database secret physical ID (not arn) from the table>'
+SECRET_ID='<database secret arn from the table>'
 DB_SECRET=$(aws secretsmanager get-secret-value \
   --secret-id "$SECRET_ID" --query SecretString --output text)
 

app.py

@@ -63,6 +63,7 @@
     pgstac_db_config=config.pgstac_db(),
     stac_api_config=config.user_stac_api(),
     titiler_pgstac_config=config.user_titiler_pgstac(),
+    stac_browser_config=config.user_stac_browser(),
     add_stactools_item_generator=False,
     **(
         {"dps_stac_item_gen_config": dps_stac_item_gen_config}  # type: ignore [arg-type]

cdk/config.py

@@ -95,6 +95,8 @@ class Config(BaseSettings):
     stac_browser_repo_tag: str
     stac_browser_custom_domain_name: str
     stac_browser_certificate_arn: str
+    user_stac_browser_custom_domain_name: str | None = None
+    user_stac_browser_certificate_arn: str | None = None
     stac_api_custom_domain_name: str
     pgstac_version: str
     web_acl_arn: str
@@ -187,6 +189,14 @@ def validate_required_pairs(self) -> Config:
                 "user_stac_catalog_transactions_auth_secret_arn",
                 "user_stac_catalog_transactions_auth_mode",
             ),
+            (
+                "user_stac_browser_custom_domain_name",
+                "user_stac_browser_certificate_arn",
+            ),
+            (
+                "user_stac_browser_certificate_arn",
+                "user_stac_browser_custom_domain_name",
+            ),
             (
                 "user_stac_catalog_transactions_auth_mode",
                 "user_stac_catalogs_enabled",
@@ -314,6 +324,17 @@ def stac_browser(self) -> StacBrowserConfig:
             certificate_arn=self.stac_browser_certificate_arn,
         )
 
+    def user_stac_browser(self) -> StacBrowserConfig | None:
+        """Build user-STAC Browser settings when its domain is configured."""
+        if self.user_stac_browser_custom_domain_name is None:
+            return None
+        assert self.user_stac_browser_certificate_arn is not None
+        return StacBrowserConfig(
+            repo_tag=self.stac_browser_repo_tag,
+            custom_domain_name=self.user_stac_browser_custom_domain_name,
+            certificate_arn=self.user_stac_browser_certificate_arn,
+        )
+
     def ingestor(self) -> IngestorConfig:
         return IngestorConfig(
             jwks_url=self.jwks_url,

cdk/constructs/DpsStacItemGenerator/runtime/pyproject.toml

@@ -19,6 +19,7 @@ dependencies = [
 dev = [
     "boto3>=1.43.42",
     "httpx>=0.28.1",
+    "psycopg>=3.3.6",
     "pytest>=9.0.3",
     "pytest-mock>=3.14.0",
 ]

cdk/constructs/DpsStacItemGenerator/runtime/src/dps_stac_item_generator/handler.py

@@ -8,7 +8,7 @@
 import boto3
 from pydantic import ValidationError
 
-from dps_stac_item_generator.item import get_stac_items
+from dps_stac_item_generator.stac import get_stac_documents
 
 if TYPE_CHECKING:
     from aws_lambda_typing.context import Context
@@ -128,6 +128,7 @@ def handler(
     )
 
     batch_item_failures: list[BatchItemFailure] = []
+    published_hierarchy_documents: set[tuple[str, str]] = set()
 
     for record in records:
         message_id = record.get("messageId")
@@ -144,21 +145,34 @@ def handler(
             logger.debug(f"[{message_id}] SNS Message content: {message_str}")
 
             catalog_json_key = get_catalog_json_key(message_str)
-            for stac_item in get_stac_items(
+            for stac_document in get_stac_documents(
                 catalog_json_key,
                 collection_id_registry=COLLECTION_ID_REGISTRY,
             ):
-                stac_item_json = stac_item.model_dump_json()
+                if isinstance(stac_document, dict):
+                    document_key = (stac_document["type"], stac_document["id"])
+                    if document_key in published_hierarchy_documents:
+                        continue
+                    stac_document_json = json.dumps(stac_document)
+                    document_description = (
+                        f"STAC {stac_document['type'].lower()} {stac_document['id']}"
+                    )
+                else:
+                    document_key = None
+                    stac_document_json = stac_document.model_dump_json()
+                    document_description = f"STAC item {stac_document.id}"
 
                 item_load_topic_arn = get_topic_arn()
                 logger.info(
-                    f"[{message_id}] Publishing STAC item {stac_item.id} "
+                    f"[{message_id}] Publishing {document_description} "
                     f"to {item_load_topic_arn}"
                 )
                 response = sns_client.publish(
                     TopicArn=item_load_topic_arn,
-                    Message=stac_item_json,
+                    Message=stac_document_json,
                 )
+                if document_key is not None:
+                    published_hierarchy_documents.add(document_key)
                 logger.info(
                     f"[{message_id}] SNS publish response MessageId: "
                     f"{response.get('MessageId')}"