From edb56a39ba623683acefccd3da6107912fc2be60 Mon Sep 17 00:00:00 2001 From: vthwang Date: Tue, 20 Oct 2026 18:15:08 -0700 Subject: [PATCH 1/7] docs: plan optional trust registry integration Define in-place enablement so existing full stacks can add a registry without rebuilding. Cover configuration, identity, Vault, recovery, and acceptance requirements. Signed-off-by: vthwang --- docs/full-stack-setup-design.md | 5 + docs/optional-trust-registry-design.md | 355 +++++++++++++++++++++++++ 2 files changed, 360 insertions(+) create mode 100644 docs/optional-trust-registry-design.md diff --git a/docs/full-stack-setup-design.md b/docs/full-stack-setup-design.md index b852c0c..107fba4 100644 --- a/docs/full-stack-setup-design.md +++ b/docs/full-stack-setup-design.md @@ -12,6 +12,11 @@ https://dids-{vta_name}.{domain} ← WebVH DID Hosting daemon (admin panel https://vtc-{vtc_name}.{domain} ← VTC REST + admin SPA + public website ``` +**Planned extension:** [Optional Trust Registry](optional-trust-registry-design.md) +adds an opt-in fifth component, including in-place enablement for existing running +stacks without teardown. It is not implemented yet; the four-component baseline +and current behavior below remain the reference for deployed sessions. + **The VTC is not optional.** Every `full_stack` session provisions all four components in one pipeline run. There is no VTC-less variant and no separate `full_stack_with_vtc` mode — an earlier iteration of this farm shipped both diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md new file mode 100644 index 0000000..cb67fd6 --- /dev/null +++ b/docs/optional-trust-registry-design.md @@ -0,0 +1,355 @@ +# Optional Trust Registry for Full Stack — Design + +Status: **planned; not implemented**. Research checked on 2026-10-20. + +Add a per-session Trust Registry to `full_stack` as an opt-in component, both +when creating a stack and when adding it to an existing running stack. Existing +stacks must keep their identities, member data, volumes, URLs, and installation. +Enabling the registry must never require tearing down or re-running VTC setup. + +This extends [full-stack-setup-design.md](full-stack-setup-design.md), whose +four-component pipeline describes the currently implemented baseline. + +## 1. What the registry adds + +The VTC remains responsible for membership, admission policy, roles, and issuing +credentials. The Trust Registry publishes queryable trust records that external +verifiers and peer communities can consult. In this integration the VTC writes +its membership projection; the registry is not the authoritative member store. + +The registry can also represent authorization and recognition relationships. +Deploying it does not automatically configure trust between communities, grant +issuer powers, or make another verifier trust this community. + +The upstream VTC explicitly supports **no-registry mode**: when neither +`registry.did` nor `registry.url` is configured, it constructs no registry client +and does not start membership synchronization. The VTC continues running. Features +that require registry access, including registry-backed cross-community +recognition, are unavailable. Upstream currently reports the registry status as +`degraded` in this mode; the Farm should display **Not enabled** for an opted-out +component rather than treating the whole stack as broken. + +Sources: [VTC runtime client selection][vtc-server], +[runtime configuration][vtc-config], [integration guide][vtc-guide]. These +describe the reviewed upstream revision, not a runtime inspection of production. + +## 2. Scope and compatibility + +| Scenario | Planned behavior | +| --- | --- | +| New `full_stack`, option omitted or false | Existing four-component flow | +| New `full_stack`, option true | Existing components plus a dedicated registry | +| Existing running `full_stack` | Owner explicitly enables the add-on in place | +| Existing platform stack | Equivalent action in the platform admin surface | +| `vta_only` | Reject registry provisioning; no change to this mode | + +Keep the modes `vta_only` and `full_stack`. Do not introduce another setup mode. +The default is off, including for migrated rows and existing API clients. +Deploying the new Farm API must not enqueue enablement for old sessions. + +First delivery includes managed and custom domains, new and existing Full Stacks, +status/progress, retry, image upgrades, backup/export inventory, and teardown. +External registries, shared multi-community registries, public REST ingress, +standalone registry deployments, and a user-facing disable/delete-add-on action +are outside this delivery. Failure recovery must still restore the prior VTC +configuration when possible, without deleting existing stack data. + +## 3. Image and version preflight + +Use **`ghcr.io/ic3software/trust-registry`**, supplied by the user: +[container package][image-package]. Integrate its immutable tags into the same +GHCR selection mechanism as other components; persist the selected immutable +image reference. Do not hard-code the Dockerfile's default `0.8.0` as the selected +version or automatically use a moving `nightly` tag. + +Research found a successful build publishing `0.28.0-c534746` and `nightly` on +2026-10-20 ([build run][image-run]). This is evidence of publication, not an +end-to-end compatibility result or a final version selection. + +The [image Dockerfile][image-dockerfile] downloads a precompiled +`trust-registry-k8s` binary. Unlike the upstream Dockerfile, this repository does +not compile Rust or declare Cargo features. Check the binary producer and verify +the actual selected image supports `vta`, `storage-fjall`, `secrets-vault`, +`standalone`, and TSP if advertised. Do not infer missing VTA support from the +upstream default build, or infer its presence from a successful image build. + +The [image entrypoint][image-entrypoint] currently holds the container when +DIDComm is enabled and `PROFILE_CONFIG` is absent. It does not recognize +`TR_VTA_CREDENTIAL` as an alternative. The Farm will explicitly run +`trust-registry` with its own working directory, as it already does for other +components, bypassing this guard. Do not inject a dummy `PROFILE_CONFIG`. +Align the image entrypoint separately if default-entrypoint support is wanted. + +Before enabling an existing stack, inspect its recorded and deployed images for +VTC messaging-based registry writes, VTA provisioning/DID editing, and compatible +mediator protocols. Upgrade only an incompatible component through the existing +upgrade flow. A newer registry image alone cannot add missing VTC capabilities. + +## 4. Identity, storage, and configuration + +Each enabled session gets its own registry Deployment and PVC, named consistently +with the existing `fs-{sid}-*` resources, proposed `fs-{sid}-trust-registry`. +Use one replica and Recreate updates: fjall must never have concurrent processes +opening one directory. Store records at an absolute path such as +`/work/trust-registry/data` on the registry PVC. + +Reuse the session's VTA, mediator, and DID host. Give the registry a dedicated +VTA context and a hosted `did:webvh`, with a deterministic path such as +`{vtc_name}-trust-registry`. The context and DID path are selected once, checked +for collisions, persisted, and reused on retry. Do not reuse the VTC context's +keys or create another community identity. + +No new public hostname, Cloudflare record, Ingress, or TLS SAN is needed initially. +Publish the registry's messaging services in its DID document and verify those +advertisements match the actual binary. Clients reach it through the mediator. +A ClusterIP Service may expose port 3232 internally. Bind to `0.0.0.0:3232` for +Pod-IP HTTP probes; the explore guide's `127.0.0.1` bind cannot serve those probes. +Keep internal access subject to the existing namespace/network policy. + +Required configuration belongs to the service that consumes it: + +| Consumer | Information to add | +| --- | --- | +| Registry environment | VTA credential reference, context ID, mediator DID, storage path, Vault cache settings, enabled transports | +| Registry `ADMIN_DIDS` | The existing **VTC service DID**, which signs record writes | +| VTC runtime `config.toml` | `[registry].did` pointing to the registry DID | +| VTC DID document | A `TrustRegistry` referral to that same registry DID | +| VTA | Registry context, keys, and appropriately scoped provisioning/runtime ACLs | + +For example, the VTC runtime change is a merge into its existing TOML: + +```toml +[registry] +did = "did:webvh:::" +``` + +The runtime setting enables the VTC client. The DID-document referral lets other +clients discover the registry. They are separate changes: setup's `registry_did` +only publishes the referral and does not configure runtime synchronization. +Neither the VTC admin DID nor the owner's PNM DID substitutes for the VTC service +DID in `ADMIN_DIDS`. + +Do not regenerate other service configurations. The mediator and DID host are +reused through their normal account/publication operations, with existing policy +changes only if preflight proves they are necessary. + +### Vault: two separate secrets + +1. **VTA access credential**: used by the registry to authenticate to its context. +2. **Identity bundle cache**: fetched from the VTA and used for messaging and + offline startup. This contains private key material. + +Store both in separate keys/paths beneath a proposed +`trust-registry/user-{uid}/session-{sid}` Vault prefix. Extend per-user Vault +policy and the API's bootstrap cleanup permissions for this prefix. Reuse +Kubernetes authentication; do not introduce static Vault tokens or plaintext +credential/cache persistence on the PVC. + +The registry supports Vault for the cache through `TR_SECRETS_VAULT_*` settings +when built with `secrets-vault`. Its `TR_VTA_CREDENTIAL` loader separately supports +file/string/AWS sources; the reviewed loader has no Vault URI implementation. +Therefore selecting Vault for the cache does not solve credential delivery. + +Planned credential delivery: a pod-local init helper authenticates to Vault using +the ServiceAccount, reads the context-scoped credential, and writes a `0600` file +to a memory-backed `emptyDir`. The registry receives a `file://` reference to that +file. Persist the credential only in Vault; keep values out of Deployment env, +ConfigMaps, API responses, and Job logs. This needs init-container and temporary +volume support in the component helper, plus a pinned helper image/client. +The cache uses a different Vault location so it cannot overwrite the credential. + +The registry obtains and uses private keys at runtime. The setup guide's phrase +"no private keys of its own" means VTA-managed identity, not remote-only signing. +Sources: [VTA startup integration][tr-vta], [secret backend][tr-secrets], +[credential loader][tr-loaders]. + +## 5. New-stack provisioning + +When opted out, do not add registry resources, Jobs, Vault paths, or setup stages. +When opted in, extend the post-admin-gate flow: + +```text +existing bootstrap and admin gate + -> deploy VTA (mediator and DID host already ready) + -> provision registry context, DID, credential, and cache configuration + -> VTC setup with registry_did (publish referral) + -> configure registry ADMIN_DIDS with the newly created VTC DID + -> deploy registry and check its messaging/write-path status + -> merge [registry].did into generated VTC runtime config + -> deploy VTC + -> verify registry round trips and synchronization readiness + -> running +``` + +Use headless provisioning Jobs or an SDK client with an explicit service grant. +Do not assume PNM is installed/configured in the API or copy a human's PNM +credential into the Farm. Exact provisioning commands and the pinned client are +an implementation preflight item. Preserve existing VTC output/claim handling; +account for its short install-token TTL when adding steps after VTC setup. + +## 6. Add to an existing stack without rebuilding + +Expose an **Enable Trust Registry** action for a running Full Stack. Keep the +parent session running and report add-on progress separately. Never call +`vtc setup`, recreate the VTA seed/context, reissue member credentials, replace +existing PVCs, or re-enroll the existing administrators. + +1. Authorize the owner (or the platform admin for the platform stack), acquire + the session maintenance lock, and check capacity, versions, configuration, + DID ownership/history, and any active upgrade/recovery operation. +2. Persist the operation and chosen image/identity parameters before mutation. + Provision only the registry resources and Vault locations. +3. Establish a short-lived setup identity authorized for registry provisioning + and the required existing VTC DID update. Revoke it after completion; the + registry's lasting credential is scoped to its own context. +4. Provision the registry identity through the existing VTA and publish it on + the existing DID host. Reuse a completed result on retry. +5. Set `ADMIN_DIDS` to the existing `session.VtcDid`, start the registry, and + verify its advertised transport and write path. +6. Append a signed update to the **existing** VTC DID history through the VTA, + adding the referral while preserving the DID, keys, other services, and + history. Publish the extended log through the existing DID host. A self-hosted + VTC log, if present, must also be updated using the supported install path. +7. Read the current VTC runtime config immediately before writing, merge only + `[registry].did`, and restart the VTC using the existing component-config + backup/write/start/rollback pattern. Do not render a fresh setup config over + the customized one or discard other keys already in `[registry]`. +8. Verify the VTC's selected transport, correlated replies, write authorization, + and sync diagnostics; reconcile existing members (§8). Mark the add-on active. + +Upstream explicitly supports changing the referral after initial mint via +VTA-side DID editing ([setup schema][vtc-setup]). This requires a signed DID-log +update, not a change to a locally served JSON file alone. + +**Downtime:** the VTC needs a restart for the new client/syncer. Provisioning +authorization may also need a brief VTA maintenance window: current Farm ACL +operations use `scale to 0 -> offline Job -> restart`. Prefer compatible live +APIs where a suitable credential already exists, but do not promise zero VTA +downtime or bypass the fjall lock. DID publication should use the live hosting +interface. Always restart a temporarily stopped component on error. Preserve +shared mediator/DID-host service for agents attached to this stack. + +The existing config handler operates on the selected component only and backs +up/restores its config (`internal/handler/setup_config.go`). Reuse that behavior +for the VTC; its readiness check is not by itself a registry-write acceptance test. + +## 7. Proposed API, persistence, and retries + +Add optional `trust_registry_enabled` and `trust_registry_image` to Full Stack +creation. Require an allowed immutable image only when enabled; old requests +retain their behavior. Reject registry enablement for `vta_only`. + +Proposed add-on routes (under `/api/v1`): + +| Method | Route | Behavior | +| --- | --- | --- | +| POST | `/setup/:id/trust-registry` | Enable or explicitly retry with an image; returns 202 and progress | +| GET | `/setup/:id/trust-registry` | Desired state, phase, identity, image, health, and sanitized error | +| POST/GET | `/admin/platform-stack/trust-registry` | Same operations, restricted to the platform stack | + +Apply existing Full Stack entitlement checks on creation and session ownership +checks on existing-stack actions. Preserve current management access semantics; +do not expose general customer-stack mutation through a platform-admin route. Reject concurrent +maintenance, upgrade, recovery, teardown, and duplicate conflicting operations. +An already-active request with the same image is an idempotent success; image +changes use the existing upgrade machinery rather than re-provisioning. + +Persist desired enablement (default false), image, context/path, registry DID, +add-on state (`disabled`, `enabling`, `active`, `failed`), resumable phase, +timestamps, and a sanitized error. No credentials/private keys in these columns. +For an existing stack, add-on failure must not mark the parent setup `failed`. +For new opted-in stacks, provisioning is complete only after the add-on is ready. + +Use a durable per-session operation claim, compatible with existing maintenance +locking and startup resumption. A client timeout or API restart must not lose the +operation or allow another worker to execute it concurrently. Reconcile completed +steps before retrying: the sealed bootstrap request is single-use, context creation +may already have succeeded, and DID-log updates must not mint another DID. + +Before VTC wiring, failure leaves the original stack configuration intact. +After wiring, restore the exact saved runtime config if necessary and restart +the VTC. DID history is append-only: reverting a referral requires another signed +update, never restoring an old log over newer history. Retain partially created +registry resources and credentials for a visible retry; do not delete a registry +that a published referral still points to. Report incomplete restoration explicitly. + +## 8. Existing members, health, and lifecycle + +Check the persisted audit cursor, pending/failed jobs, current roster, and existing +registry mirror when enabling an old stack. Upstream can replay retained audit +history from a missing cursor, but that does not guarantee a complete historical +backfill after retention or previous partial synchronization. Reconcile against +the current roster and queue only eligible missing records through supported VTC +operations. Do not replay stale removals as fresh grants or reset cursors blindly. + +Honor member `registryConsent`/`publishConsent` and publication policy. Enabling +the component cannot grant consent on a member's behalf. Report synchronization +progress separately from service readiness; a large backlog need not block all +community use. If the pinned VTC has no supported reconciliation operation, that +is a delivery prerequisite to resolve, not a reason to rebuild the community. + +HTTP `/health` can return 200 with `status: degraded`. Monitor body status and +write-path details, plus VTC sync queue/failure information. A successful read-only +probe cannot prove VTC write authorization. Verify signed put/query/delete in an +isolated test scope and exercise the VTC sync path with a consented test member +in staging. Do not publish synthetic members into a customer's live trust graph. + +Include the component wherever deployed-component inventory is used: session +responses, progress/logs, version selection, self-service/admin upgrades, recovery, +resource profiles/capacity, backup/export, and full-session teardown. Capacity must +distinguish opted-out stacks from five-component stacks and admit an add-on using +its incremental cost. Set resource requests/limits and PVC size from measurements; +do not invent production sizing from the explore guide. + +Back up the record PVC and account for both Vault secrets. Extend deletion to the +registry Jobs, Deployment, Service, PVC, and session Vault prefix. Audit policies +and bootstrap permissions must agree on the new prefix. A Farm rollback or a +down-migration must not silently orphan running registries or drop their data. + +## 9. Implementation order and acceptance + +1. Verify pinned image features/entrypoint, production component compatibility, + headless provisioning and DID editing, and the existing-member reconciliation + mechanism. Resolve these gates before touching a live stack. +2. Implement opt-in persistence, inventory, Vault policy/credential delivery, + Kubernetes resources, capacity, and the shared resumable enablement steps. +3. Wire the new-stack branch and existing-stack enable/retry routes, maintenance + and upgrade exclusions, config restoration, and startup recovery. +4. Add owner/platform UI, OpenAPI, diagnostics, image upgrades, backup/export, + teardown, and operator documentation across the relevant repositories. +5. Validate in development/staging; then separately schedule any production + enablement. Writing this plan authorizes no live migration or image rollout. + +Required acceptance coverage: + +- Omitted/false options and migrated rows still create/run four-component stacks. +- Existing-stack enablement preserves original DIDs, keys, members, PVCs, URLs, + and admin installation; only documented maintenance restarts occur. +- Referral and runtime configuration agree; unauthorized writes are refused and + the existing VTC can publish, query, update, and remove a consented test member. +- Non-consenting members remain unpublished, including during initial backfill. +- Restarts preserve trust records; populated Vault cache supports the documented + VTA-offline case without plaintext persistence. +- Failures/restarts at credential handoff, DID publication, config update, and + deployment resume without duplicate contexts/DIDs or a stuck stopped VTC/VTA. +- Add-on failure remains visible without invalidating the existing parent stack; + config restoration and incomplete-referral restoration are accurately reported. +- Upgrades use a single writer; full teardown cleans only this session's registry + resources/secrets. Capacity and concurrent-operation guards cover add-on work. + +Run each changed repository's verification entry points. For this API, `make test` +runs PostgreSQL image parity, `go vet ./...`, and `go test ./...`. Container startup, +transport compatibility, and in-place enablement also need staging verification; +passing Go checks alone cannot establish them. + +[image-package]: https://github.com/ic3software/trust-registry-image/pkgs/container/trust-registry +[image-run]: https://github.com/ic3software/trust-registry-image/actions/runs/39370122443 +[image-dockerfile]: https://github.com/ic3software/trust-registry-image/blob/edb20bd3559ad12d1ebe7e36046f5eca9824a4b2/Dockerfile +[image-entrypoint]: https://github.com/ic3software/trust-registry-image/blob/edb20bd3559ad12d1ebe7e36046f5eca9824a4b2/entrypoint.sh +[vtc-server]: https://github.com/OpenVTC/verifiable-trust-infrastructure/blob/5b7757980945618f860e3a016eac8167ea475fc4/vtc-service/src/server.rs +[vtc-config]: https://github.com/OpenVTC/verifiable-trust-infrastructure/blob/5b7757980945618f860e3a016eac8167ea475fc4/vtc-service/src/config.rs +[vtc-guide]: https://github.com/OpenVTC/verifiable-trust-infrastructure/blob/5b7757980945618f860e3a016eac8167ea475fc4/docs/03-vtc/trust-registry.md +[vtc-setup]: https://github.com/OpenVTC/verifiable-trust-infrastructure/blob/5b7757980945618f860e3a016eac8167ea475fc4/vtc-service/src/setup/from_toml.rs +[tr-vta]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/vta.rs +[tr-secrets]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/secret_store.rs +[tr-loaders]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/loaders/mod.rs From 0217f15390a715d3c40542d529393478f4abd333 Mon Sep 17 00:00:00 2001 From: vthwang Date: Wed, 21 Oct 2026 03:47:56 -0700 Subject: [PATCH 2/7] feat: provision optional Trust Registry for Full Stack Enable dedicated registries on new and existing stacks while preserving community data. - Persist enablement state and deliver scoped credentials through Vault - Merge VTC configuration and publish signed Registry referrals - Integrate upgrades, resources, teardown, and failed Job cleanup Native hosted DID provisioning remains blocked by the upstream VTA CLI signer issue. Signed-off-by: vthwang --- docs/full-stack-setup-design.md | 5 +- docs/optional-trust-registry-design.md | 74 ++- internal/apidocs/openapi.yaml | 145 +++++- internal/capacity/capacity.go | 7 + internal/config/config.go | 2 + internal/handler/admin_platform_stack.go | 36 +- .../handler/admin_platform_stack_admins.go | 25 + internal/handler/admin_upgrade.go | 21 +- internal/handler/capacity_service.go | 18 +- internal/handler/resource_defaults.go | 20 +- internal/handler/setup.go | 34 +- internal/handler/setup_export.go | 10 +- internal/handler/setup_fullstack.go | 38 ++ internal/handler/setup_resources.go | 18 +- internal/handler/setup_trust_registry.go | 224 +++++++++ internal/handler/setup_trust_registry_test.go | 68 +++ internal/handler/user_upgrade.go | 6 +- internal/k8s/component_jobs.go | 17 + internal/k8s/component_resources.go | 14 +- internal/k8s/fullstack_names.go | 8 + internal/k8s/trust_registry.go | 118 +++++ internal/k8s/trust_registry_test.go | 142 ++++++ internal/k8s/workload_resources.go | 11 +- internal/model/setup_session.go | 7 + internal/model/trust_registry_test.go | 17 + internal/model/upgrade.go | 18 +- internal/resourceprofile/store.go | 4 +- internal/router/router.go | 13 + internal/setup/orchestrator_fullstack.go | 2 +- internal/setup/orchestrator_recovery.go | 13 +- internal/setup/orchestrator_vtc.go | 4 + internal/setup/resume_coverage_test.go | 3 +- internal/setup/trust_registry.go | 449 ++++++++++++++++++ internal/setup/trust_registry_config.go | 110 +++++ internal/setup/trust_registry_config_test.go | 100 ++++ .../setup/trust_registry_deployment_test.go | 130 +++++ internal/setup/trust_registry_failure_test.go | 23 + internal/setup/trust_registry_init_test.go | 106 +++++ internal/upgrade/runner.go | 6 + internal/vault/client.go | 6 +- internal/vault/trust_registry.go | 86 ++++ internal/vault/trust_registry_test.go | 74 +++ .../000044_optional_trust_registry.down.sql | 23 + .../000044_optional_trust_registry.up.sql | 18 + 44 files changed, 2177 insertions(+), 96 deletions(-) create mode 100644 internal/handler/setup_trust_registry.go create mode 100644 internal/handler/setup_trust_registry_test.go create mode 100644 internal/k8s/trust_registry.go create mode 100644 internal/k8s/trust_registry_test.go create mode 100644 internal/model/trust_registry_test.go create mode 100644 internal/setup/trust_registry.go create mode 100644 internal/setup/trust_registry_config.go create mode 100644 internal/setup/trust_registry_config_test.go create mode 100644 internal/setup/trust_registry_deployment_test.go create mode 100644 internal/setup/trust_registry_failure_test.go create mode 100644 internal/setup/trust_registry_init_test.go create mode 100644 internal/vault/trust_registry.go create mode 100644 internal/vault/trust_registry_test.go create mode 100644 migrations/000044_optional_trust_registry.down.sql create mode 100644 migrations/000044_optional_trust_registry.up.sql diff --git a/docs/full-stack-setup-design.md b/docs/full-stack-setup-design.md index 107fba4..e4a7bc5 100644 --- a/docs/full-stack-setup-design.md +++ b/docs/full-stack-setup-design.md @@ -12,10 +12,9 @@ https://dids-{vta_name}.{domain} ← WebVH DID Hosting daemon (admin panel https://vtc-{vtc_name}.{domain} ← VTC REST + admin SPA + public website ``` -**Planned extension:** [Optional Trust Registry](optional-trust-registry-design.md) +**Optional extension:** [Optional Trust Registry](optional-trust-registry-design.md) adds an opt-in fifth component, including in-place enablement for existing running -stacks without teardown. It is not implemented yet; the four-component baseline -and current behavior below remain the reference for deployed sessions. +stacks without teardown. The four-component baseline below remains the default. **The VTC is not optional.** Every `full_stack` session provisions all four components in one pipeline run. There is no VTC-less variant and no separate diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md index cb67fd6..121203d 100644 --- a/docs/optional-trust-registry-design.md +++ b/docs/optional-trust-registry-design.md @@ -1,6 +1,7 @@ # Optional Trust Registry for Full Stack — Design -Status: **planned; not implemented**. Research checked on 2026-10-20. +Status: **implemented in the working branch; image publication and staging acceptance pending**. +Research checked on 2026-10-20. Add a per-session Trust Registry to `full_stack` as an opt-in component, both when creating a stack and when adding it to an existing running stack. Existing @@ -35,7 +36,7 @@ describe the reviewed upstream revision, not a runtime inspection of production. ## 2. Scope and compatibility -| Scenario | Planned behavior | +| Scenario | Behavior | | --- | --- | | New `full_stack`, option omitted or false | Existing four-component flow | | New `full_stack`, option true | Existing components plus a dedicated registry | @@ -150,12 +151,15 @@ when built with `secrets-vault`. Its `TR_VTA_CREDENTIAL` loader separately suppo file/string/AWS sources; the reviewed loader has no Vault URI implementation. Therefore selecting Vault for the cache does not solve credential delivery. -Planned credential delivery: a pod-local init helper authenticates to Vault using +Credential delivery: a pod-local Vault CLI init container authenticates to Vault using the ServiceAccount, reads the context-scoped credential, and writes a `0600` file to a memory-backed `emptyDir`. The registry receives a `file://` reference to that file. Persist the credential only in Vault; keep values out of Deployment env, ConfigMaps, API responses, and Job logs. This needs init-container and temporary -volume support in the component helper, plus a pinned helper image/client. +volume support in the component helper. The init container uses the pinned +`hashicorp/vault:2.0.2` image, matching the Farm Vault; it does not depend on a +published Farm API image or a separately configured helper image. It revokes its +short-lived login token after reading the credential. The cache uses a different Vault location so it cannot overwrite the credential. The registry obtains and uses private keys at runtime. The setup guide's phrase @@ -171,20 +175,21 @@ When opted in, extend the post-admin-gate flow: ```text existing bootstrap and admin gate -> deploy VTA (mediator and DID host already ready) - -> provision registry context, DID, credential, and cache configuration - -> VTC setup with registry_did (publish referral) - -> configure registry ADMIN_DIDS with the newly created VTC DID + -> VTC setup and deploy VTC (existing core pipeline) + -> provision registry context, DID and Vault credential + -> configure registry ADMIN_DIDS with the VTC DID -> deploy registry and check its messaging/write-path status - -> merge [registry].did into generated VTC runtime config - -> deploy VTC - -> verify registry round trips and synchronization readiness - -> running + -> append the signed VTC referral through the VTA + -> merge [registry].did into the VTC runtime config and restart VTC + -> verify the published referral and finish add-on enablement + -> running (Registry has its own active/failed state) ``` -Use headless provisioning Jobs or an SDK client with an explicit service grant. +Use headless native VTA CLI Jobs under the session maintenance claim. Do not assume PNM is installed/configured in the API or copy a human's PNM -credential into the Farm. Exact provisioning commands and the pinned client are -an implementation preflight item. Preserve existing VTC output/claim handling; +credential into the Farm. The VTA creates the context and hosted DID, and imports +a generated, context-scoped access DID whose credential is persisted with Vault +CAS=0. Preserve existing VTC output/claim handling; account for its short install-token TTL when adding steps after VTC setup. ## 6. Add to an existing stack without rebuilding @@ -199,9 +204,9 @@ existing PVCs, or re-enroll the existing administrators. DID ownership/history, and any active upgrade/recovery operation. 2. Persist the operation and chosen image/identity parameters before mutation. Provision only the registry resources and Vault locations. -3. Establish a short-lived setup identity authorized for registry provisioning - and the required existing VTC DID update. Revoke it after completion; the - registry's lasting credential is scoped to its own context. +3. Generate or reuse a context-scoped registry access credential in Vault. + Native VTA CLI Jobs perform provisioning and DID editing while the VTA is + stopped; no lasting unrestricted setup credential is introduced. 4. Provision the registry identity through the existing VTA and publish it on the existing DID host. Reuse a completed result on retry. 5. Set `ADMIN_DIDS` to the existing `session.VtcDid`, start the registry, and @@ -214,8 +219,11 @@ existing PVCs, or re-enroll the existing administrators. `[registry].did`, and restart the VTC using the existing component-config backup/write/start/rollback pattern. Do not render a fresh setup config over the customized one or discard other keys already in `[registry]`. -8. Verify the VTC's selected transport, correlated replies, write authorization, - and sync diagnostics; reconcile existing members (§8). Mark the add-on active. +8. Verify the published signed referral and the restarted VTC. Mark the add-on + active after deployment/configuration checks. This state does not assert that + membership writes have succeeded: staging acceptance must verify the VTC's + selected transport, correlated replies, authorization, sync diagnostics, and + existing-member publication (§8). Upstream explicitly supports changing the referral after initial mint via VTA-side DID editing ([setup schema][vtc-setup]). This requires a signed DID-log @@ -244,7 +252,7 @@ Proposed add-on routes (under `/api/v1`): | Method | Route | Behavior | | --- | --- | --- | | POST | `/setup/:id/trust-registry` | Enable or explicitly retry with an image; returns 202 and progress | -| GET | `/setup/:id/trust-registry` | Desired state, phase, identity, image, health, and sanitized error | +| GET | `/setup/:id/trust-registry` | Desired state, phase, identity, image, and sanitized error | | POST/GET | `/admin/platform-stack/trust-registry` | Same operations, restricted to the platform stack | Apply existing Full Stack entitlement checks on creation and session ownership @@ -258,7 +266,15 @@ Persist desired enablement (default false), image, context/path, registry DID, add-on state (`disabled`, `enabling`, `active`, `failed`), resumable phase, timestamps, and a sanitized error. No credentials/private keys in these columns. For an existing stack, add-on failure must not mark the parent setup `failed`. -For new opted-in stacks, provisioning is complete only after the add-on is ready. +For new opted-in stacks, finish the add-on attempt before returning the baseline +stack to running. An add-on failure stays visible and retryable without failing +the already deployed four-component stack. + +After persisting an add-on failure, remove its failed Jobs and their Pods, +ConfigMaps, and temporary input Secrets. Keep the phase and sanitized error in +the session for the UI. Successful identity Jobs remain recovery receipts; +active Jobs are never removed by failure cleanup. Retrying reuses the persisted +context, credential and DID rather than relying on a failed Pod. Use a durable per-session operation claim, compatible with existing maintenance locking and startup resumption. A client timeout or API restart must not lose the @@ -279,14 +295,18 @@ Check the persisted audit cursor, pending/failed jobs, current roster, and exist registry mirror when enabling an old stack. Upstream can replay retained audit history from a missing cursor, but that does not guarantee a complete historical backfill after retention or previous partial synchronization. Reconcile against -the current roster and queue only eligible missing records through supported VTC -operations. Do not replay stale removals as fresh grants or reset cursors blindly. +the current roster during staging acceptance. The implementation retains and lets +the VTC process its existing audit history and queue; it does not directly edit +fjall or synthesize member events. Missing history after manual data repair needs +operator reconciliation through supported VTC operations. Do not replay stale +removals as fresh grants or reset cursors blindly. Honor member `registryConsent`/`publishConsent` and publication policy. Enabling the component cannot grant consent on a member's behalf. Report synchronization progress separately from service readiness; a large backlog need not block all community use. If the pinned VTC has no supported reconciliation operation, that -is a delivery prerequisite to resolve, not a reason to rebuild the community. +must be reconciled explicitly before claiming complete historical publication; +it is not a reason to rebuild the community. HTTP `/health` can return 200 with `status: degraded`. Monitor body status and write-path details, plus VTC sync queue/failure information. A successful read-only @@ -298,8 +318,10 @@ Include the component wherever deployed-component inventory is used: session responses, progress/logs, version selection, self-service/admin upgrades, recovery, resource profiles/capacity, backup/export, and full-session teardown. Capacity must distinguish opted-out stacks from five-component stacks and admit an add-on using -its incremental cost. Set resource requests/limits and PVC size from measurements; -do not invent production sizing from the explore guide. +its incremental cost. The working branch starts with a 10m CPU request, 64Mi +memory request, 256Mi memory limit, and 200Mi PVC. These are provisional factory +values, not measured production sizing. Validate them against expected records +and traffic in staging; memory defaults can be changed through the Farm UI. Back up the record PVC and account for both Vault secrets. Extend deletion to the registry Jobs, Deployment, Service, PVC, and session Vault prefix. Audit policies diff --git a/internal/apidocs/openapi.yaml b/internal/apidocs/openapi.yaml index c550bb0..c9524b2 100644 --- a/internal/apidocs/openapi.yaml +++ b/internal/apidocs/openapi.yaml @@ -87,6 +87,23 @@ components: name: vtafarm_admin schemas: + TrustRegistryState: + type: object + required: [enabled, state, phase, image, did, error] + properties: + enabled: { type: boolean } + state: { type: string, enum: [disabled, enabling, active, failed] } + phase: { type: string } + image: { type: string } + did: { type: string } + error: { type: string } + EnableTrustRegistryRequest: + type: object + required: [image] + properties: + image: + type: string + description: Immutable image selected from the trust-registry component image list. MobileConnection: type: object required: [request_id, status, vta_did, operation, expires_at] @@ -314,7 +331,7 @@ components: example: "myvta" component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] from_image: type: string description: The image the session runs before the upgrade. @@ -359,7 +376,7 @@ components: properties: component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] from_image: type: string to_image: @@ -588,6 +605,13 @@ components: Required for full_stack. Full image URL selected from GET /setup/images?component=vtc. Must be built with the `vault-secrets` feature. example: "ghcr.io/ic3software/vtc:0.3.0" + trust_registry_enabled: + type: boolean + default: false + description: Full Stack only. Create a dedicated Trust Registry; existing clients remain opted out. + trust_registry_image: + type: string + description: Required when enabled; select from GET /setup/images?component=trust-registry. vtc_name: type: string example: "myvtc" @@ -766,6 +790,10 @@ components: dids_image: type: string description: full_stack only. Current dids image. + trust_registry: + $ref: '#/components/schemas/TrustRegistryState' + trust_registry_image: + type: string vtc_image: type: string description: full_stack only. Current VTC image. @@ -2254,7 +2282,7 @@ paths: type: object required: [component, memory_request, memory_limit, customized] properties: - component: { type: string, enum: [vta, mediator, dids, vtc] } + component: { type: string, enum: [vta, mediator, dids, vtc, trust-registry] } memory_request: { type: string, example: 128Mi } memory_limit: { type: string, example: 512Mi } customized: @@ -2318,7 +2346,7 @@ paths: put: summary: Save component memory defaults description: | - Atomically replaces all four components' memory defaults. Each request + Saves the four core components and optionally Trust Registry memory defaults. Each request and limit must be between 16Mi and 1Gi inclusive, with request <= limit. Values are Kubernetes quantities. CPU settings are unchanged. New deployments and subsequent capacity estimates read these settings; @@ -2338,12 +2366,12 @@ paths: resources: type: array minItems: 4 - maxItems: 4 + maxItems: 5 items: type: object required: [component, memory_request, memory_limit] properties: - component: { type: string, enum: [mediator, vtc, vta, dids] } + component: { type: string, enum: [mediator, vtc, vta, dids, trust-registry] } memory_request: { type: string, example: 16Mi } memory_limit: { type: string, example: 64Mi } responses: @@ -2389,14 +2417,14 @@ paths: resources: type: array minItems: 1 - maxItems: 4 + maxItems: 5 items: type: object required: [component, memory_request, memory_limit] properties: component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] memory_request: type: string example: 128Mi @@ -2416,7 +2444,7 @@ paths: type: object properties: session_id: { type: string } - component: { type: string, enum: [vta, mediator, dids, vtc] } + component: { type: string, enum: [vta, mediator, dids, vtc, trust-registry] } status: { type: string, enum: [applied, failed, skipped] } error: { type: string } "404": { description: One or more sessions were not found } @@ -2454,7 +2482,7 @@ paths: items: type: object properties: - component: { type: string, enum: [vta, mediator, dids, vtc] } + component: { type: string, enum: [vta, mediator, dids, vtc, trust-registry] } defaults: { $ref: "#/components/schemas/ResourceProfile" } desired: type: object @@ -2603,6 +2631,10 @@ paths: mediator_image: { type: string } dids_image: { type: string } vtc_image: { type: string } + trust_registry_enabled: { type: boolean, default: false } + trust_registry_image: + type: string + description: Required when enabled; selected from the trust-registry image list. portable: { type: boolean, default: true } pre_rotation_count: { type: integer, default: 1 } responses: @@ -2718,6 +2750,9 @@ paths: mediator: { type: string } dids: { type: string } vtc: { type: string } + trust-registry: { type: string } + trust_registry: + $ref: '#/components/schemas/TrustRegistryState' collected: type: object description: | @@ -2985,7 +3020,7 @@ paths: description: Which component's image tags to list. schema: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] default: vta responses: "200": @@ -3507,7 +3542,7 @@ paths: properties: component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] image: type: string example: "ghcr.io/ic3software/vta:0.9.0-917881c" @@ -3700,7 +3735,7 @@ paths: type: string component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] from_image: type: string to_image: @@ -4873,7 +4908,7 @@ paths: description: Which component's image tags to list. schema: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] default: vta responses: "200": @@ -5210,6 +5245,86 @@ paths: "409": { description: Farm DID host is not ready, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } } "422": { description: Public DID does not resolve, content: { application/json: { schema: { $ref: "#/components/schemas/Error" } } } } + /api/v1/setup/{id}/trust-registry: + parameters: + - { name: id, in: path, required: true, schema: { type: string } } + get: + tags: [Setup] + summary: Read Trust Registry enablement progress + security: [{ CookieAuthUser: [] }] + responses: + '200': + description: Registry state (independent of the parent stack status). + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '404': { description: Stack not found or not owned by this caller. } + post: + tags: [Setup] + summary: Enable or retry Trust Registry on a running Full Stack + description: Preserves existing identities and members; may briefly restart VTA and VTC. Active registries use the image upgrade API. + security: [{ CookieAuthUser: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/EnableTrustRegistryRequest' } + responses: + '200': + description: Already enabled with this image. + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '202': + description: Durable enablement queued; poll GET for progress and failure details. + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '400': { description: Not a Full Stack or malformed request. } + '404': { description: Stack not found or not owned by this caller. } + '409': { description: Stack not running or another maintenance/upgrade operation is active. } + '422': { description: Image is not in the immutable registry image list. } + '503': { description: Capacity, Vault, image source, or init helper is unavailable. } + + /api/v1/admin/platform-stack/trust-registry: + get: + tags: [Setup] + summary: Read Trust Registry enablement progress + security: [{ CookieAuthAdmin: [] }] + responses: + '200': + description: Registry state (independent of the parent stack status). + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '404': { description: Stack not found or not owned by this caller. } + post: + tags: [Setup] + summary: Enable or retry Trust Registry on a running Full Stack + description: Preserves existing identities and members; may briefly restart VTA and VTC. Active registries use the image upgrade API. + security: [{ CookieAuthAdmin: [] }] + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/EnableTrustRegistryRequest' } + responses: + '200': + description: Already enabled with this image. + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '202': + description: Durable enablement queued; poll GET for progress and failure details. + content: + application/json: + schema: { $ref: '#/components/schemas/TrustRegistryState' } + '400': { description: Not a Full Stack or malformed request. } + '404': { description: Stack not found or not owned by this caller. } + '409': { description: Stack not running or another maintenance/upgrade operation is active. } + '422': { description: Image is not in the immutable registry image list. } + '503': { description: Capacity, Vault, image source, or init helper is unavailable. } + /api/v1/setup/{id}: get: summary: Get setup session @@ -5361,7 +5476,7 @@ paths: properties: component: type: string - enum: [vta, mediator, dids, vtc] + enum: [vta, mediator, dids, vtc, trust-registry] image: type: string example: "ghcr.io/ic3software/vta:0.9.0-917881c" diff --git a/internal/capacity/capacity.go b/internal/capacity/capacity.go index 5b1a24a..ba7a633 100644 --- a/internal/capacity/capacity.go +++ b/internal/capacity/capacity.go @@ -54,6 +54,13 @@ var ( }} ) +var TrustRegistry = Mode{Name: "trust_registry", Components: []Component{planningComponent(k8s.ComponentTrustRegistry, 200*mi)}} + +func FullStackWithRegistry() Mode { + components := slices.Clone(FullStack.Components) + return Mode{Name: "full_stack_registry", Components: append(components, TrustRegistry.Components...)} +} + func planningComponent(name string, storageBytes int64) Component { resources := k8s.DefaultResourceRequirements(name) return Component{ diff --git a/internal/config/config.go b/internal/config/config.go index fbeff85..ee736b6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -108,6 +108,7 @@ type GHCRConfig struct { PackageName string // e.g. "vta" MediatorPackageName string // e.g. "mediator" DIDHostingDaemonPackageName string // e.g. "did-hosting-daemon" + TrustRegistryPackageName string VtcPackageName string // e.g. "vtc" } @@ -204,6 +205,7 @@ func Load() *Config { PackageName: getEnv("GITHUB_PACKAGE_NAME", ""), MediatorPackageName: getEnv("GITHUB_MEDIATOR_PACKAGE_NAME", "mediator"), DIDHostingDaemonPackageName: getEnv("GITHUB_DID_HOSTING_DAEMON_PACKAGE_NAME", "did-hosting-daemon"), + TrustRegistryPackageName: getEnv("GITHUB_TRUST_REGISTRY_PACKAGE_NAME", "trust-registry"), VtcPackageName: getEnv("GITHUB_VTC_PACKAGE_NAME", "vtc"), }, DidHosting: DidHostingConfig{ diff --git a/internal/handler/admin_platform_stack.go b/internal/handler/admin_platform_stack.go index e7014bf..ac41471 100644 --- a/internal/handler/admin_platform_stack.go +++ b/internal/handler/admin_platform_stack.go @@ -87,10 +87,12 @@ type createPlatformStackRequest struct { // until step_vta_setup has run. The only thing that differs from a user's // session is who may resume it: any admin, through // POST /admin/setup-sessions/:id/admin, rather than the one owning user. - VtaImage string `json:"vta_image" binding:"required"` - MediatorImage string `json:"mediator_image" binding:"required"` - DidsImage string `json:"dids_image" binding:"required"` - VtcImage string `json:"vtc_image" binding:"required"` + VtaImage string `json:"vta_image" binding:"required"` + MediatorImage string `json:"mediator_image" binding:"required"` + DidsImage string `json:"dids_image" binding:"required"` + VtcImage string `json:"vtc_image" binding:"required"` + TrustRegistryEnabled bool `json:"trust_registry_enabled"` + TrustRegistryImage string `json:"trust_registry_image"` Portable *bool `json:"portable"` PreRotationCount *int `json:"pre_rotation_count"` @@ -126,6 +128,17 @@ func (h *SetupHandler) CreatePlatformStack(c *gin.Context) { return } + if req.TrustRegistryEnabled { + if !h.orch.RegistryAvailable() { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Trust Registry is unavailable because Kubernetes or Vault is not configured"}) + return + } + if !h.validRegistryImage(c, req.TrustRegistryImage) || !h.capacityAllows(c, capacity.FullStackWithRegistry()) { + return + } + } else { + req.TrustRegistryImage = "" + } label := req.Label if label == "" { label = platformLabelDefault @@ -245,10 +258,15 @@ func (h *SetupHandler) CreatePlatformStack(c *gin.Context) { MediatorImage: req.MediatorImage, DidsImage: req.DidsImage, VtcImage: req.VtcImage, + TrustRegistryEnabled: req.TrustRegistryEnabled, + TrustRegistryImage: req.TrustRegistryImage, Portable: portable, PreRotationCount: preRotationCount, } + if req.TrustRegistryEnabled { + session.TrustRegistryState = "enabling" + } // A single insert: there is no random id left to collide, so the retry loop // that used to wrap this went with unique_id. if createErr := h.db.Create(&session).Error; createErr != nil { @@ -336,11 +354,13 @@ func (h *SetupHandler) GetPlatformStack(c *gin.Context) { "did_hosting_admin_did": session.DIDHostingAdminDid, "vtc_did": session.VtcDid, }, + "trust_registry": registryJSON(&session), "images": gin.H{ - "vta": session.VtaImage, - "mediator": session.MediatorImage, - "dids": session.DidsImage, - "vtc": session.VtcImage, + "vta": session.VtaImage, + "mediator": session.MediatorImage, + "dids": session.DidsImage, + "vtc": session.VtcImage, + "trust-registry": session.TrustRegistryImage, }, // What this stack provides to default-connected vta_only sessions, read straight off // this row at create time. It is reported rather than pasted anywhere: diff --git a/internal/handler/admin_platform_stack_admins.go b/internal/handler/admin_platform_stack_admins.go index e5f3d50..bde4005 100644 --- a/internal/handler/admin_platform_stack_admins.go +++ b/internal/handler/admin_platform_stack_admins.go @@ -97,6 +97,31 @@ var errAclJobBusy = errors.New( // closes the same race across API replicas. Callers must invoke the returned // release function. func (h *SetupHandler) acquireSessionMaintenance(sessionID uint) (func(), error) { + var session model.SetupSession + if err := h.db.First(&session, sessionID).Error; err != nil { + return nil, err + } + if err := registryBusy(&session); err != nil { + return nil, fmt.Errorf("%w: %s", errAclJobBusy, err) + } + release, err := h.acquireStackMaintenance(sessionID) + if err != nil { + return nil, err + } + // A registry request can have committed while this replica was acquiring + // the database lease. Recheck before any scale or configuration write. + if err = h.db.First(&session, sessionID).Error; err != nil { + release() + return nil, err + } + if err = registryBusy(&session); err != nil { + release() + return nil, fmt.Errorf("%w: %s", errAclJobBusy, err) + } + return release, nil +} + +func (h *SetupHandler) acquireStackMaintenance(sessionID uint) (func(), error) { if !aclJobLocks.TryLock(sessionID) { return nil, errAclJobBusy } diff --git a/internal/handler/admin_upgrade.go b/internal/handler/admin_upgrade.go index f6d138a..60c151f 100644 --- a/internal/handler/admin_upgrade.go +++ b/internal/handler/admin_upgrade.go @@ -21,10 +21,11 @@ type UpgradeHandler struct { db *gorm.DB runner *upgrade.Runner // nil when K8s isn't configured - ghcr *ghcr.Client // nil when not configured - mediatorGhcr *ghcr.Client - didsGhcr *ghcr.Client - vtcGhcr *ghcr.Client + ghcr *ghcr.Client // nil when not configured + mediatorGhcr *ghcr.Client + didsGhcr *ghcr.Client + vtcGhcr *ghcr.Client + trustRegistryGhcr *ghcr.Client } func NewUpgradeHandler( @@ -42,6 +43,8 @@ func NewUpgradeHandler( } } +func (h *UpgradeHandler) ConfigureTrustRegistry(client *ghcr.Client) { h.trustRegistryGhcr = client } + func (h *UpgradeHandler) ghcrFor(component string) *ghcr.Client { switch component { case "vta": @@ -50,6 +53,8 @@ func (h *UpgradeHandler) ghcrFor(component string) *ghcr.Client { return h.mediatorGhcr case "dids": return h.didsGhcr + case "trust-registry": + return h.trustRegistryGhcr case "vtc": return h.vtcGhcr } @@ -57,7 +62,7 @@ func (h *UpgradeHandler) ghcrFor(component string) *ghcr.Client { } type componentImage struct { - Component string `json:"component" binding:"required,oneof=vta mediator dids vtc"` + Component string `json:"component" binding:"required,oneof=vta mediator dids vtc trust-registry"` Image string `json:"image" binding:"required"` } @@ -243,6 +248,10 @@ func (h *UpgradeHandler) resolveTargets(req *createUpgradeRequest) ([]upgradeTar targets := make([]upgradeTargetPair, 0, len(sessions)*len(req.Components)) for _, s := range sessions { + if registryBusy(&s) != nil { + skipped = append(skipped, skippedItem{SessionID: s.VtaName, Reason: "Trust Registry enablement is in progress"}) + continue + } var active int64 if err := h.db.Model(&model.UpgradeTask{}).Where("session_id = ? AND status IN ?", s.ID, []string{model.UpgradeTaskPending, model.UpgradeTaskRunning, model.UpgradeTaskRollingBack}).Count(&active).Error; err != nil { @@ -254,7 +263,7 @@ func (h *UpgradeHandler) resolveTargets(req *createUpgradeRequest) ([]upgradeTar } for _, ci := range req.Components { switch { - case !slices.Contains(model.UpgradeComponentModes[ci.Component], s.Mode): + case !slices.Contains(model.UpgradeComponentModes[ci.Component], s.Mode) || s.ComponentImage(ci.Component) == "": if !req.All { skipped = append(skipped, skippedItem{SessionID: s.VtaName, Component: ci.Component, Reason: "mode " + s.Mode + " has no " + ci.Component + " component"}) diff --git a/internal/handler/capacity_service.go b/internal/handler/capacity_service.go index 0df75dd..73a2c7d 100644 --- a/internal/handler/capacity_service.go +++ b/internal/handler/capacity_service.go @@ -107,11 +107,23 @@ func estimatesFrom(stats *k8s.ClusterStats, modes ...capacity.Mode) map[string]c // ModeFits reports whether one more session of mode still fits. determinable is // false when the cluster couldn't be read — callers fail open on that. func (s *CapacityService) ModeFits(ctx context.Context, mode capacity.Mode) (fits, determinable bool) { - est, _, ok := s.Estimates(ctx) - if !ok { + if s.k8s == nil { + return false, false + } + stats, err := s.stats(ctx) + if err != nil { return false, false } - return est[mode.Name].Count >= 1, true + profiles, err := resourceprofile.Load(ctx, s.db) + if err != nil { + return false, false + } + mode.Components = append([]capacity.Component(nil), mode.Components...) + for i := range mode.Components { + requirements := profiles.Requirements(mode.Components[i].Name) + mode.Components[i].MemBytes = requirements.Limits.Memory().Value() + } + return estimatesFrom(stats, mode)[mode.Name].Count >= 1, true } // freeResources turns cluster stats into per-node/per-disk planning headroom. diff --git a/internal/handler/resource_defaults.go b/internal/handler/resource_defaults.go index bb0b588..8114c7e 100644 --- a/internal/handler/resource_defaults.go +++ b/internal/handler/resource_defaults.go @@ -27,18 +27,30 @@ func (h *SetupHandler) AdminSaveResourceDefaults(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "resources are required"}) return } - if len(input.Resources) != 4 { - c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "resources must contain all four components"}) + if len(input.Resources) != 4 && len(input.Resources) != 5 { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "resources must contain the four core components and optionally Trust Registry"}) return } if err := validateMemoryInputs(input.Resources); err != nil { c.JSON(http.StatusUnprocessableEntity, gin.H{"error": err.Error()}) return } - rows := make([]model.ResourceDefault, 0, 4) + for _, required := range []string{"vta", "mediator", "dids", "vtc"} { + found := false + for _, item := range input.Resources { + if item.Component == required { + found = true + } + } + if !found { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "resources must include each core component"}) + return + } + } + rows := make([]model.ResourceDefault, 0, len(input.Resources)) profiles := resourceprofile.Factory() // Fixed ordering avoids conflicting lock order when admins save concurrently. - for _, component := range []string{"mediator", "vtc", "vta", "dids"} { + for _, component := range []string{"mediator", "vtc", "vta", "dids", "trust-registry"} { for _, item := range input.Resources { if item.Component != component { continue diff --git a/internal/handler/setup.go b/internal/handler/setup.go index 773f147..539a4b2 100644 --- a/internal/handler/setup.go +++ b/internal/handler/setup.go @@ -40,9 +40,10 @@ type SetupHandler struct { mobileWorker *additionalMobileWorker // full_stack mode - mediatorGhcr *ghcr.Client // nil when not configured - didsGhcr *ghcr.Client // nil when not configured - vtcGhcr *ghcr.Client // nil when not configured + mediatorGhcr *ghcr.Client // nil when not configured + didsGhcr *ghcr.Client // nil when not configured + vtcGhcr *ghcr.Client // nil when not configured + trustRegistryGhcr *ghcr.Client } func NewSetupHandler( @@ -120,8 +121,10 @@ func (h *SetupHandler) Images(c *gin.Context) { client = h.didsGhcr case "vtc": client = h.vtcGhcr + case "trust-registry": + client = h.trustRegistryGhcr default: - c.JSON(http.StatusBadRequest, gin.H{"error": "unknown component " + component + " (expected vta, mediator, dids, or vtc)"}) + c.JSON(http.StatusBadRequest, gin.H{"error": "unknown component " + component + " (expected vta, mediator, dids, vtc, or trust-registry)"}) return } @@ -162,10 +165,12 @@ type createSetupRequest struct { // full_stack only — all three images are required for that mode. vtc_name // is globally unique and DNS-safe like vta_name (becomes the vtc- // subdomain) and doubles as the VTA context id. - MediatorImage string `json:"mediator_image"` - DidsImage string `json:"dids_image"` - VtcImage string `json:"vtc_image"` - VtcName string `json:"vtc_name"` + MediatorImage string `json:"mediator_image"` + DidsImage string `json:"dids_image"` + VtcImage string `json:"vtc_image"` + VtcName string `json:"vtc_name"` + TrustRegistryEnabled bool `json:"trust_registry_enabled"` + TrustRegistryImage string `json:"trust_registry_image"` // VTA-only Customize / Connect to. Omit both to use the platform stack. DIDHostingDid string `json:"did_hosting_did"` MediatorDid string `json:"mediator_did"` @@ -187,6 +192,19 @@ func (h *SetupHandler) Create(c *gin.Context) { return } + if req.Mode != model.ModeFullStack && (req.TrustRegistryEnabled || req.TrustRegistryImage != "") { + c.JSON(http.StatusBadRequest, gin.H{"error": "Trust Registry requires a Full Stack"}) + return + } + if !req.TrustRegistryEnabled { + req.TrustRegistryImage = "" + } + if req.TrustRegistryEnabled && (!h.orch.RegistryAvailable() || !h.validRegistryImage(c, req.TrustRegistryImage)) { + if !c.Writer.Written() { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Trust Registry is unavailable because Kubernetes or Vault is not configured"}) + } + return + } userID := c.MustGet(middleware.ContextUserID).(uint) var user model.User diff --git a/internal/handler/setup_export.go b/internal/handler/setup_export.go index 1e19af1..d2032cd 100644 --- a/internal/handler/setup_export.go +++ b/internal/handler/setup_export.go @@ -30,12 +30,16 @@ func exportComponents(s *model.SetupSession) []exportComponent { {"vta", fmt.Sprintf("app=vta,session-id=%d", s.ID), "/work/vta/config.toml"}, } } - return []exportComponent{ + components := []exportComponent{ {"vta", fmt.Sprintf("app=fs-vta,session-id=%d", s.ID), "/work/vta/config.toml"}, {"mediator", fmt.Sprintf("app=fs-mediator,session-id=%d", s.ID), "/work/mediator/conf/mediator.toml"}, {"dids", fmt.Sprintf("app=fs-dids,session-id=%d", s.ID), "/work/dids/config.toml"}, {"vtc", fmt.Sprintf("app=fs-vtc,session-id=%d", s.ID), "/work/vtc/config.toml"}, } + if s.TrustRegistryEnabled && s.TrustRegistryDid != "" { + components = append(components, exportComponent{"trust-registry", fmt.Sprintf("app=fs-trust-registry,session-id=%d", s.ID), ""}) + } + return components } // GET /api/v1/setup/:id/export/configs @@ -139,6 +143,10 @@ func (h *SetupHandler) exportOne(ctx context.Context, ns string, comp exportComp return comp.name + ".log", body, err } + if comp.name == "trust-registry" { + content, err := h.k8s.RegistryEnvironment(ctx, ns, comp.selector) + return "trust-registry.env", content, err + } pod, container, err := h.k8s.RunningPod(ctx, ns, comp.selector) if err != nil { return "", "", err diff --git a/internal/handler/setup_fullstack.go b/internal/handler/setup_fullstack.go index fd893c6..548057b 100644 --- a/internal/handler/setup_fullstack.go +++ b/internal/handler/setup_fullstack.go @@ -10,7 +10,9 @@ import ( "time" "github.com/gin-gonic/gin" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "github.com/ic3software/vtafarm-api/internal/capacity" "github.com/ic3software/vtafarm-api/internal/k8s" "github.com/ic3software/vtafarm-api/internal/middleware" "github.com/ic3software/vtafarm-api/internal/model" @@ -79,6 +81,9 @@ func (h *SetupHandler) createFullStack(c *gin.Context, req createSetupRequest, d } } + if req.TrustRegistryEnabled && !h.capacityAllows(c, capacity.FullStackWithRegistry()) { + return + } portable := true if req.Portable != nil { portable = *req.Portable @@ -181,10 +186,15 @@ func (h *SetupHandler) createFullStack(c *gin.Context, req createSetupRequest, d MediatorImage: req.MediatorImage, DidsImage: req.DidsImage, VtcImage: req.VtcImage, + TrustRegistryEnabled: req.TrustRegistryEnabled, + TrustRegistryImage: req.TrustRegistryImage, AdminDid: req.AdminDid, Portable: portable, PreRotationCount: preRotationCount, } + if req.TrustRegistryEnabled { + session.TrustRegistryState = "enabling" + } // A single insert: there is no random id left to collide, so the retry loop // that used to wrap this went with unique_id. if createErr := h.persistUserSession(c.Request.Context(), &session); createErr != nil { @@ -261,6 +271,8 @@ func (h *SetupHandler) getFullStack(c *gin.Context, session *model.SetupSession) "created_at": session.CreatedAt, "updated_at": session.UpdatedAt, } + resp["trust_registry"] = registryJSON(session) + resp["trust_registry_image"] = session.TrustRegistryImage resp["dids_enroll_used"] = session.DidsEnrollUsed resp["vtc_install_used"] = session.VtcInstallUsed @@ -303,6 +315,19 @@ func (h *SetupHandler) getFullStack(c *gin.Context, session *model.SetupSession) // doesn't strand the rest of the teardown. func (h *SetupHandler) deleteFullStack(c *gin.Context, session *model.SetupSession) { ctx := c.Request.Context() + if h.k8s != nil && session.TrustRegistryDid != "" { + ns := h.k8s.UserNamespace(fmt.Sprint(session.UserID)) + // Stop the writer before deleting its Vault cache so it cannot recreate + // that cache while the rest of the stack is being removed. + err := h.k8s.ScaleComponentDeployment(ctx, ns, k8s.FSTrustRegistryName(session.ID), 0) + if err == nil { + err = h.k8s.WaitForComponentPodsGone(ctx, ns, fmt.Sprintf("app=fs-trust-registry,session-id=%d", session.ID), 2*time.Minute) + } + if err != nil && !apierrors.IsNotFound(err) { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Could not stop Trust Registry before deletion; retry after checking the workload"}) + return + } + } // A custom domain's records belong to the user: we never created them and // have no access to their zone. Removing the four CNAMEs is theirs to do, @@ -328,6 +353,7 @@ func (h *SetupHandler) deleteFullStack(c *gin.Context, session *model.SetupSessi h.orch.TeardownMediatorVault(ctx, session.UserID, session.ID) h.orch.TeardownDidsVault(ctx, session.UserID, session.ID) h.orch.TeardownVtcVault(ctx, session.UserID, session.ID) + h.orch.TeardownTrustRegistryVault(ctx, session.UserID, session.ID) } h.k8s.DeleteAllComponentJobs(ctx, ns, session.ID) @@ -335,6 +361,7 @@ func (h *SetupHandler) deleteFullStack(c *gin.Context, session *model.SetupSessi h.k8s.DeleteComponentResources(ctx, ns, k8s.FSMediatorName(session.ID)) h.k8s.DeleteComponentResources(ctx, ns, k8s.FSDidsName(session.ID)) h.k8s.DeleteComponentResources(ctx, ns, k8s.FSVtcName(session.ID)) + h.k8s.DeleteComponentResources(ctx, ns, k8s.FSTrustRegistryName(session.ID)) // The Certificate goes; **its Secret deliberately stays**. Recreating a // session on the same domain asks for the exact same four names, which @@ -438,6 +465,8 @@ func (h *SetupHandler) logsFullStack(c *gin.Context, session *model.SetupSession source = "vtc_acl_grant" case "step_vtc_setup": source = "vtc_setup" + case "step_trust_registry": + source = "registry_operation" case "deploy_vtc": source = "vtc" case "deploy_vta": @@ -456,6 +485,15 @@ func (h *SetupHandler) logsFullStack(c *gin.Context, session *model.SetupSession } switch source { + case "trust-registry": + streamPod("trust-registry") + case "registry_operation": + jobs := map[string]string{"preflight": "preflight", "provision_identity": "identity", "publish_referral": "referral", "configure_vtc": "config"} + if phase := jobs[session.TrustRegistryPhase]; phase != "" { + streamJob(setup.RegistryJobName(sid, phase)) + } else { + streamPod("trust-registry") + } case "vta_setup": streamJob(k8s.FSJobVtaSetup(sid)) case "mediator_p1": diff --git a/internal/handler/setup_resources.go b/internal/handler/setup_resources.go index a929bdd..4911855 100644 --- a/internal/handler/setup_resources.go +++ b/internal/handler/setup_resources.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "sort" "strconv" "strings" "time" @@ -68,12 +69,16 @@ func sessionWorkloadTargets(session *model.SetupSession) []workloadTarget { Selector: fmt.Sprintf("app=vta,session-id=%d", session.ID), }} } - return []workloadTarget{ + targets := []workloadTarget{ {Component: k8s.ComponentVTA, Deployment: k8s.FSVtaName(session.ID), Selector: fmt.Sprintf("app=fs-vta,session-id=%d", session.ID)}, {Component: k8s.ComponentMediator, Deployment: k8s.FSMediatorName(session.ID), Selector: fmt.Sprintf("app=fs-mediator,session-id=%d", session.ID)}, {Component: k8s.ComponentDids, Deployment: k8s.FSDidsName(session.ID), Selector: fmt.Sprintf("app=fs-dids,session-id=%d", session.ID)}, {Component: k8s.ComponentVTC, Deployment: k8s.FSVtcName(session.ID), Selector: fmt.Sprintf("app=fs-vtc,session-id=%d", session.ID)}, } + if session.TrustRegistryEnabled && session.TrustRegistryDid != "" { + targets = append(targets, workloadTarget{Component: k8s.ComponentTrustRegistry, Deployment: k8s.FSTrustRegistryName(session.ID), Selector: fmt.Sprintf("app=fs-trust-registry,session-id=%d", session.ID)}) + } + return targets } func sessionWorkloadTarget(session *model.SetupSession, component string) (workloadTarget, bool) { @@ -199,12 +204,19 @@ func (h *SetupHandler) AdminApplySessionResources(c *gin.Context) { return } sessionByName := make(map[string]*model.SetupSession, len(sessions)) + sort.Slice(sessions, func(i, j int) bool { return sessions[i].ID < sessions[j].ID }) for i := range sessions { session := &sessions[i] if session.Status != "running" { c.JSON(http.StatusConflict, gin.H{"error": fmt.Sprintf("session %s is not running", session.VtaName)}) return } + release, err := h.acquireSessionMaintenance(session.ID) + if err != nil { + c.JSON(http.StatusConflict, gin.H{"error": "Stack maintenance is in progress for " + session.VtaName}) + return + } + defer release() sessionByName[session.VtaName] = session } @@ -350,8 +362,8 @@ func validateResourceBatch(input *resourceBatchInput) error { if len(input.SessionIDs) == 0 || len(input.SessionIDs) > maxResourceBatchSessions { return fmt.Errorf("session_ids must contain between 1 and %d sessions", maxResourceBatchSessions) } - if len(input.Resources) == 0 || len(input.Resources) > 4 { - return fmt.Errorf("resources must contain between 1 and 4 components") + if len(input.Resources) == 0 || len(input.Resources) > 5 { + return fmt.Errorf("resources must contain between 1 and 5 components") } seenSessions := map[string]bool{} for i, id := range input.SessionIDs { diff --git a/internal/handler/setup_trust_registry.go b/internal/handler/setup_trust_registry.go new file mode 100644 index 0000000..2718959 --- /dev/null +++ b/internal/handler/setup_trust_registry.go @@ -0,0 +1,224 @@ +package handler + +import ( + "context" + "errors" + "log" + "net/http" + "slices" + "time" + + "github.com/gin-gonic/gin" + "gorm.io/gorm" + + "github.com/ic3software/vtafarm-api/internal/capacity" + "github.com/ic3software/vtafarm-api/internal/ghcr" + "github.com/ic3software/vtafarm-api/internal/model" +) + +func (h *SetupHandler) ConfigureTrustRegistry(client *ghcr.Client) { h.trustRegistryGhcr = client } + +func registryJSON(s *model.SetupSession) gin.H { + state := s.TrustRegistryState + if state == "" { + state = "disabled" + } + return gin.H{"enabled": s.TrustRegistryEnabled, "state": state, "phase": s.TrustRegistryPhase, "image": s.TrustRegistryImage, "did": s.TrustRegistryDid, "error": s.TrustRegistryError} +} + +func (h *SetupHandler) GetTrustRegistry(c *gin.Context) { + if s := h.userSession(c); s != nil { + c.JSON(http.StatusOK, registryJSON(s)) + } +} +func (h *SetupHandler) EnableTrustRegistry(c *gin.Context) { + if s := h.userSession(c); s != nil { + h.enableTrustRegistry(c, s) + } +} +func (h *SetupHandler) AdminGetTrustRegistry(c *gin.Context) { + if s := h.platformSession(c); s != nil { + c.JSON(http.StatusOK, registryJSON(s)) + } +} +func (h *SetupHandler) AdminEnableTrustRegistry(c *gin.Context) { + if s := h.platformSession(c); s != nil { + h.enableTrustRegistry(c, s) + } +} + +func (h *SetupHandler) validRegistryImage(c *gin.Context, image string) bool { + if h.trustRegistryGhcr == nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Trust Registry image source is not configured"}) + return false + } + tags, err := h.trustRegistryGhcr.ListTags(c.Request.Context()) + if err != nil { + c.JSON(http.StatusBadGateway, gin.H{"error": "Could not load Trust Registry images"}) + return false + } + if !slices.ContainsFunc(tags, func(tag ghcr.ImageTag) bool { return tag.Image == image }) { + c.JSON(http.StatusUnprocessableEntity, gin.H{"error": "Select an immutable image from the Trust Registry image list"}) + return false + } + return true +} + +func (h *SetupHandler) enableTrustRegistry(c *gin.Context, s *model.SetupSession) { + if !s.IsFullStack() { + c.JSON(http.StatusBadRequest, gin.H{"error": "Trust Registry requires a Full Stack"}) + return + } + if s.Status != "running" { + c.JSON(http.StatusConflict, gin.H{"error": "The Full Stack must be running before enabling Trust Registry"}) + return + } + var request struct { + Image string `json:"image" binding:"required"` + } + if c.ShouldBindJSON(&request) != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "An image is required"}) + return + } + if !h.orch.RegistryAvailable() { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Trust Registry is unavailable because Kubernetes or Vault is not configured"}) + return + } + if s.TrustRegistryState == "active" { + if s.TrustRegistryImage != request.Image { + c.JSON(http.StatusConflict, gin.H{"error": "Use image upgrades to change the active Registry image"}) + return + } + c.JSON(http.StatusOK, registryJSON(s)) + return + } + if s.TrustRegistryState == "enabling" { + if s.TrustRegistryImage != request.Image { + c.JSON(http.StatusConflict, gin.H{"error": "Registry enablement is already in progress"}) + return + } + h.kickTrustRegistry(s.ID) + c.JSON(http.StatusAccepted, registryJSON(s)) + return + } + if !h.validRegistryImage(c, request.Image) { + return + } + if !h.capacityAllows(c, capacity.TrustRegistry) { + return + } + release, err := h.acquireStackMaintenance(s.ID) + if err != nil { + c.JSON(http.StatusConflict, gin.H{"error": "Another stack maintenance operation is in progress"}) + return + } + defer func() { release() }() + var upgrading int64 + if err = h.db.Model(&model.UpgradeTask{}).Where("session_id = ? AND status IN ?", s.ID, []string{model.UpgradeTaskPending, model.UpgradeTaskRunning, model.UpgradeTaskRollingBack}).Count(&upgrading).Error; err != nil || upgrading > 0 { + c.JSON(http.StatusConflict, gin.H{"error": "Wait for the image upgrade or rollback to finish"}) + return + } + result := h.db.Model(&model.SetupSession{}).Where("id = ? AND status = ? AND trust_registry_state IN ?", s.ID, "running", []string{"disabled", "failed"}).Updates(map[string]any{"trust_registry_enabled": true, "trust_registry_state": "enabling", "trust_registry_image": request.Image, "trust_registry_phase": "queued", "trust_registry_error": "", "updated_at": time.Now()}) + if result.Error != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Could not save registry operation"}) + return + } + if result.RowsAffected != 1 { + c.JSON(http.StatusConflict, gin.H{"error": "Stack state changed; reload and try again"}) + return + } + s.TrustRegistryEnabled = true + s.TrustRegistryState = "enabling" + s.TrustRegistryImage = request.Image + s.TrustRegistryPhase = "queued" + s.TrustRegistryError = "" + // The worker can claim maintenance after this request releases it. + release() + release = func() {} + h.kickTrustRegistry(s.ID) + c.JSON(http.StatusAccepted, registryJSON(s)) +} + +func (h *SetupHandler) kickTrustRegistry(id uint) { go h.runTrustRegistry(context.Background(), id) } + +func (h *SetupHandler) StartTrustRegistryWorker(ctx context.Context) { + go func() { + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + for { + var sessions []model.SetupSession + if err := h.db.WithContext(ctx).Where("status = ? AND trust_registry_state = ?", "running", "enabling").Find(&sessions).Error; err == nil { + for _, s := range sessions { + go h.runTrustRegistry(ctx, s.ID) + } + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() +} + +func (h *SetupHandler) runTrustRegistry(parent context.Context, id uint) { + ctx, cancel := context.WithTimeout(parent, 10*time.Minute) + defer cancel() + if h.orch == nil { + return + } + // Serialize registry workers across replicas, independently of the HTTP + // request lifetime. The maintenance lease blocks other stack mutations. + // Keep the lock through cancellation cleanup, which restarts the daemons + // under its own bounded context after the operation deadline expires. + err := h.db.Transaction(func(tx *gorm.DB) error { + if tx.Dialector.Name() == "postgres" { + var locked bool + if err := tx.Raw("SELECT pg_try_advisory_xact_lock(?, ?)", 565441, int64(id)).Scan(&locked).Error; err != nil { + return err + } + if !locked { + return nil + } + } + var s model.SetupSession + if err := h.db.First(&s, id).Error; err != nil { + return err + } + if s.Status != "running" || s.TrustRegistryState != "enabling" { + return nil + } + // The advisory lock proves no other registry worker owns this session. + // Other maintenance rechecks enabling after its lease, so the timestamp + // left by a dead registry worker can be released without a 15-minute outage. + if err := h.db.Model(&model.VtaAclSnapshot{}).Where("session_id = ?", id).Update("maintenance_started_at", nil).Error; err != nil { + return err + } + release, err := h.acquireStackMaintenance(id) + if err != nil { + return nil + } + defer release() + err = h.orch.EnableTrustRegistry(ctx, &s) + if parent.Err() != nil { + return nil + } // A new replica resumes the persisted phase. + if err != nil { + if saveErr := h.db.Model(&model.SetupSession{}).Where("id = ?", id).Updates(map[string]any{"trust_registry_state": "failed", "trust_registry_error": err.Error(), "updated_at": time.Now()}).Error; saveErr != nil { + return saveErr + } + return h.orch.CleanupFailedRegistryJobs(&s) + } + return nil + }) + if err != nil { + log.Printf("[setup] registry worker %d failed: %v", id, err) + } +} + +func registryBusy(s *model.SetupSession) error { + if s.TrustRegistryState == "enabling" { + return errors.New("Trust Registry enablement is in progress") + } + return nil +} diff --git a/internal/handler/setup_trust_registry_test.go b/internal/handler/setup_trust_registry_test.go new file mode 100644 index 0000000..5174eec --- /dev/null +++ b/internal/handler/setup_trust_registry_test.go @@ -0,0 +1,68 @@ +package handler + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/ic3software/vtafarm-api/internal/middleware" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/testutil" +) + +func TestRegistryDefaultsAndOwnerBoundary(t *testing.T) { + db := testutil.Postgres(t) + session := testutil.SetupSession(t, db, model.ModeFullStack) + if session.TrustRegistryEnabled || session.TrustRegistryState != "disabled" { + t.Fatal("existing/default stacks must remain opted out") + } + h := &SetupHandler{db: db} + for _, tt := range []struct { + name string + user uint + want int + }{ + {"owner", session.UserID, http.StatusOK}, {"another account", session.UserID + 1, http.StatusNotFound}, + } { + t.Run(tt.name, func(t *testing.T) { + r := gin.New() + r.Use(func(c *gin.Context) { c.Set(middleware.ContextUserID, tt.user) }) + r.GET("/setup/:id/trust-registry", h.GetTrustRegistry) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/setup/"+session.VtaName+"/trust-registry", nil)) + if w.Code != tt.want { + t.Fatalf("status=%d want %d", w.Code, tt.want) + } + }) + } + if err := db.Model(&session).Updates(map[string]any{"trust_registry_enabled": true, "trust_registry_state": "enabling"}).Error; err != nil { + t.Fatal(err) + } + if _, err := h.acquireSessionMaintenance(session.ID); err == nil { + t.Fatal("maintenance raced with registry enablement") + } + if err := db.Model(&session).Update("trust_registry_state", "failed").Error; err != nil { + t.Fatal(err) + } + var stored model.SetupSession + if err := db.First(&stored, session.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Status != session.Status { + t.Fatal("registry failure changed the parent setup state") + } +} + +func TestVtaOnlyRegistryEnablementIsRejectedWithoutInfrastructure(t *testing.T) { + h := &SetupHandler{} + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"image":"registry:1-test"}`)) + c.Request.Header.Set("Content-Type", "application/json") + h.enableTrustRegistry(c, &model.SetupSession{Mode: model.ModeVtaOnly, Status: "running"}) + if w.Code != http.StatusBadRequest { + t.Fatalf("status=%d", w.Code) + } +} diff --git a/internal/handler/user_upgrade.go b/internal/handler/user_upgrade.go index 4bc242f..ce57dcc 100644 --- a/internal/handler/user_upgrade.go +++ b/internal/handler/user_upgrade.go @@ -65,6 +65,10 @@ func (h *UpgradeHandler) CreateForSession(c *gin.Context) { return } + if err := registryBusy(session); err != nil { + c.JSON(http.StatusConflict, gin.H{"error": err.Error()}) + return + } seen := map[string]bool{} for _, ci := range req.Components { if seen[ci.Component] { @@ -72,7 +76,7 @@ func (h *UpgradeHandler) CreateForSession(c *gin.Context) { return } seen[ci.Component] = true - if !slices.Contains(model.UpgradeComponentModes[ci.Component], session.Mode) { + if !slices.Contains(model.UpgradeComponentModes[ci.Component], session.Mode) || session.ComponentImage(ci.Component) == "" { c.JSON(http.StatusBadRequest, gin.H{"error": "mode " + session.Mode + " has no " + ci.Component + " component"}) return } diff --git a/internal/k8s/component_jobs.go b/internal/k8s/component_jobs.go index 29b3c32..6b2ab20 100644 --- a/internal/k8s/component_jobs.go +++ b/internal/k8s/component_jobs.go @@ -3,6 +3,7 @@ package k8s import ( "context" "fmt" + "time" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -11,6 +12,22 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// PrepareRegistryJob keeps identity receipts and attaches to interrupted active Jobs. +func (c *Client) PrepareRegistryJob(ctx context.Context, ns, name string, keepSuccess bool) error { + job, err := c.kube.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + if k8serrors.IsNotFound(err) { + return nil + } + if err != nil { + return err + } + if (job.Status.Failed > 0 || (!keepSuccess && job.Status.Succeeded > 0)) && job.Status.Active == 0 { + c.DeleteComponentJob(ctx, ns, name) + return c.WaitForComponentPodsGone(ctx, ns, "job-name="+name, 2*time.Minute) + } + return nil +} + // PVCMount describes one PVC mounted into a full_stack Job or Deployment. // Cross-component steps (design §4) mount two of these at once — e.g. the // VTA and mediator PVCs together for step_mediator_reprov. diff --git a/internal/k8s/component_resources.go b/internal/k8s/component_resources.go index 07192ea..6813c04 100644 --- a/internal/k8s/component_resources.go +++ b/internal/k8s/component_resources.go @@ -48,7 +48,11 @@ type ComponentDeploymentSpec struct { // limit (incompressible); CPU deliberately gets requests only — a CFS quota // would throttle bursts even when node CPU is idle, while the request alone // gives fair-share under contention. - Resources corev1.ResourceRequirements + Resources corev1.ResourceRequirements + Recreate bool + InitContainers []corev1.Container + ExtraVolumes []corev1.Volume + ExtraMounts []corev1.VolumeMount } // ComponentResources builds the standard farm resource shape: CPU+memory @@ -83,6 +87,8 @@ func (c *Client) CreateComponentDeployment(ctx context.Context, ns string, spec } var ports []corev1.ContainerPort + volumes = append(volumes, spec.ExtraVolumes...) + mounts = append(mounts, spec.ExtraMounts...) if spec.Port != 0 { ports = []corev1.ContainerPort{{ContainerPort: spec.Port, Protocol: corev1.ProtocolTCP}} } @@ -103,15 +109,21 @@ func (c *Client) CreateComponentDeployment(ctx context.Context, ns string, spec } } + strategy := appsv1.DeploymentStrategy{} + if spec.Recreate { + strategy.Type = appsv1.RecreateDeploymentStrategyType + } _, err := c.kube.AppsV1().Deployments(ns).Create(ctx, &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{Name: spec.Name, Namespace: ns, Labels: spec.Labels}, Spec: appsv1.DeploymentSpec{ Replicas: &replicas, + Strategy: strategy, Selector: &metav1.LabelSelector{MatchLabels: spec.Labels}, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{Labels: spec.Labels}, Spec: corev1.PodSpec{ ServiceAccountName: spec.ServiceAccount, + InitContainers: spec.InitContainers, Containers: []corev1.Container{{ Name: spec.Name, Image: spec.Image, diff --git a/internal/k8s/fullstack_names.go b/internal/k8s/fullstack_names.go index 12fb9a7..a7b26c2 100644 --- a/internal/k8s/fullstack_names.go +++ b/internal/k8s/fullstack_names.go @@ -15,6 +15,9 @@ func FSVtaName(sessionID uint) string { return fmt.Sprintf("fs-%d-vta", ses func FSMediatorName(sessionID uint) string { return fmt.Sprintf("fs-%d-mediator", sessionID) } func FSDidsName(sessionID uint) string { return fmt.Sprintf("fs-%d-dids", sessionID) } func FSVtcName(sessionID uint) string { return fmt.Sprintf("fs-%d-vtc", sessionID) } +func FSTrustRegistryName(sessionID uint) string { + return fmt.Sprintf("fs-%d-trust-registry", sessionID) +} // CustomTLSSecret names both the Certificate and the Secret it writes — // cert-manager takes the secretName from us, so keeping them identical means @@ -105,5 +108,10 @@ func allFSJobNames(sessionID uint) []string { FSJobVtcInvite(sessionID), FSJobConfigUpdate(sessionID), FSJobConfigRollback(sessionID), + fmt.Sprintf("fs-%d-tr-preflight", sessionID), + fmt.Sprintf("fs-%d-tr-identity", sessionID), + fmt.Sprintf("fs-%d-tr-referral", sessionID), + fmt.Sprintf("fs-%d-tr-config", sessionID), + fmt.Sprintf("fs-%d-tr-restore", sessionID), } } diff --git a/internal/k8s/trust_registry.go b/internal/k8s/trust_registry.go new file mode 100644 index 0000000..b75b495 --- /dev/null +++ b/internal/k8s/trust_registry.go @@ -0,0 +1,118 @@ +package k8s + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + "time" + + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" +) + +// CleanupFailedRegistryJobs leaves active work and successful identity receipts +// intact. The caller must persist the operation error before removing its Jobs. +func (c *Client) CleanupFailedRegistryJobs(ctx context.Context, ns string, names []string) error { + for _, name := range names { + job, err := c.kube.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return err + } + failed := false + for _, condition := range job.Status.Conditions { + failed = failed || condition.Type == batchv1.JobFailed && condition.Status == corev1.ConditionTrue + } + if !failed || job.Status.Active > 0 || job.Status.Succeeded > 0 { + continue + } + propagation := metav1.DeletePropagationForeground + uid := job.UID + err = c.kube.BatchV1().Jobs(ns).Delete(ctx, name, metav1.DeleteOptions{ + PropagationPolicy: &propagation, + Preconditions: &metav1.Preconditions{UID: &uid}, + }) + if err != nil && !apierrors.IsNotFound(err) { + return fmt.Errorf("delete failed registry Job %s: %w", name, err) + } + if err = wait.PollUntilContextTimeout(ctx, 500*time.Millisecond, time.Minute, true, func(ctx context.Context) (bool, error) { + _, err := c.kube.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + if err == nil { + return false, nil + } + if !apierrors.IsNotFound(err) { + return false, err + } + pods, err := c.kube.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + name}) + return err == nil && len(pods.Items) == 0, err + }); err != nil { + return fmt.Errorf("wait for failed registry Job %s cleanup: %w", name, err) + } + if err = c.kube.CoreV1().Secrets(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + if err = c.kube.CoreV1().ConfigMaps(ns).Delete(ctx, name, metav1.DeleteOptions{}); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + return nil +} + +// FinishRegistryJobs reattaches to Jobs left by an interrupted API before a +// daemon can reopen the same exclusive fjall store. +func (c *Client) FinishRegistryJobs(ctx context.Context, ns string, names []string) error { + for _, name := range names { + job, err := c.kube.BatchV1().Jobs(ns).Get(ctx, name, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + continue + } + if err != nil { + return err + } + if job.Status.Succeeded > 0 || job.Status.Failed > 0 && job.Status.Active == 0 { + continue + } + if _, _, err = c.WaitForJob(ctx, ns, name); err != nil { + cleanup, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + c.DeleteComponentJob(cleanup, ns, name) + stopErr := c.WaitForComponentPodsGone(cleanup, ns, "job-name="+name, time.Minute) + cancel() + if stopErr != nil { + return fmt.Errorf("registry Job could not stop: %w", stopErr) + } + return err + } + } + return nil +} + +// RegistryEnvironment exports only the non-secret settings set by the Farm. +// The credential is supplied through a memory volume, never through this list. +func (c *Client) RegistryEnvironment(ctx context.Context, ns, selector string) (string, error) { + deployments, err := c.kube.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) + if err != nil { + return "", err + } + if len(deployments.Items) != 1 || len(deployments.Items[0].Spec.Template.Spec.Containers) != 1 { + return "", fmt.Errorf("registry deployment is missing or ambiguous") + } + allowed := map[string]bool{"MEDIATOR_DID": true, "ADMIN_DIDS": true, "ACL_MODE": true, "LISTEN_ADDRESS": true, "ENABLE_DIDCOMM": true, "ENABLE_TSP": true, "TR_VTA_CONTEXT_ID": true, "TR_STORAGE_BACKEND": true, "TR_FJALL_PATH": true, "TR_SECRETS_VAULT_ADDR": true, "TR_SECRETS_VAULT_KV_MOUNT": true, "TR_SECRETS_VAULT_SECRET_PATH": true, "TR_SECRETS_VAULT_SECRET_KEY": true, "TR_SECRETS_VAULT_AUTH_METHOD": true, "TR_SECRETS_VAULT_K8S_ROLE": true, "TR_SECRETS_VAULT_K8S_MOUNT": true, "TR_SECRETS_VAULT_SKIP_VERIFY": true} + var lines []string + for _, env := range deployments.Items[0].Spec.Template.Spec.Containers[0].Env { + if env.ValueFrom != nil { + continue + } + if allowed[env.Name] { + lines = append(lines, env.Name+"="+strconv.Quote(env.Value)) + } + } + sort.Strings(lines) + return strings.Join(lines, "\n") + "\n", nil +} diff --git a/internal/k8s/trust_registry_test.go b/internal/k8s/trust_registry_test.go new file mode 100644 index 0000000..2d08009 --- /dev/null +++ b/internal/k8s/trust_registry_test.go @@ -0,0 +1,142 @@ +package k8s + +import ( + "context" + "errors" + "strings" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" +) + +func TestFailedRegistryJobCleanupIsScopedAndPreservesRecovery(t *testing.T) { + for _, tt := range []struct { + name string + status batchv1.JobStatus + denied bool + removed bool + }{ + {"failed", batchv1.JobStatus{Failed: 1, Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, false, true}, + {"successful receipt", batchv1.JobStatus{Succeeded: 1}, false, false}, + {"active", batchv1.JobStatus{Active: 1}, false, false}, + {"controller still retrying", batchv1.JobStatus{Failed: 1}, false, false}, + {"delete denied", batchv1.JobStatus{Failed: 1, Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}}}, true, false}, + } { + t.Run(tt.name, func(t *testing.T) { + kube := fake.NewSimpleClientset( + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "ns", UID: "job-uid"}, Status: tt.status}, + &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "unrelated", Namespace: "ns"}}, + &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "pod", Namespace: "ns", Labels: map[string]string{"job-name": "job"}}}, + &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "ns"}}, + &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "ns"}}, + &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: "vta-data", Namespace: "ns"}}, + ) + kube.PrependReactor("delete", "jobs", func(action ktesting.Action) (bool, runtime.Object, error) { + if tt.denied { + return true, nil, apierrors.NewForbidden(schema.GroupResource{Group: "batch", Resource: "jobs"}, "job", errors.New("denied")) + } + options := action.(ktesting.DeleteAction).GetDeleteOptions() + if options.PropagationPolicy == nil || *options.PropagationPolicy != metav1.DeletePropagationForeground || options.Preconditions == nil || options.Preconditions.UID == nil || *options.Preconditions.UID != "job-uid" { + t.Error("cleanup must target this exact Job and cascade to its Pods") + } + // Fake clients have no garbage collector; emulate the dependent Pod deletion. + _ = kube.Tracker().Delete(corev1.SchemeGroupVersion.WithResource("pods"), "ns", "pod") + return false, nil, nil + }) + client := &Client{kube: kube} + if err := client.CleanupFailedRegistryJobs(context.Background(), "ns", []string{"missing", "job"}); (err != nil) != tt.denied { + t.Fatalf("unexpected cleanup result: %v", err) + } + _, jobErr := kube.BatchV1().Jobs("ns").Get(context.Background(), "job", metav1.GetOptions{}) + _, podErr := kube.CoreV1().Pods("ns").Get(context.Background(), "pod", metav1.GetOptions{}) + _, secretErr := kube.CoreV1().Secrets("ns").Get(context.Background(), "job", metav1.GetOptions{}) + _, configErr := kube.CoreV1().ConfigMaps("ns").Get(context.Background(), "job", metav1.GetOptions{}) + for _, err := range []error{jobErr, podErr, secretErr, configErr} { + if apierrors.IsNotFound(err) != tt.removed { + t.Fatal("incorrect retention of Job, Pod, or temporary inputs") + } + } + if _, err := kube.BatchV1().Jobs("ns").Get(context.Background(), "unrelated", metav1.GetOptions{}); err != nil { + t.Fatal("cleanup touched an unrelated Job") + } + if _, err := kube.CoreV1().PersistentVolumeClaims("ns").Get(context.Background(), "vta-data", metav1.GetOptions{}); err != nil { + t.Fatal("cleanup touched persistent identity data") + } + if tt.removed { + if err := client.CleanupFailedRegistryJobs(context.Background(), "ns", []string{"job"}); err != nil { + t.Fatal("cleanup is not idempotent") + } + } + }) + } +} + +func TestRegistryJobRetryRetainsIdentityAndActiveJobs(t *testing.T) { + for _, tt := range []struct { + name string + keep bool + status batchv1.JobStatus + removed bool + }{ + {"identity receipt", true, batchv1.JobStatus{Succeeded: 1}, false}, + {"reapply runtime config", false, batchv1.JobStatus{Succeeded: 1}, true}, + {"failed identity", true, batchv1.JobStatus{Failed: 1}, true}, + {"active identity", true, batchv1.JobStatus{Active: 1}, false}, + } { + t.Run(tt.name, func(t *testing.T) { + kube := fake.NewSimpleClientset(&batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "ns"}, Status: tt.status}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "job", Namespace: "ns"}}) + client := &Client{kube: kube} + if err := client.PrepareRegistryJob(context.Background(), "ns", "job", tt.keep); err != nil { + t.Fatal(err) + } + _, err := kube.BatchV1().Jobs("ns").Get(context.Background(), "job", metav1.GetOptions{}) + if apierrors.IsNotFound(err) != tt.removed { + t.Fatal("wrong job retry decision") + } + _, err = kube.CoreV1().Secrets("ns").Get(context.Background(), "job", metav1.GetOptions{}) + if apierrors.IsNotFound(err) != tt.removed { + t.Fatal("stale input secret was retained") + } + }) + } +} + +func TestRegistryDeploymentKeepsCredentialInMemoryAndOutOfExports(t *testing.T) { + kube := fake.NewSimpleClientset() + client := &Client{kube: kube} + spec := ComponentDeploymentSpec{Name: "registry", Labels: map[string]string{"app": "registry"}, Image: "registry:image", ServiceAccount: VtaServiceAccount, Command: []string{"trust-registry"}, Port: 3232, HealthCheckPath: "/health", Recreate: true, + Env: []corev1.EnvVar{{Name: "MEDIATOR_DID", Value: "did:webvh:mediator"}, {Name: "TR_VTA_CONTEXT_ID", Value: "registry-7"}, {Name: "TR_SECRETS_SEED", Value: "test-only-secret"}}, + ExtraVolumes: []corev1.Volume{{Name: "credential", VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{Medium: corev1.StorageMediumMemory}}}}, + ExtraMounts: []corev1.VolumeMount{{Name: "credential", MountPath: "/run/trust-registry", ReadOnly: true}}, + InitContainers: []corev1.Container{{Name: "credential", Image: "hashicorp/vault:2.0.2", Command: []string{"sh", "-ec", "vault kv get"}}}, + } + if err := client.CreateComponentDeployment(context.Background(), "ns", spec); err != nil { + t.Fatal(err) + } + deployment, err := kube.AppsV1().Deployments("ns").Get(context.Background(), "registry", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if deployment.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType { + t.Fatal("registry may roll two fjall writers concurrently") + } + pod := deployment.Spec.Template.Spec + if len(pod.InitContainers) != 1 || pod.Volumes[0].EmptyDir.Medium != corev1.StorageMediumMemory || !pod.Containers[0].VolumeMounts[0].ReadOnly { + t.Fatal("credential memory mount is not isolated") + } + content, err := client.RegistryEnvironment(context.Background(), "ns", "app=registry") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(content, "MEDIATOR_DID") || strings.Contains(content, "test-only-secret") { + t.Fatal("registry export omitted references or leaked secret material") + } +} diff --git a/internal/k8s/workload_resources.go b/internal/k8s/workload_resources.go index fc4bb3a..4741d21 100644 --- a/internal/k8s/workload_resources.go +++ b/internal/k8s/workload_resources.go @@ -13,10 +13,11 @@ import ( ) const ( - ComponentVTA = "vta" - ComponentMediator = "mediator" - ComponentDids = "dids" - ComponentVTC = "vtc" + ComponentVTA = "vta" + ComponentMediator = "mediator" + ComponentDids = "dids" + ComponentVTC = "vtc" + ComponentTrustRegistry = "trust-registry" ) // ResourceProfile is the resource shape used when a new component is created. @@ -39,6 +40,8 @@ func DefaultResourceProfile(component string) (ResourceProfile, bool) { return ResourceProfile{CPURequest: "10m", MemoryRequest: "64Mi", MemoryLimit: "128Mi"}, true case ComponentVTC: return ResourceProfile{CPURequest: "10m", MemoryRequest: "64Mi", MemoryLimit: "256Mi"}, true + case ComponentTrustRegistry: + return ResourceProfile{CPURequest: "10m", MemoryRequest: "64Mi", MemoryLimit: "256Mi"}, true default: return ResourceProfile{}, false } diff --git a/internal/model/setup_session.go b/internal/model/setup_session.go index 4a15fb7..b34e564 100644 --- a/internal/model/setup_session.go +++ b/internal/model/setup_session.go @@ -163,6 +163,13 @@ type SetupSession struct { VtcName string `gorm:"column:vtc_name;not null" json:"vtc_name,omitempty"` VtcImage string `gorm:"column:vtc_image;not null;default:''" json:"vtc_image,omitempty"` + TrustRegistryEnabled bool `gorm:"not null;default:false" json:"trust_registry_enabled"` + TrustRegistryImage string `gorm:"not null;default:''" json:"trust_registry_image,omitempty"` + TrustRegistryDid string `gorm:"column:trust_registry_did;not null;default:''" json:"trust_registry_did,omitempty"` + TrustRegistryState string `gorm:"not null;default:disabled" json:"trust_registry_state"` + TrustRegistryPhase string `gorm:"not null;default:''" json:"trust_registry_phase,omitempty"` + TrustRegistryError string `gorm:"not null;default:''" json:"trust_registry_error,omitempty"` + // full_stack — collected outputs. VtcSetupKeyDid is the ephemeral // did:key from step_vtc_setup_key, kept for audit/debug only — nothing // reads it back from the DB. VtcAdminDid is the VTC's own pre-claim diff --git a/internal/model/trust_registry_test.go b/internal/model/trust_registry_test.go new file mode 100644 index 0000000..fd7ea80 --- /dev/null +++ b/internal/model/trust_registry_test.go @@ -0,0 +1,17 @@ +package model + +import "testing" + +func TestRegistryUpgradeInventoryRequiresActiveComponent(t *testing.T) { + s := SetupSession{Mode: ModeFullStack, TrustRegistryImage: "ghcr.io/ic3software/trust-registry:0.28.0-test"} + for _, state := range []string{"", "disabled", "enabling", "failed"} { + s.TrustRegistryState = state + if s.ComponentImage("trust-registry") != "" { + t.Fatal("undeployed registry was offered as an upgrade target") + } + } + s.TrustRegistryState = "active" + if s.ComponentImage("trust-registry") != s.TrustRegistryImage { + t.Fatal("active registry missing from upgrade inventory") + } +} diff --git a/internal/model/upgrade.go b/internal/model/upgrade.go index 7c8b35f..edf3d9b 100644 --- a/internal/model/upgrade.go +++ b/internal/model/upgrade.go @@ -20,7 +20,7 @@ const ( // UpgradeComponents are the deployable components an admin can upgrade — // the same set GET /setup/images serves tags for. -var UpgradeComponents = []string{"vta", "mediator", "dids", "vtc"} +var UpgradeComponents = []string{"vta", "mediator", "dids", "vtc", "trust-registry"} // UpgradeBatch is one image rollout over a set of sessions. Exactly one of // AdminID / UserID is set (DB CHECK): an admin batch can span many sessions, @@ -59,10 +59,11 @@ type UpgradeTask struct { // UpgradeComponentModes maps each component to the session modes that run it. // A session is only a valid upgrade target for components its mode deploys. var UpgradeComponentModes = map[string][]string{ - "vta": {ModeVtaOnly, ModeFullStack}, - "mediator": {ModeFullStack}, - "dids": {ModeFullStack}, - "vtc": {ModeFullStack}, + "vta": {ModeVtaOnly, ModeFullStack}, + "mediator": {ModeFullStack}, + "dids": {ModeFullStack}, + "vtc": {ModeFullStack}, + "trust-registry": {ModeFullStack}, } // UpgradeImageColumn returns the setup_sessions column holding the given @@ -75,6 +76,8 @@ func UpgradeImageColumn(component string) string { return "mediator_image" case "dids": return "dids_image" + case "trust-registry": + return "trust_registry_image" case "vtc": return "vtc_image" } @@ -90,6 +93,11 @@ func (s *SetupSession) ComponentImage(component string) string { return s.MediatorImage case "dids": return s.DidsImage + case "trust-registry": + if s.TrustRegistryState == "active" { + return s.TrustRegistryImage + } + return "" case "vtc": return s.VtcImage } diff --git a/internal/resourceprofile/store.go b/internal/resourceprofile/store.go index 73c630c..d2ba801 100644 --- a/internal/resourceprofile/store.go +++ b/internal/resourceprofile/store.go @@ -14,8 +14,8 @@ import ( type Profiles map[string]k8s.ResourceProfile func Factory() Profiles { - profiles := make(Profiles, 4) - for _, component := range []string{k8s.ComponentMediator, k8s.ComponentVTC, k8s.ComponentVTA, k8s.ComponentDids} { + profiles := make(Profiles, 5) + for _, component := range []string{k8s.ComponentMediator, k8s.ComponentVTC, k8s.ComponentVTA, k8s.ComponentDids, k8s.ComponentTrustRegistry} { profiles[component], _ = k8s.DefaultResourceProfile(component) } return profiles diff --git a/internal/router/router.go b/internal/router/router.go index bde4e00..a56d0d6 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -76,6 +76,14 @@ func Setup( dhFactory, k8sClient, orch, ghcrClient, mediatorGhcrClient, didsGhcrClient, vtcGhcrClient, ) + var registryGhcr *ghcr.Client + if cfg.GHCR.Owner != "" && cfg.GHCR.TrustRegistryPackageName != "" { + registryGhcr = ghcr.New(cfg.GHCR.Token, cfg.GHCR.Owner, cfg.GHCR.TrustRegistryPackageName) + } + sh.ConfigureTrustRegistry(registryGhcr) + if k8sClient != nil && cfg.OrchestratorResume { + sh.StartTrustRegistryWorker(workerCtx) + } if k8sClient != nil { sh.StartAdditionalMobileWorker(workerCtx, cfg.OrchestratorResume) } @@ -133,6 +141,7 @@ func Setup( // Image upgrades — shared by the admin batch routes and the user // self-service routes below; background runner, DB-backed queue. uph := handler.NewUpgradeHandler(db, upgradeRunner, ghcrClient, mediatorGhcrClient, didsGhcrClient, vtcGhcrClient) + uph.ConfigureTrustRegistry(registryGhcr) mobile := handler.NewMobileConnectionHandler(db, orch, sh.KickAdditionalMobile, cfg.VTAClaim, trustTask.Enabled()) dashH := handler.NewDashboardHandler(db, k8sClient) poolH := handler.NewPoolHandler(db, sh, dashH, mobile) @@ -214,6 +223,8 @@ func Setup( // a domains row for our own zone. adminAuth.POST("/admin/platform-stack", sh.CreatePlatformStack) adminAuth.GET("/admin/platform-stack", sh.GetPlatformStack) + adminAuth.GET("/admin/platform-stack/trust-registry", sh.AdminGetTrustRegistry) + adminAuth.POST("/admin/platform-stack/trust-registry", sh.AdminEnableTrustRegistry) adminAuth.GET("/admin/platform-stack/config", sh.AdminGetPlatformStackConfigs) adminAuth.POST("/admin/platform-stack/config/validate", sh.AdminValidatePlatformStackConfigs) adminAuth.PUT("/admin/platform-stack/config", sh.AdminApplyPlatformStackConfigs) @@ -289,6 +300,8 @@ func Setup( userAuth.POST("/setup", sh.Create) userAuth.POST("/setup/connection/inspect", middleware.RateLimit(20, time.Minute), sh.InspectConnection) userAuth.GET("/setup/:id", sh.Get) + userAuth.GET("/setup/:id/trust-registry", sh.GetTrustRegistry) + userAuth.POST("/setup/:id/trust-registry", sh.EnableTrustRegistry) userAuth.POST("/setup/:id/recover", sh.RecoverSession) userAuth.DELETE("/setup/:id", sh.Delete) userAuth.GET("/setup/:id/logs", sh.Logs) diff --git a/internal/setup/orchestrator_fullstack.go b/internal/setup/orchestrator_fullstack.go index 4ca3c03..9628477 100644 --- a/internal/setup/orchestrator_fullstack.go +++ b/internal/setup/orchestrator_fullstack.go @@ -1212,7 +1212,7 @@ var ( // Deployment stages resume exactly at the failed/current deployment. Going // through Start or Provision would repeat setup jobs that already succeeded. - fsResumeDeployment = []string{"deploy_dids", "deploy_mediator", "deploy_vta", "deploy_vtc"} + fsResumeDeployment = []string{"deploy_dids", "deploy_mediator", "deploy_vta", "deploy_vtc", "step_trust_registry"} // fsResumeTerminal: nothing is owed. awaiting_admin_did is only terminal // while admin_did is empty — resumeFullStack queries that case separately, diff --git a/internal/setup/orchestrator_recovery.go b/internal/setup/orchestrator_recovery.go index c720a68..6c623a0 100644 --- a/internal/setup/orchestrator_recovery.go +++ b/internal/setup/orchestrator_recovery.go @@ -58,6 +58,10 @@ func (o *Orchestrator) recoverDeployment(ctx context.Context, sessionID uint) { if s.Mode != model.ModeFullStack { return } + if s.Status == "step_trust_registry" { + o.finishRecoveredSession(ctx, sessionID, s) + return + } if fail(s.Status, "failed to reconcile recovery images", o.reconcileFullStackRecoveryImages(ctx, ns, s)) { return } @@ -89,7 +93,7 @@ func (o *Orchestrator) recoverDeployment(ctx context.Context, sessionID uint) { if fail("deploy_vtc", "failed to deploy vtc", o.fsDeployVtc(ctx, ns, s)) { return } - o.finishRecoveredSession(sessionID, s) + o.finishRecoveredSession(ctx, sessionID, s) } } @@ -179,10 +183,13 @@ func (o *Orchestrator) continueRecoveredPostGate( if fail("deploy_vtc", "failed to deploy vtc", o.fsDeployVtc(ctx, ns, s)) { return } - o.finishRecoveredSession(s.ID, s) + o.finishRecoveredSession(ctx, s.ID, s) } -func (o *Orchestrator) finishRecoveredSession(sessionID uint, s *model.SetupSession) { +func (o *Orchestrator) finishRecoveredSession(ctx context.Context, sessionID uint, s *model.SetupSession) { + if !o.finishRegistrySetup(ctx, s) { + return + } o.db.Model(&model.SetupSession{}).Where("id = ?", sessionID).Updates(map[string]any{ "status": "running", "error_msg": "", "failed_stage": "", "updated_at": time.Now(), }) diff --git a/internal/setup/orchestrator_vtc.go b/internal/setup/orchestrator_vtc.go index e83ee85..8881efb 100644 --- a/internal/setup/orchestrator_vtc.go +++ b/internal/setup/orchestrator_vtc.go @@ -103,6 +103,10 @@ func (o *Orchestrator) runFullStackFinish(ctx context.Context, sessionID uint, a return } + if !o.finishRegistrySetup(ctx, s) { + return + } + o.fsSetStatus(sessionID, "running") log.Printf("[orchestrator] fs-vtc session %d: running at https://%s", sessionID, s.VtcFQDN()) } diff --git a/internal/setup/resume_coverage_test.go b/internal/setup/resume_coverage_test.go index 45b32bd..65d6c38 100644 --- a/internal/setup/resume_coverage_test.go +++ b/internal/setup/resume_coverage_test.go @@ -26,6 +26,7 @@ import ( var pipelineSources = []string{ "orchestrator_fullstack.go", "orchestrator_vtc.go", + "trust_registry.go", } // Both forms the pipeline uses to write the column: @@ -163,7 +164,7 @@ func TestCustomDomainStatusesAreResumable(t *testing.T) { } func TestDeploymentStatusesUseExactRecovery(t *testing.T) { - want := []string{"deploy_dids", "deploy_mediator", "deploy_vta", "deploy_vtc"} + want := []string{"deploy_dids", "deploy_mediator", "deploy_vta", "deploy_vtc", "step_trust_registry"} if len(fsResumeDeployment) != len(want) { t.Fatalf("fsResumeDeployment = %v, want %v", fsResumeDeployment, want) } diff --git a/internal/setup/trust_registry.go b/internal/setup/trust_registry.go new file mode 100644 index 0000000..ba56a53 --- /dev/null +++ b/internal/setup/trust_registry.go @@ -0,0 +1,449 @@ +package setup + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "regexp" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/resourceprofile" + "github.com/ic3software/vtafarm-api/internal/vault" + "github.com/ic3software/vtafarm-api/internal/webvh" +) + +const RegistryPVCSize = "200Mi" + +// Match the Farm's Vault image; credential delivery needs only the Vault CLI. +const registryVaultInitImage = "hashicorp/vault:2.0.2" + +const registryCredentialInit = `set -eu +umask 077 +credential_file=/run/trust-registry/credential.json +mkdir -p /run/trust-registry +VAULT_TOKEN="$(vault write -field=token "auth/$TR_SECRETS_VAULT_K8S_MOUNT/login" \ + "role=$TR_SECRETS_VAULT_K8S_ROLE" jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token)" +export VAULT_TOKEN +trap 'rm -f "$credential_file.tmp"; vault token revoke -self >/dev/null 2>&1 || true' EXIT +vault kv get -mount="$TR_SECRETS_VAULT_KV_MOUNT" -field=credential \ + "$TR_CREDENTIAL_VAULT_PATH" > "$credential_file.tmp" +mv "$credential_file.tmp" "$credential_file" +` + +func (o *Orchestrator) RegistryAvailable() bool { + return o != nil && o.k8s != nil && o.vault != nil +} + +func (o *Orchestrator) RegistryPhase(s *model.SetupSession, phase string) error { + s.TrustRegistryPhase = phase + return o.db.Model(s).Updates(map[string]any{"trust_registry_phase": phase, "updated_at": time.Now()}).Error +} + +func RegistryJobName(id uint, phase string) string { return fmt.Sprintf("fs-%d-tr-%s", id, phase) } + +func registryJobNames(id uint) []string { + var names []string + for _, phase := range []string{"preflight", "identity", "referral", "config", "restore"} { + names = append(names, RegistryJobName(id, phase)) + } + return names +} + +func (o *Orchestrator) CleanupFailedRegistryJobs(s *model.SetupSession) error { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + return o.k8s.CleanupFailedRegistryJobs(ctx, o.k8s.UserNamespace(fmt.Sprint(s.UserID)), registryJobNames(s.ID)) +} + +func registryJobFailure(phase, reason, output string) error { + // Only retain known diagnostic text; arbitrary CLI output can contain keys + // or configuration values and must not be copied into the session response. + for _, diagnostic := range []string{ + "could not sign the DID-management task with this VTA's operational key", + "Registry image needs the vta feature", + "Registry image needs Vault Kubernetes authentication", + } { + if strings.Contains(output, diagnostic) { + return fmt.Errorf("%s job failed: %s (%s)", phase, diagnostic, reason) + } + } + return fmt.Errorf("%s job failed (%s)", phase, reason) +} + +func (o *Orchestrator) registryJob(ctx context.Context, ns string, s *model.SetupSession, phase, image, dir, command string, mounts []k8s.PVCMount, inputs map[string][]byte) (string, error) { + name := RegistryJobName(s.ID, phase) + if err := o.k8s.PrepareRegistryJob(ctx, ns, name, phase == "identity"); err != nil { + return "", err + } + if err := o.k8s.CreateComponentJob(ctx, ns, k8s.ComponentJobSpec{Name: name, Image: image, Command: []string{"sh", "-c", command}, WorkingDir: dir, ServiceAccount: k8s.VtaServiceAccount, PVCMounts: mounts, SecretData: inputs, Env: fsNoColorEnv(), RetainForRecovery: true}); err != nil { + return "", err + } + ok, reason, err := o.k8s.WaitForJob(ctx, ns, name) + if err != nil { + cleanup, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + o.k8s.DeleteComponentJob(cleanup, ns, name) + stopErr := o.k8s.WaitForComponentPodsGone(cleanup, ns, "job-name="+name, time.Minute) + return "", errors.Join(err, stopErr) + } + if !ok { + output, _ := o.k8s.JobLogs(ctx, ns, name) + return "", registryJobFailure(phase, reason, output) + } + return o.k8s.JobLogs(ctx, ns, name) +} + +func (o *Orchestrator) registryStopped(ctx context.Context, ns string, s *model.SetupSession, component string, fn func() error) (err error) { + name := fmt.Sprintf("fs-%d-%s", s.ID, component) + selector := fmt.Sprintf("app=fs-%s,session-id=%d", component, s.ID) + if err = o.k8s.ScaleComponentDeployment(ctx, ns, name, 0); err != nil { + return err + } + defer func() { + restartCtx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) + defer cancel() + e := o.k8s.ScaleComponentDeployment(restartCtx, ns, name, 1) + if e == nil { + e = o.k8s.WaitForComponentDeploymentReadyOrRestarts(restartCtx, ns, name, selector, 2*time.Minute, 3) + } + if e != nil { + err = errors.Join(err, fmt.Errorf("%s did not restart: %w", component, e)) + } + }() + if err = o.k8s.WaitForComponentPodsGone(ctx, ns, selector, 2*time.Minute); err != nil { + return err + } + return fn() +} + +func (o *Orchestrator) registryVtcConfig(ctx context.Context, ns string, s *model.SetupSession) (string, error) { + pod, container, err := o.k8s.RunningPod(ctx, ns, fmt.Sprintf("app=fs-vtc,session-id=%d", s.ID)) + if err != nil { + return "", err + } + return o.k8s.ExecCapture(ctx, ns, pod, container, []string{"cat", "/work/vtc/config.toml"}) +} + +// EnableTrustRegistry is shared by initial setup and in-place enablement. Its +// caller owns the session maintenance claim; it never re-runs VTC setup. +func (o *Orchestrator) EnableTrustRegistry(ctx context.Context, s *model.SetupSession) error { + if !o.RegistryAvailable() { + return errors.New("Trust Registry requires Kubernetes and Vault") + } + ns := o.k8s.UserNamespace(fmt.Sprint(s.UserID)) + if err := o.k8s.FinishRegistryJobs(ctx, ns, registryJobNames(s.ID)); err != nil { + return err + } + for _, component := range []string{"vta", "vtc"} { + name := fmt.Sprintf("fs-%d-%s", s.ID, component) + if err := o.k8s.ScaleComponentDeployment(ctx, ns, name, 1); err != nil { + return err + } + if err := o.k8s.WaitForComponentDeploymentReady(ctx, ns, name, 2*time.Minute); err != nil { + if component == "vtc" { + restoreErr := o.restoreRegistryVtc(ns, s) + if restoreErr == nil { + return errors.New("interrupted VTC configuration was restored; retry registry enablement") + } + return errors.Join(fmt.Errorf("recover VTC before registry enablement: %w", err), restoreErr) + } + return fmt.Errorf("recover %s before registry enablement: %w", component, err) + } + } + if err := o.RegistryPhase(s, "preflight"); err != nil { + return err + } + if s.VtcDid == "" || s.VtaDid == "" || s.MediatorDid == "" { + return errors.New("VTA, VTC and mediator identities must already exist") + } + current, err := o.registryVtcConfig(ctx, ns, s) + if err != nil { + return errors.New("cannot read existing VTC configuration") + } + // Validate before provisioning or stopping any existing component. + placeholder := s.TrustRegistryDid + if placeholder == "" { + placeholder = "did:webvh:pending:registry:pending" + } + if _, err := MergeRegistryConfig(current, placeholder); err != nil { + return err + } + _, err = o.registryJob(ctx, ns, s, "preflight", s.TrustRegistryImage, "/tmp", `set -eu +binary=$(command -v trust-registry) +grep -aq TR_VTA_CREDENTIAL "$binary" || { echo 'Registry image needs the vta feature'; exit 1; } +grep -aq TR_SECRETS_VAULT_K8S_ROLE "$binary" || { echo 'Registry image needs Vault Kubernetes authentication'; exit 1; } +echo 'Registry feature preflight passed'`, nil, nil) + if err != nil { + return fmt.Errorf("registry image preflight failed; use a build with vta, secrets-vault, storage-fjall and standalone: %w", err) + } + // --help never opens the live keystore. Older images fail before downtime. + pod, container, err := o.k8s.RunningPod(ctx, ns, fmt.Sprintf("app=fs-vta,session-id=%d", s.ID)) + if err != nil { + return err + } + for _, command := range [][]string{{"vta", "did-mgmt", "dids", "create", "--help"}, {"vta", "did-mgmt", "dids", "edit", "--help"}} { + if _, err = o.k8s.ExecCapture(ctx, ns, pod, container, command); err != nil { + return errors.New("VTA image needs non-interactive DID creation/editing; upgrade VTA before enabling the registry") + } + } + document, err := webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) + if err != nil { + return fmt.Errorf("cannot validate existing VTC DID history: %w", err) + } + if _, _, err = RegistryReferral(document, s.VtcDid, placeholder); err != nil { + return err + } + if err = o.vault.EnsureUserAccess(ctx, s.UserID, ns, k8s.VtaServiceAccount); err != nil { + return err + } + credential, err := o.vault.RegistryCredential(ctx, s.UserID, s.ID, s.VtaDid, s.PublicURL()) + if err != nil { + return errors.New("cannot persist registry credential in Vault; check the API registry Vault policy") + } + contextID := fmt.Sprintf("trust-registry-%d", s.ID) + path := s.VtcName + "-trust-registry" + if err = o.RegistryPhase(s, "provision_identity"); err != nil { + return err + } + if s.TrustRegistryDid == "" { + var output string + err = o.registryStopped(ctx, ns, s, "vta", func() error { + cmd := fmt.Sprintf(`set -eu +vta contexts get %s >/dev/null 2>&1 || vta contexts create --id %s --name 'Trust Registry' +vta acl get %s >/dev/null 2>&1 || vta import-did --did %s --role admin --context %s --label trust-registry +existing=$(vta did-mgmt dids list --context %s 2>&1) +if printf '%%s' "$existing" | grep -q 'DID:'; then + printf '%%s\n' "$existing" +else + vta did-mgmt dids create --context %s --server dids --path %s --mediator-service --pre-rotation 1 +fi +vta did-mgmt dids list --context %s`, shellQuote(contextID), shellQuote(contextID), shellQuote(credential.Did), shellQuote(credential.Did), shellQuote(contextID), shellQuote(contextID), shellQuote(contextID), shellQuote(path), shellQuote(contextID)) + var e error + output, e = o.registryJob(ctx, ns, s, "identity", s.VtaImage, "/work/vta", cmd, []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, nil) + return e + }) + if err != nil { + return err + } + matches := regexp.MustCompile(`DID:\s+(did:webvh:\S+)`).FindAllStringSubmatch(stripANSI(output), -1) + identities := map[string]bool{} + for _, match := range matches { + identities[match[1]] = true + } + if len(identities) != 1 { + return errors.New("registry context must contain exactly one hosted DID; inspect identity job before retrying") + } + for did := range identities { + s.TrustRegistryDid = did + } + if err = o.db.Model(s).Update("trust_registry_did", s.TrustRegistryDid).Error; err != nil { + return err + } + } + if err = webvh.NewResolver(webvh.DefaultTimeout).ResolveDID(ctx, s.TrustRegistryDid); err != nil { + return fmt.Errorf("registry DID did not resolve: %w", err) + } + if err = o.RegistryPhase(s, "deploy_registry"); err != nil { + return err + } + if err = o.deployTrustRegistry(ctx, ns, s, contextID); err != nil { + return err + } + if err = o.RegistryPhase(s, "publish_referral"); err != nil { + return err + } + // Resolve again immediately before editing; never apply the preflight snapshot. + document, err = webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) + if err != nil { + return err + } + updated, changed, err := RegistryReferral(document, s.VtcDid, s.TrustRegistryDid) + if err != nil { + return err + } + if changed { + err = o.registryStopped(ctx, ns, s, "vta", func() error { + _, e := o.registryJob(ctx, ns, s, "referral", s.VtaImage, "/work/vta", "vta did-mgmt dids edit --did "+shellQuote(s.VtcDid)+" --document /config/document.json --no-confirm", []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, map[string][]byte{"document.json": updated}) + return e + }) + if err != nil { + return err + } + } + if err = o.RegistryPhase(s, "configure_vtc"); err != nil { + return err + } + current, err = o.registryVtcConfig(ctx, ns, s) + if err != nil { + return err + } + merged, err := MergeRegistryConfig(current, s.TrustRegistryDid) + if err != nil { + return err + } + if current != merged { + err = o.registryStopped(ctx, ns, s, "vtc", func() error { + _, e := o.registryJob(ctx, ns, s, "config", s.VtcImage, "/work/vtc", `set -eu; umask 077 +cp -p config.toml config.toml.vtafarm-registry-backup +cp -p config.toml config.toml.vtafarm-registry-next +cat /config/vtc.toml > config.toml.vtafarm-registry-next +mv config.toml.vtafarm-registry-next config.toml`, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, map[string][]byte{"vtc.toml": []byte(merged)}) + return e + }) + if err != nil { + return errors.Join(err, o.restoreRegistryVtc(ns, s)) + } + } + if err = o.RegistryPhase(s, "verify"); err != nil { + return err + } + document, err = webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) + if err != nil { + return err + } + if _, changed, err = RegistryReferral(document, s.VtcDid, s.TrustRegistryDid); err != nil || changed { + return errors.New("published VTC referral has not converged; retry after DID hosting is updated") + } + s.TrustRegistryState = "active" + return o.db.Model(s).Updates(map[string]any{"trust_registry_state": "active", "trust_registry_phase": "complete", "trust_registry_error": "", "updated_at": time.Now()}).Error +} + +func (o *Orchestrator) restoreRegistryVtc(ns string, s *model.SetupSession) error { + ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) + defer cancel() + return o.registryStopped(ctx, ns, s, "vtc", func() error { + _, err := o.registryJob(ctx, ns, s, "restore", s.VtcImage, "/work/vtc", `set -eu; test -f config.toml.vtafarm-registry-backup; cp -p config.toml.vtafarm-registry-backup config.toml`, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, nil) + return err + }) +} + +func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *model.SetupSession, contextID string) error { + name := k8s.FSTrustRegistryName(s.ID) + if err := o.k8s.CreateComponentPVC(ctx, ns, name, RegistryPVCSize); err != nil { + return err + } + labels := fsLabels("trust-registry", s.ID) + if err := o.k8s.CreateComponentService(ctx, ns, name, labels, 3232); err != nil { + return err + } + values := map[string]string{ + "ENABLE_DIDCOMM": "true", + "ENABLE_TSP": "false", + "MEDIATOR_DID": s.MediatorDid, + "ADMIN_DIDS": s.VtcDid, + "ACL_MODE": "ExplicitDeny", + "TR_VTA_CREDENTIAL": "file:///run/trust-registry/credential.json", + "TR_VTA_CONTEXT_ID": contextID, + "TR_STORAGE_BACKEND": "fjall", + "TR_FJALL_PATH": "/work/trust-registry/data", + "TR_SECRETS_VAULT_ADDR": o.vaultAddr, + "VAULT_ADDR": o.vaultAddr, + "TR_SECRETS_VAULT_KV_MOUNT": o.vault.KVMount(), + "TR_SECRETS_VAULT_SECRET_PATH": vault.TrustRegistryPrefix(s.UserID, s.ID) + "/identity-cache", + "TR_SECRETS_VAULT_SECRET_KEY": "bundle", + "TR_SECRETS_VAULT_AUTH_METHOD": "kubernetes", + "TR_SECRETS_VAULT_K8S_MOUNT": o.vault.K8sAuthMount(), + "TR_SECRETS_VAULT_K8S_ROLE": vault.UserName(s.UserID), + "TR_SECRETS_VAULT_SKIP_VERIFY": "true", + "VAULT_SKIP_VERIFY": "true", + "TR_CREDENTIAL_VAULT_PATH": vault.TrustRegistryPrefix(s.UserID, s.ID) + "/credential", + "LISTEN_ADDRESS": "0.0.0.0:3232", + "RUST_LOG": "info", + } + env := fsNoColorEnv() + for key, value := range values { + env = append(env, corev1.EnvVar{Name: key, Value: value}) + } + profiles, err := resourceprofile.Load(ctx, o.db) + if err != nil { + return err + } + mount := corev1.VolumeMount{Name: "registry-credential", MountPath: "/run/trust-registry", ReadOnly: true} + initMount := mount + initMount.ReadOnly = false + root := int64(0) + if err = o.k8s.CreateComponentDeployment(ctx, ns, k8s.ComponentDeploymentSpec{ + Name: name, Image: s.TrustRegistryImage, + Command: []string{"trust-registry"}, WorkingDir: "/work/trust-registry", + ServiceAccount: k8s.VtaServiceAccount, Env: env, + Port: 3232, HealthCheckPath: "/health", Labels: labels, + Resources: profiles.Requirements(k8s.ComponentTrustRegistry), + PVCMounts: []k8s.PVCMount{{Name: "registry-data", ClaimName: name, MountPath: "/work/trust-registry"}}, + ExtraVolumes: []corev1.Volume{{ + Name: mount.Name, + VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ + Medium: corev1.StorageMediumMemory, + SizeLimit: resource.NewQuantity(64<<10, resource.BinarySI), + }}, + }}, + ExtraMounts: []corev1.VolumeMount{mount}, Recreate: true, + InitContainers: []corev1.Container{{ + Name: "vault-credential", Image: registryVaultInitImage, + Command: []string{"sh", "-ec", registryCredentialInit}, Env: env, + VolumeMounts: []corev1.VolumeMount{initMount}, + Resources: k8s.ComponentResources("10m", "16Mi", "128Mi"), + SecurityContext: &corev1.SecurityContext{RunAsUser: &root}, + }}, + }); err != nil { + return err + } + if err = o.k8s.RecoverDeploymentImage(ctx, ns, name, s.TrustRegistryImage); err != nil { + return err + } + if err = o.k8s.WaitForComponentDeploymentReadyOrRestarts(ctx, ns, name, fmt.Sprintf("app=fs-trust-registry,session-id=%d", s.ID), 3*time.Minute, 3); err != nil { + return err + } + // HTTP status alone does not report the independently failing write path. + pod, container, err := o.k8s.RunningPod(ctx, ns, fmt.Sprintf("app=fs-trust-registry,session-id=%d", s.ID)) + if err != nil { + return err + } + body, err := o.k8s.ExecCapture(ctx, ns, pod, container, []string{"curl", "--fail", "--silent", "--max-time", "10", "http://127.0.0.1:3232/health"}) + if err != nil { + return errors.New("registry health request failed") + } + var health struct { + Status string `json:"status"` + Writes string `json:"writes"` + } + if json.Unmarshal([]byte(body), &health) != nil || health.Status != "OK" || health.Writes != "ok" { + return errors.New("registry messaging write path is not healthy; inspect registry logs") + } + return nil +} + +func (o *Orchestrator) TeardownTrustRegistryVault(ctx context.Context, userID, sessionID uint) { + if o.vault != nil { + if err := o.vault.DeleteTrustRegistrySecrets(ctx, userID, sessionID); err != nil { + log.Printf("[orchestrator] warn: registry Vault cleanup failed for session %d: %v", sessionID, err) + } + } +} + +func (o *Orchestrator) finishRegistrySetup(ctx context.Context, s *model.SetupSession) bool { + if !s.TrustRegistryEnabled || s.TrustRegistryState == "active" { + return true + } + o.fsSetStatus(s.ID, "step_trust_registry") + if err := o.EnableTrustRegistry(ctx, s); err != nil { + if ctx.Err() != nil { + return false + } + if saveErr := o.db.Model(s).Updates(map[string]any{"trust_registry_state": "failed", "trust_registry_error": err.Error()}).Error; saveErr != nil { + log.Printf("[orchestrator] registry failure for session %d could not be recorded; retaining Job", s.ID) + return false + } + if cleanupErr := o.CleanupFailedRegistryJobs(s); cleanupErr != nil { + log.Printf("[orchestrator] failed registry Job cleanup for session %d: %v", s.ID, cleanupErr) + } + } + return true +} diff --git a/internal/setup/trust_registry_config.go b/internal/setup/trust_registry_config.go new file mode 100644 index 0000000..7fc95c6 --- /dev/null +++ b/internal/setup/trust_registry_config.go @@ -0,0 +1,110 @@ +package setup + +import ( + "encoding/json" + "errors" + "reflect" + "regexp" + "strings" + + "github.com/pelletier/go-toml/v2" +) + +var registrySection = regexp.MustCompile(`(?m)^[\t ]*\[registry\][\t ]*(?:#.*)?$`) +var tomlSection = regexp.MustCompile(`(?m)^[\t ]*\[`) +var registryDidSetting = regexp.MustCompile(`(?m)^[\t ]*did[\t ]*=.*$`) + +// MergeRegistryConfig preserves unrelated settings and comments, and refuses to +// silently move a community from a registry it already uses. +func MergeRegistryConfig(content, did string) (string, error) { + if !strings.HasPrefix(did, "did:webvh:") { + return "", errors.New("registry must have a did:webvh identity") + } + var config map[string]any + if err := toml.Unmarshal([]byte(content), &config); err != nil { + return "", errors.New("invalid VTC TOML") + } + if existing, ok := config["registry"].(map[string]any); ok { + if target, ok := existing["did"].(string); ok && target != "" && target != did { + return "", errors.New("VTC already points to another registry") + } + if target, ok := existing["url"].(string); ok && target != "" { + return "", errors.New("VTC has an external registry URL; reconcile it before enabling a managed registry") + } + } + encodedDid, _ := json.Marshal(did) + line := "did = " + string(encodedDid) + section := registrySection.FindStringIndex(content) + if section == nil { + if _, exists := config["registry"]; exists { + return "", errors.New("use a [registry] table instead of an inline registry configuration") + } + content = strings.TrimRight(content, "\n") + "\n\n[registry]\n" + line + "\n" + } else { + end := len(content) + if next := tomlSection.FindStringIndex(content[section[1]:]); next != nil { + end = section[1] + next[0] + } + body := content[section[1]:end] + if registryDidSetting.MatchString(body) { + body = registryDidSetting.ReplaceAllStringFunc(body, func(string) string { return line }) + } else { + body = "\n" + line + body + } + content = content[:section[1]] + body + content[end:] + } + var check map[string]any + if err := toml.Unmarshal([]byte(content), &check); err != nil { + return "", errors.New("cannot merge registry into VTC TOML") + } + expected, ok := config["registry"].(map[string]any) + if !ok { + expected = map[string]any{} + config["registry"] = expected + } + expected["did"] = did + // Text ranges can resemble tables inside TOML multiline strings. Refuse + // any edit that changes more than the parsed registry DID setting. + if !reflect.DeepEqual(config, check) { + return "", errors.New("cannot preserve custom VTC TOML while merging registry") + } + return content, nil +} + +func RegistryReferral(document []byte, vtcDid, registryDid string) ([]byte, bool, error) { + var doc map[string]any + if json.Unmarshal(document, &doc) != nil || doc["id"] != vtcDid { + return nil, false, errors.New("VTC DID document identity mismatch") + } + services, ok := doc["service"].([]any) + if doc["service"] != nil && !ok { + return nil, false, errors.New("VTC DID services must be an array") + } + found := false + for _, entry := range services { + service, ok := entry.(map[string]any) + if !ok { + return nil, false, errors.New("invalid VTC DID service") + } + isRegistry := service["type"] == "TrustRegistry" + if types, ok := service["type"].([]any); ok { + for _, kind := range types { + isRegistry = isRegistry || kind == "TrustRegistry" + } + } + if isRegistry || service["id"] == vtcDid+"#trust-registry" { + endpoint, _ := service["serviceEndpoint"].(map[string]any) + if !found && endpoint["uri"] == registryDid && service["type"] == "TrustRegistry" { + found = true + continue + } + return nil, false, errors.New("VTC DID already refers to another registry") + } + } + if found { + return document, false, nil + } + doc["service"] = append(services, map[string]any{"id": vtcDid + "#trust-registry", "type": "TrustRegistry", "serviceEndpoint": map[string]string{"uri": registryDid, "profile": "https://trustoverip.org/profiles/trqp/v2"}}) + encoded, err := json.Marshal(doc) + return encoded, true, err +} diff --git a/internal/setup/trust_registry_config_test.go b/internal/setup/trust_registry_config_test.go new file mode 100644 index 0000000..a8ef097 --- /dev/null +++ b/internal/setup/trust_registry_config_test.go @@ -0,0 +1,100 @@ +package setup + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "github.com/pelletier/go-toml/v2" +) + +func TestRegistryConfigPreservesCustomizedSettings(t *testing.T) { + for _, content := range []string{ + "# custom site\npublic_url = \"https://community.example\"\n[rooms]\nmax_files = 9\n", + "public_url = \"https://community.example\"\n[registry] # keep this\nhttp_timeout_seconds = 17\n[rooms]\nmax_files = 9\n", + "[registry]\ndid = \"\"\nhealth_probe_interval_seconds = 42\n", + } { + const did = "did:webvh:abc:host.example:registry" + merged, err := MergeRegistryConfig(content, did) + if err != nil { + t.Fatal(err) + } + var before, after map[string]any + if err = toml.Unmarshal([]byte(content), &before); err != nil { + t.Fatal(err) + } + if err = toml.Unmarshal([]byte(merged), &after); err != nil { + t.Fatal(err) + } + registry := after["registry"].(map[string]any) + if registry["did"] != did { + t.Fatal("registry DID missing") + } + delete(registry, "did") + if old, ok := before["registry"].(map[string]any); ok { + delete(old, "did") + } else { + delete(after, "registry") + } + if !reflect.DeepEqual(before, after) { + t.Fatal("an unrelated configuration value changed") + } + again, err := MergeRegistryConfig(merged, did) + if err != nil || again != merged { + t.Fatal("merge is not idempotent") + } + if strings.Contains(content, "# keep this") && !strings.Contains(merged, "# keep this") { + t.Fatal("comment was discarded") + } + } +} + +func TestRegistryConfigRefusesConflictsAndMalformedInputs(t *testing.T) { + for _, content := range []string{"[registry]\ndid=\"did:webvh:other\"\n", "[registry]\nurl=\"https://other.example\"\n", "registry = { health_probe_interval_seconds = 42 }\n", "[invalid", "welcome = '''\n[registry]\ndid = \"\"\n'''\n"} { + if _, err := MergeRegistryConfig(content, "did:webvh:ours"); err == nil { + t.Fatal("accepted conflicting or malformed config") + } + } +} + +func TestRegistryReferralRefusesMultipleOrArrayTypedRegistryServices(t *testing.T) { + for _, services := range []string{ + `[{"type":["TrustRegistry"],"serviceEndpoint":{"uri":"did:webvh:other"}}]`, + `[{"type":"TrustRegistry","serviceEndpoint":{"uri":"did:webvh:registry"}},{"type":"TrustRegistry","serviceEndpoint":{"uri":"did:webvh:other"}}]`, + } { + if _, _, err := RegistryReferral([]byte(`{"id":"did:webvh:community","service":`+services+`}`), "did:webvh:community", "did:webvh:registry"); err == nil { + t.Fatal("accepted conflicting registry referrals") + } + } +} + +func TestRegistryReferralPreservesIdentityKeysAndServices(t *testing.T) { + const did = "did:webvh:community" + document := []byte(`{"id":"did:webvh:community","verificationMethod":[{"id":"did:webvh:community#key-0","publicKeyMultibase":"zPublic"}],"authentication":["did:webvh:community#key-0"],"service":[{"id":"did:webvh:community#didcomm","type":"DIDCommMessaging","serviceEndpoint":{"uri":"did:webvh:mediator"}}],"custom":{"name":"existing"}}`) + updated, changed, err := RegistryReferral(document, did, "did:webvh:registry") + if err != nil || !changed { + t.Fatalf("referral: %v", err) + } + var before, after map[string]any + _ = json.Unmarshal(document, &before) + _ = json.Unmarshal(updated, &after) + services := after["service"].([]any) + if len(services) != 2 { + t.Fatal("original services were replaced") + } + after["service"] = services[:1] + if !reflect.DeepEqual(before, after) { + t.Fatal("existing DID document fields changed") + } + _, changed, err = RegistryReferral(updated, did, "did:webvh:registry") + if err != nil || changed { + t.Fatal("existing referral was not idempotent") + } + if _, _, err = RegistryReferral(updated, did, "did:webvh:other"); err == nil { + t.Fatal("silently replaced another registry") + } + if _, _, err = RegistryReferral(document, "did:webvh:wrong", "did:webvh:registry"); err == nil { + t.Fatal("accepted another identity") + } +} diff --git a/internal/setup/trust_registry_deployment_test.go b/internal/setup/trust_registry_deployment_test.go new file mode 100644 index 0000000..3999033 --- /dev/null +++ b/internal/setup/trust_registry_deployment_test.go @@ -0,0 +1,130 @@ +package setup + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "gorm.io/driver/postgres" + "gorm.io/gorm" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes/scheme" + + "github.com/ic3software/vtafarm-api/internal/config" + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/vault" +) + +// Capture the actual orchestration request, stopping before pods or exec are +// needed. DryRun supplies factory resource defaults without a database server. +func TestRegistryDeploymentCarriesVTAAndWriteAuthorization(t *testing.T) { + var deployment appsv1.Deployment + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + if r.Method != http.MethodPost { + t.Errorf("unexpected Kubernetes request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + object, _, err := scheme.Codecs.UniversalDeserializer().Decode(body, nil, nil) + if err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + if strings.HasSuffix(r.URL.Path, "/deployments") { + deployment = *object.(*appsv1.Deployment) + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"kind":"Status","apiVersion":"v1","status":"Failure","message":"captured deployment","code":403}`)) + return + } + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(object) + })) + defer server.Close() + + t.Setenv("KUBERNETES_SERVICE_HOST", "") + kubeconfig := filepath.Join(t.TempDir(), "config") + content := fmt.Sprintf(`apiVersion: v1 +kind: Config +clusters: +- name: test + cluster: + server: %s +contexts: +- name: test + context: + cluster: test + user: test +current-context: test +users: +- name: test + user: {} +`, server.URL) + if err := os.WriteFile(kubeconfig, []byte(content), 0600); err != nil { + t.Fatal(err) + } + client, err := k8s.NewClient(&config.Config{K8s: config.K8sConfig{Kubeconfig: kubeconfig, NamespacePrefix: "test"}}) + if err != nil { + t.Fatal(err) + } + db, err := gorm.Open(postgres.Open("host=127.0.0.1 user=unused dbname=unused"), &gorm.Config{DisableAutomaticPing: true, DryRun: true}) + if err != nil { + t.Fatal(err) + } + pool, _ := db.DB() + defer pool.Close() + vaultClient, err := vault.New(vault.Config{Addr: server.URL, RoleID: "unused", SecretID: "unused", K8sAuthMount: "farm-auth"}) + if err != nil { + t.Fatal(err) + } + o := &Orchestrator{db: db, k8s: client, vault: vaultClient, vaultAddr: server.URL} + session := &model.SetupSession{ID: 9, UserID: 7, VtcDid: "did:webvh:community", MediatorDid: "did:webvh:mediator", TrustRegistryImage: "registry:test-build"} + err = o.deployTrustRegistry(context.Background(), "test-7", session, "trust-registry-9") + if err == nil || !strings.Contains(err.Error(), "captured deployment") { + t.Fatalf("did not reach deployment capture: %v", err) + } + pod := deployment.Spec.Template.Spec + if len(pod.Containers) != 1 || len(pod.InitContainers) != 1 { + t.Fatal("missing registry or Vault credential init container") + } + env := map[string]string{} + for _, value := range pod.Containers[0].Env { + env[value.Name] = value.Value + } + for key, expected := range map[string]string{ + "TR_VTA_CONTEXT_ID": "trust-registry-9", + "TR_VTA_CREDENTIAL": "file:///run/trust-registry/credential.json", + "ADMIN_DIDS": "did:webvh:community", + "MEDIATOR_DID": "did:webvh:mediator", + "ENABLE_DIDCOMM": "true", + "ENABLE_TSP": "false", + "TR_SECRETS_VAULT_K8S_MOUNT": "farm-auth", + } { + if env[key] != expected { + t.Errorf("runtime %s = %q, want %q", key, env[key], expected) + } + } + if pod.InitContainers[0].Image != registryVaultInitImage || deployment.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType { + t.Fatal("wrong credential init image or concurrent storage writers") + } + for _, volume := range pod.Volumes { + if volume.Name == "registry-credential" && volume.EmptyDir != nil && volume.EmptyDir.Medium == corev1.StorageMediumMemory { + return + } + } + t.Fatal("credential volume is not memory-backed") +} diff --git a/internal/setup/trust_registry_failure_test.go b/internal/setup/trust_registry_failure_test.go new file mode 100644 index 0000000..f580136 --- /dev/null +++ b/internal/setup/trust_registry_failure_test.go @@ -0,0 +1,23 @@ +package setup + +import ( + "strings" + "testing" +) + +func TestRegistryJobFailureRetainsUsefulDiagnosticsWithoutRawLogs(t *testing.T) { + const diagnostic = "could not sign the DID-management task with this VTA's operational key" + for _, output := range []string{ + "Error: " + diagnostic + "\nprivateKeyMultibase=test-key-never-persist", + "privateKeyMultibase=test-key-never-persist", + "", + } { + failure := registryJobFailure("identity", "Job has reached the specified backoff limit", output).Error() + if !strings.Contains(failure, "identity job failed") || strings.Contains(failure, "test-key-never-persist") || strings.Contains(failure, "check this setup job's logs") { + t.Fatal("failure summary leaked raw logs or required a deleted Pod") + } + if strings.Contains(output, diagnostic) && !strings.Contains(failure, diagnostic) { + t.Fatal("failure summary lost the known signing error") + } + } +} diff --git a/internal/setup/trust_registry_init_test.go b/internal/setup/trust_registry_init_test.go new file mode 100644 index 0000000..f58129b --- /dev/null +++ b/internal/setup/trust_registry_init_test.go @@ -0,0 +1,106 @@ +package setup + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/vault" +) + +func TestRegistryAvailableWithoutHelperImageConfiguration(t *testing.T) { + o := &Orchestrator{k8s: &k8s.Client{}, vault: &vault.Client{}} + if !o.RegistryAvailable() { + t.Fatal("Registry must not require a separately configured helper image") + } + o.vault = nil + if o.RegistryAvailable() { + t.Fatal("Registry must still require Vault") + } +} + +func TestRegistryCredentialInitKeepsSecretsPrivateAndCleansUpFailures(t *testing.T) { + for _, fail := range []bool{false, true} { + name := "success" + if fail { + name = "failed read" + } + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + bin := filepath.Join(dir, "vault") + fakeCLI := `#!/bin/sh +set -eu +case "$1" in +write) + test "$2" = -field=token + test "$3" = auth/farm-auth/login + test "$4" = role=vtafarm-user-7 + test "$5" = jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token + printf '%s' "$TEST_TOKEN" + ;; +kv) + test "$VAULT_TOKEN" = "$TEST_TOKEN" + test "$2" = get + test "$3" = -mount=secret + test "$4" = -field=credential + test "$5" = trust-registry/user-7/session-9/credential + printf '%s' "$TEST_CREDENTIAL" + test "$TEST_FAIL" = false + ;; +token) + test "$2" = revoke + test "$3" = -self + touch "$TEST_REVOKED" + ;; +*) exit 99 ;; +esac +` + if err := os.WriteFile(bin, []byte(fakeCLI), 0700); err != nil { + t.Fatal(err) + } + outputDir := filepath.Join(dir, "credential") + credentialFile := filepath.Join(outputDir, "credential.json") + const token = "test-token-never-log" + const credential = `{"did":"did:key:test","privateKeyMultibase":"test-key-never-log"}` + failed := "false" + if fail { + failed = "true" + } + cmd := exec.Command("sh", "-ec", strings.ReplaceAll(registryCredentialInit, "/run/trust-registry", outputDir)) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), + "TR_SECRETS_VAULT_K8S_MOUNT=farm-auth", "TR_SECRETS_VAULT_K8S_ROLE=vtafarm-user-7", + "TR_SECRETS_VAULT_KV_MOUNT=secret", "TR_CREDENTIAL_VAULT_PATH=trust-registry/user-7/session-9/credential", + "TEST_TOKEN="+token, "TEST_CREDENTIAL="+credential, "TEST_FAIL="+failed, "TEST_REVOKED="+filepath.Join(dir, "revoked")) + output, err := cmd.CombinedOutput() + if (err != nil) != fail { + t.Fatalf("unexpected init result: %v", err) + } + if strings.Contains(string(output), token) || strings.Contains(string(output), "test-key-never-log") { + t.Fatal("init leaked credentials to container logs") + } + if _, err := os.Stat(filepath.Join(dir, "revoked")); err != nil { + t.Fatal("init did not revoke its Vault token") + } + if _, err := os.Stat(credentialFile + ".tmp"); !os.IsNotExist(err) { + t.Fatal("init retained temporary credential material") + } + if fail { + if _, err := os.Stat(credentialFile); !os.IsNotExist(err) { + t.Fatal("failed init published a partial credential") + } + return + } + content, err := os.ReadFile(credentialFile) + if err != nil || string(content) != credential { + t.Fatal("init did not write the credential") + } + info, err := os.Stat(credentialFile) + if err != nil || info.Mode().Perm() != 0600 { + t.Fatal("credential permissions are not 0600") + } + }) + } +} diff --git a/internal/upgrade/runner.go b/internal/upgrade/runner.go index c1f92e5..11430db 100644 --- a/internal/upgrade/runner.go +++ b/internal/upgrade/runner.go @@ -156,6 +156,10 @@ func (r *Runner) runTask(batch *model.UpgradeBatch, taskID uint) { return } + if session.TrustRegistryState == "enabling" { + r.failTask(batch, &task, "Trust Registry enablement is in progress") + return + } deployName, err := deploymentName(&session, task.Component) if err != nil { r.failTask(batch, &task, err.Error()) @@ -290,6 +294,8 @@ func deploymentName(s *model.SetupSession, component string) (string, error) { return k8s.FSMediatorName(s.ID), nil case "dids": return k8s.FSDidsName(s.ID), nil + case "trust-registry": + return k8s.FSTrustRegistryName(s.ID), nil case "vtc": return k8s.FSVtcName(s.ID), nil } diff --git a/internal/vault/client.go b/internal/vault/client.go index 9b40b67..2e773f0 100644 --- a/internal/vault/client.go +++ b/internal/vault/client.go @@ -75,6 +75,8 @@ func New(cfg Config) (*Client, error) { // KVMount returns the configured KV v2 mount (e.g. "secret"). func (c *Client) KVMount() string { return c.cfg.KVMount } +func (c *Client) K8sAuthMount() string { return c.cfg.K8sAuthMount } + // VtaPrefix is the KV v2 path (under the mount) containing a session's VTA // secrets. The farm Vault policy globs over secret/{data,metadata}/vta/user-/*. func VtaPrefix(userID, sessionID uint) string { @@ -140,7 +142,9 @@ func (c *Client) EnsureUserAccess(ctx context.Context, userID uint, namespace, s `path "%[1]s/data/dids/user-%[2]d/*" { capabilities = ["create", "update", "read", "delete"] }`+"\n"+ `path "%[1]s/metadata/dids/user-%[2]d/*" { capabilities = ["read", "list", "delete"] }`+"\n"+ `path "%[1]s/data/vtc/user-%[2]d/*" { capabilities = ["read", "create", "update", "delete"] }`+"\n"+ - `path "%[1]s/metadata/vtc/user-%[2]d/*" { capabilities = ["read", "delete"] }`, + `path "%[1]s/metadata/vtc/user-%[2]d/*" { capabilities = ["read", "delete"] }`+"\n"+ + `path "%[1]s/data/trust-registry/user-%[2]d/*" { capabilities = ["read", "create", "update", "delete"] }`+"\n"+ + `path "%[1]s/metadata/trust-registry/user-%[2]d/*" { capabilities = ["read", "list", "delete"] }`, c.cfg.KVMount, userID, ) if err := c.do(ctx, http.MethodPut, "/v1/sys/policies/acl/"+name, token, diff --git a/internal/vault/trust_registry.go b/internal/vault/trust_registry.go new file mode 100644 index 0000000..12df36f --- /dev/null +++ b/internal/vault/trust_registry.go @@ -0,0 +1,86 @@ +package vault + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/ic3software/vtafarm-api/internal/didkey" + "github.com/ic3software/vtafarm-api/internal/didresolver" +) + +func TrustRegistryPrefix(userID, sessionID uint) string { + return fmt.Sprintf("trust-registry/user-%d/session-%d", userID, sessionID) +} + +type RegistryCredential struct { + Did string `json:"did"` + PrivateKeyMultibase string `json:"privateKeyMultibase"` + VtaDid string `json:"vtaDid"` + VtaURL string `json:"vtaUrl"` +} + +// RegistryCredential creates once with Vault CAS=0 so retries keep the ACL identity. +func (c *Client) RegistryCredential(ctx context.Context, userID, sessionID uint, vtaDid, vtaURL string) (RegistryCredential, error) { + token, err := c.login(ctx) + if err != nil { + return RegistryCredential{}, err + } + path := "/v1/" + c.cfg.KVMount + "/data/" + TrustRegistryPrefix(userID, sessionID) + "/credential" + read := func() (RegistryCredential, error) { + var response struct { + Data struct { + Data map[string]string `json:"data"` + } `json:"data"` + } + if err := c.do(ctx, http.MethodGet, path, token, nil, &response); err != nil { + return RegistryCredential{}, err + } + var credential RegistryCredential + if err := json.Unmarshal([]byte(response.Data.Data["credential"]), &credential); err != nil { + return credential, errors.New("invalid registry credential in Vault") + } + if credential.Did == "" || credential.PrivateKeyMultibase == "" || credential.VtaDid != vtaDid { + return credential, errors.New("registry credential does not match this VTA") + } + return credential, nil + } + credential, err := read() + if err == nil { + return credential, nil + } + var response *responseError + if !errors.As(err, &response) || response.statusCode != http.StatusNotFound { + return credential, err + } + pub, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return credential, err + } + did, err := didkey.FromPublicKey(pub) + if err != nil { + return credential, err + } + credential = RegistryCredential{Did: did, PrivateKeyMultibase: "z" + didresolver.EncodeBase58BTC(append([]byte{0x80, 0x26}, priv.Seed()...)), VtaDid: vtaDid, VtaURL: vtaURL} + encoded, err := json.Marshal(credential) + if err != nil { + return RegistryCredential{}, err + } + if err := c.do(ctx, http.MethodPost, path, token, map[string]any{"options": map[string]int{"cas": 0}, "data": map[string]string{"credential": string(encoded)}}, nil); err != nil { + // Another worker may have won the CAS; read its credential rather than rotating it. + return read() + } + return credential, nil +} + +func (c *Client) DeleteTrustRegistrySecrets(ctx context.Context, userID, sessionID uint) error { + token, err := c.login(ctx) + if err != nil { + return err + } + return c.deleteSecretTree(ctx, token, TrustRegistryPrefix(userID, sessionID)) +} diff --git a/internal/vault/trust_registry_test.go b/internal/vault/trust_registry_test.go new file mode 100644 index 0000000..1049c09 --- /dev/null +++ b/internal/vault/trust_registry_test.go @@ -0,0 +1,74 @@ +package vault + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "net/http" + "strings" + "testing" + + "github.com/ic3software/vtafarm-api/internal/didkey" + "github.com/ic3software/vtafarm-api/internal/didresolver" +) + +func TestRegistryCredentialIsScopedStableAndCASProtected(t *testing.T) { + stored := "" + writes := 0 + client, closeServer := testClient(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/secret/data/trust-registry/user-7/session-9/credential" { + t.Error("wrong secret scope") + http.Error(w, "not found", 404) + return + } + if r.Method == http.MethodGet { + if stored == "" { + http.Error(w, "missing", 404) + return + } + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"data": map[string]string{"credential": stored}}}) + return + } + var request struct { + Options struct { + CAS int `json:"cas"` + } `json:"options"` + Data map[string]string `json:"data"` + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Fatal(err) + } + if request.Options.CAS != 0 { + t.Error("credential creation must use CAS=0") + } + stored = request.Data["credential"] + writes++ + w.WriteHeader(http.StatusNoContent) + }) + defer closeServer() + first, err := client.RegistryCredential(context.Background(), 7, 9, "did:webvh:vta", "https://vta.example") + if err != nil { + t.Fatal(err) + } + second, err := client.RegistryCredential(context.Background(), 7, 9, "did:webvh:vta", "https://vta.example") + if err != nil { + t.Fatal(err) + } + if first != second || writes != 1 { + t.Fatal("retry rotated the registry access credential") + } + key, err := didresolver.DecodeBase58BTC(strings.TrimPrefix(first.PrivateKeyMultibase, "z")) + if err != nil { + t.Fatal(err) + } + if len(key) != 34 || key[0] != 0x80 || key[1] != 0x26 { + t.Fatal("invalid Ed25519 secret multicodec") + } + derived, _ := didkey.FromPublicKey(ed25519.NewKeyFromSeed(key[2:]).Public().(ed25519.PublicKey)) + if derived != first.Did { + t.Fatal("credential DID does not match the private key") + } + if _, err = client.RegistryCredential(context.Background(), 7, 9, "did:webvh:other", "https://other.example"); err == nil { + t.Fatal("accepted credential for another VTA") + } +} diff --git a/migrations/000044_optional_trust_registry.down.sql b/migrations/000044_optional_trust_registry.down.sql new file mode 100644 index 0000000..112d2b0 --- /dev/null +++ b/migrations/000044_optional_trust_registry.down.sql @@ -0,0 +1,23 @@ +DO $$ BEGIN + IF EXISTS (SELECT 1 FROM setup_sessions WHERE trust_registry_enabled) THEN + RAISE EXCEPTION 'Trust Registry sessions still exist; refusing to discard their inventory'; + END IF; +END $$; +ALTER TABLE setup_sessions + DROP CONSTRAINT setup_sessions_trust_registry_mode, + DROP CONSTRAINT setup_sessions_trust_registry_state, + DROP COLUMN trust_registry_enabled, + DROP COLUMN trust_registry_image, + DROP COLUMN trust_registry_did, + DROP COLUMN trust_registry_state, + DROP COLUMN trust_registry_phase, + DROP COLUMN trust_registry_error; + +DELETE FROM resource_defaults WHERE component = 'trust-registry'; +DELETE FROM workload_resources WHERE component = 'trust-registry'; +ALTER TABLE resource_defaults DROP CONSTRAINT resource_defaults_component_check; +ALTER TABLE resource_defaults ADD CONSTRAINT resource_defaults_component_check + CHECK (component IN ('vta', 'mediator', 'dids', 'vtc')); +ALTER TABLE workload_resources DROP CONSTRAINT workload_resources_component_check; +ALTER TABLE workload_resources ADD CONSTRAINT workload_resources_component_check + CHECK (component IN ('vta', 'mediator', 'dids', 'vtc')); diff --git a/migrations/000044_optional_trust_registry.up.sql b/migrations/000044_optional_trust_registry.up.sql new file mode 100644 index 0000000..a0c42a3 --- /dev/null +++ b/migrations/000044_optional_trust_registry.up.sql @@ -0,0 +1,18 @@ +ALTER TABLE setup_sessions + ADD COLUMN trust_registry_enabled boolean NOT NULL DEFAULT false, + ADD COLUMN trust_registry_image text NOT NULL DEFAULT '', + ADD COLUMN trust_registry_did text NOT NULL DEFAULT '', + ADD COLUMN trust_registry_state text NOT NULL DEFAULT 'disabled', + ADD COLUMN trust_registry_phase text NOT NULL DEFAULT '', + ADD COLUMN trust_registry_error text NOT NULL DEFAULT '', + ADD CONSTRAINT setup_sessions_trust_registry_mode CHECK + (NOT trust_registry_enabled OR mode = 'full_stack'), + ADD CONSTRAINT setup_sessions_trust_registry_state CHECK + (trust_registry_state IN ('disabled', 'enabling', 'active', 'failed')); + +ALTER TABLE resource_defaults DROP CONSTRAINT resource_defaults_component_check; +ALTER TABLE resource_defaults ADD CONSTRAINT resource_defaults_component_check + CHECK (component IN ('vta', 'mediator', 'dids', 'vtc', 'trust-registry')); +ALTER TABLE workload_resources DROP CONSTRAINT workload_resources_component_check; +ALTER TABLE workload_resources ADD CONSTRAINT workload_resources_component_check + CHECK (component IN ('vta', 'mediator', 'dids', 'vtc', 'trust-registry')); From 86a61bf6a70de4922daea0867e5a72f74676d5e4 Mon Sep 17 00:00:00 2001 From: vthwang Date: Thu, 22 Oct 2026 13:48:40 -0700 Subject: [PATCH 3/7] feat: use headless trust registry setup Prepare Registry identities before VTC creation and preserve resumable setup receipts. - replace Vault-delivered bootstrap credentials with the Registry headless setup flow - surface sanitized Job logs and retain recovery receipts - pass the Registry DID through new Full Stack VTC setup Signed-off-by: vthwang --- docs/optional-trust-registry-design.md | 128 +++++---- internal/handler/setup_fullstack.go | 5 +- internal/handler/setup_trust_registry_test.go | 31 +++ internal/k8s/fullstack_names.go | 3 + internal/k8s/setup_jobs.go | 12 +- internal/k8s/trust_registry.go | 4 +- internal/setup/orchestrator_vtc.go | 3 + internal/setup/templates_vtc.go | 4 + internal/setup/templates_vtc_test.go | 18 ++ internal/setup/trust_registry.go | 201 +++++--------- .../setup/trust_registry_deployment_test.go | 22 +- internal/setup/trust_registry_failure_test.go | 146 +++++++++- internal/setup/trust_registry_init_test.go | 90 +------ internal/setup/trust_registry_setup.go | 170 ++++++++++++ internal/setup/trust_registry_setup_test.go | 253 ++++++++++++++++++ internal/vault/trust_registry.go | 68 ----- internal/vault/trust_registry_test.go | 74 ----- 17 files changed, 792 insertions(+), 440 deletions(-) create mode 100644 internal/setup/trust_registry_setup.go create mode 100644 internal/setup/trust_registry_setup_test.go delete mode 100644 internal/vault/trust_registry_test.go diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md index 121203d..5b27be0 100644 --- a/docs/optional-trust-registry-design.md +++ b/docs/optional-trust-registry-design.md @@ -1,7 +1,7 @@ # Optional Trust Registry for Full Stack — Design Status: **implemented in the working branch; image publication and staging acceptance pending**. -Research checked on 2026-10-20. +Headless setup integration checked on 2026-10-22; live staging acceptance remains pending. Add a per-session Trust Registry to `full_stack` as an opt-in component, both when creating a stack and when adding it to an existing running stack. Existing @@ -67,6 +67,12 @@ Research found a successful build publishing `0.28.0-c534746` and `nightly` on 2026-10-20 ([build run][image-run]). This is evidence of publication, not an end-to-end compatibility result or a final version selection. +On 2026-10-22, `0.28.0-be33b15` was pulled and checked locally: its headless +setup flags, phase-1 key-file permissions, recipe validation, and refusal to +overwrite an existing credential match this integration. This was not a live +VTA provisioning test. VTA revision `4df0200f` includes the required template +from PR #2252; no image pins are changed by this integration. + The [image Dockerfile][image-dockerfile] downloads a precompiled `trust-registry-k8s` binary. Unlike the upstream Dockerfile, this repository does not compile Rust or declare Cargo features. Check the binary producer and verify @@ -134,33 +140,28 @@ Do not regenerate other service configurations. The mediator and DID host are reused through their normal account/publication operations, with existing policy changes only if preflight proves they are necessary. -### Vault: two separate secrets - -1. **VTA access credential**: used by the registry to authenticate to its context. -2. **Identity bundle cache**: fetched from the VTA and used for messaging and - offline startup. This contains private key material. - -Store both in separate keys/paths beneath a proposed -`trust-registry/user-{uid}/session-{sid}` Vault prefix. Extend per-user Vault -policy and the API's bootstrap cleanup permissions for this prefix. Reuse -Kubernetes authentication; do not introduce static Vault tokens or plaintext -credential/cache persistence on the PVC. - -The registry supports Vault for the cache through `TR_SECRETS_VAULT_*` settings -when built with `secrets-vault`. Its `TR_VTA_CREDENTIAL` loader separately supports -file/string/AWS sources; the reviewed loader has no Vault URI implementation. -Therefore selecting Vault for the cache does not solve credential delivery. - -Credential delivery: a pod-local Vault CLI init container authenticates to Vault using -the ServiceAccount, reads the context-scoped credential, and writes a `0600` file -to a memory-backed `emptyDir`. The registry receives a `file://` reference to that -file. Persist the credential only in Vault; keep values out of Deployment env, -ConfigMaps, API responses, and Job logs. This needs init-container and temporary -volume support in the component helper. The init container uses the pinned -`hashicorp/vault:2.0.2` image, matching the Farm Vault; it does not depend on a -published Farm API image or a separately configured helper image. It revokes its -short-lived login token after reading the credential. -The cache uses a different Vault location so it cannot overwrite the credential. +### Credential file and Vault identity cache + +The headless `trust-registry setup` command now owns credential creation. It writes +`/work/trust-registry/vta-credential.json` with mode `0600` on the Registry PVC; +the Deployment uses that file directly through `TR_VTA_CREDENTIAL`. The temporary +setup key lives on the same PVC. Farm neither generates private keys nor copies +credential values through Job logs, ConfigMaps, database rows, or Deployment env. +The PVC therefore contains sensitive state and its backups must protect both the +record store and the VTA credential. + +The identity bundle cache remains in Vault at +`trust-registry/user-{uid}/session-{sid}/identity-cache`, using Kubernetes auth +and the existing per-user policy. Building with `secrets-vault` is still required. +There is no credential-fetching init container, Vault CLI helper image, or +Farm-minted permanent access DID in the headless flow. Legacy Vault credentials +are left untouched until ordinary session teardown. + +Provisioning advertises only DIDComm, with the existing mediator. Farm keeps +`ENABLE_REST=true` at runtime solely for its internal HTTP service and `/health`. +It does not set `TR_PUBLIC_URL`, so the runtime DID builder advertises no REST +endpoint and agrees with the provisioned DID. Loading `vta.env` verbatim would +turn off `/health`; the Deployment supplies this deliberate internal override. The registry obtains and uses private keys at runtime. The setup guide's phrase "no private keys of its own" means VTA-managed identity, not remote-only signing. @@ -175,22 +176,28 @@ When opted in, extend the post-admin-gate flow: ```text existing bootstrap and admin gate -> deploy VTA (mediator and DID host already ready) - -> VTC setup and deploy VTC (existing core pipeline) - -> provision registry context, DID and Vault credential + -> trust-registry setup --setup-key-out (Registry PVC) + -> stop VTA, grant setup DID with --admin-expires 1h --admin-handoff, restart VTA + -> trust-registry setup --from (running VTA mints/publishes DID and admin credential) + -> persist registry_did from the public key=value completion block + -> VTC setup with registry_did, then deploy VTC -> configure registry ADMIN_DIDS with the VTC DID -> deploy registry and check its messaging/write-path status - -> append the signed VTC referral through the VTA -> merge [registry].did into the VTC runtime config and restart VTC -> verify the published referral and finish add-on enablement -> running (Registry has its own active/failed state) ``` -Use headless native VTA CLI Jobs under the session maintenance claim. -Do not assume PNM is installed/configured in the API or copy a human's PNM -credential into the Farm. The VTA creates the context and hosted DID, and imports -a generated, context-scoped access DID whose credential is persisted with Vault -CAS=0. Preserve existing VTC output/claim handling; -account for its short install-token TTL when adding steps after VTC setup. +Follow [upstream headless setup][tr-headless], using Registry Jobs for both setup +phases and a single offline VTA context grant. No PNM credential or offline +`vta did-mgmt dids create` chain is needed. The Registry image must include +headless setup plus `vta`, `secrets-vault`, `storage-fjall` and `standalone`; the VTA +must include [the trust-registry template][vta-tr-template]. + +A Registry setup failure is recorded independently and lets VTC setup proceed +without a referral. It is not immediately retried at the end of that same run. +The user can retry enablement after correcting the cause. Preserve the VTC's +existing claim/install output; its setup now follows Registry provisioning. ## 6. Add to an existing stack without rebuilding @@ -204,11 +211,11 @@ existing PVCs, or re-enroll the existing administrators. DID ownership/history, and any active upgrade/recovery operation. 2. Persist the operation and chosen image/identity parameters before mutation. Provision only the registry resources and Vault locations. -3. Generate or reuse a context-scoped registry access credential in Vault. - Native VTA CLI Jobs perform provisioning and DID editing while the VTA is - stopped; no lasting unrestricted setup credential is introduced. -4. Provision the registry identity through the existing VTA and publish it on - the existing DID host. Reuse a completed result on retry. +3. Run Registry phase 1 on its PVC. Stop the VTA only for the context-scoped + setup grant, with `--admin-expires 1h --admin-handoff`, then restart it. +4. Run Registry phase 2 against the live VTA. It publishes the DID on the + registered `dids` server and writes the long-term credential locally. + Reuse successful Job receipts on retry. 5. Set `ADMIN_DIDS` to the existing `session.VtcDid`, start the registry, and verify its advertised transport and write path. 6. Append a signed update to the **existing** VTC DID history through the VTA, @@ -229,6 +236,14 @@ Upstream explicitly supports changing the referral after initial mint via VTA-side DID editing ([setup schema][vtc-setup]). This requires a signed DID-log update, not a change to a locally served JSON file alone. +The existing Farm referral update still uses offline `vta did-mgmt dids edit`. +That CLI has only REST available for publishing to an external DID host. A host +advertising only DIDComm/TSP therefore still blocks **existing-stack referral +updates**. Headless Registry provisioning does not fix this separate path; +it needs an authenticated online VTA edit or upstream offline transport support. +New stacks avoid that edit by passing `registry_did` to VTC setup. Do not report +existing-stack activation as verified until the referral publication succeeds. + **Downtime:** the VTC needs a restart for the new client/syncer. Provisioning authorization may also need a brief VTA maintenance window: current Farm ACL operations use `scale to 0 -> offline Job -> restart`. Prefer compatible live @@ -241,6 +256,22 @@ The existing config handler operates on the selected component only and backs up/restores its config (`internal/handler/setup_config.go`). Reuse that behavior for the VTC; its readiness check is not by itself a registry-write acceptance test. +### Headless retries and legacy attempts + +The new context is `trust-registry-{sid}-headless`. The earlier offline flow's +`trust-registry-{sid}` context did not carry admin-handoff; it is never silently +repurposed or deleted. Successful setup-key, ACL-grant, and setup Jobs have no +TTL and remain receipts. Retrying uses the same setup key and completion output; +it does not re-run a successful context creation or replace a credential file. + +Upstream phase 2 refuses an existing credential file. If provisioning was +interrupted after creating the remote identity but before saving a complete +receipt, keep the original output and files for reconciliation; do not delete +the credential or context to make a retry pass. An expired/consumed one-hour grant +also requires reconciliation rather than a permanent `import-did` fallback. +An already-provisioned legacy registry needs a separate credential/context +migration before converting its Deployment to the headless file layout. + ## 7. Proposed API, persistence, and retries Add optional `trust_registry_enabled` and `trust_registry_image` to Full Stack @@ -272,14 +303,14 @@ the already deployed four-component stack. After persisting an add-on failure, remove its failed Jobs and their Pods, ConfigMaps, and temporary input Secrets. Keep the phase and sanitized error in -the session for the UI. Successful identity Jobs remain recovery receipts; +the session for the UI. Successful setup-key, ACL-grant, and setup Jobs remain recovery receipts; active Jobs are never removed by failure cleanup. Retrying reuses the persisted context, credential and DID rather than relying on a failed Pod. Use a durable per-session operation claim, compatible with existing maintenance locking and startup resumption. A client timeout or API restart must not lose the operation or allow another worker to execute it concurrently. Reconcile completed -steps before retrying: the sealed bootstrap request is single-use, context creation +steps before retrying: the temporary admin handoff is single-use, context creation may already have succeeded, and DID-log updates must not mint another DID. Before VTC wiring, failure leaves the original stack configuration intact. @@ -323,7 +354,7 @@ memory request, 256Mi memory limit, and 200Mi PVC. These are provisional factory values, not measured production sizing. Validate them against expected records and traffic in staging; memory defaults can be changed through the Farm UI. -Back up the record PVC and account for both Vault secrets. Extend deletion to the +Back up the Registry PVC (records and credential) and the Vault identity cache. Extend deletion to the registry Jobs, Deployment, Service, PVC, and session Vault prefix. Audit policies and bootstrap permissions must agree on the new prefix. A Farm rollback or a down-migration must not silently orphan running registries or drop their data. @@ -333,7 +364,7 @@ down-migration must not silently orphan running registries or drop their data. 1. Verify pinned image features/entrypoint, production component compatibility, headless provisioning and DID editing, and the existing-member reconciliation mechanism. Resolve these gates before touching a live stack. -2. Implement opt-in persistence, inventory, Vault policy/credential delivery, +2. Implement opt-in persistence, inventory, Vault cache policy and headless credential creation, Kubernetes resources, capacity, and the shared resumable enablement steps. 3. Wire the new-stack branch and existing-stack enable/retry routes, maintenance and upgrade exclusions, config restoration, and startup recovery. @@ -351,7 +382,7 @@ Required acceptance coverage: the existing VTC can publish, query, update, and remove a consented test member. - Non-consenting members remain unpublished, including during initial backfill. - Restarts preserve trust records; populated Vault cache supports the documented - VTA-offline case without plaintext persistence. + VTA-offline case without a plaintext identity-cache copy. - Failures/restarts at credential handoff, DID publication, config update, and deployment resume without duplicate contexts/DIDs or a stuck stopped VTC/VTA. - Add-on failure remains visible without invalidating the existing parent stack; @@ -375,3 +406,6 @@ passing Go checks alone cannot establish them. [tr-vta]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/vta.rs [tr-secrets]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/secret_store.rs [tr-loaders]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/c534746480d55fc1e86b40528c5ce7d2fea48184/trust-registry/src/configs/loaders/mod.rs + +[tr-headless]: https://github.com/affinidi/affinidi-trust-registry-rs/blob/be33b15/VTA_SETUP.md#headless-setup-automated +[vta-tr-template]: https://github.com/OpenVTC/verifiable-trust-infrastructure/pull/2252 diff --git a/internal/handler/setup_fullstack.go b/internal/handler/setup_fullstack.go index 548057b..9e6e142 100644 --- a/internal/handler/setup_fullstack.go +++ b/internal/handler/setup_fullstack.go @@ -465,6 +465,9 @@ func (h *SetupHandler) logsFullStack(c *gin.Context, session *model.SetupSession source = "vtc_acl_grant" case "step_vtc_setup": source = "vtc_setup" + if session.TrustRegistryEnabled && session.TrustRegistryState != "failed" && session.TrustRegistryDid == "" { + source = "registry_operation" + } case "step_trust_registry": source = "registry_operation" case "deploy_vtc": @@ -488,7 +491,7 @@ func (h *SetupHandler) logsFullStack(c *gin.Context, session *model.SetupSession case "trust-registry": streamPod("trust-registry") case "registry_operation": - jobs := map[string]string{"preflight": "preflight", "provision_identity": "identity", "publish_referral": "referral", "configure_vtc": "config"} + jobs := map[string]string{"preflight": "preflight", "setup_key": "setup-key", "grant_access": "acl-grant", "provision_identity": "setup", "publish_referral": "referral", "configure_vtc": "config"} if phase := jobs[session.TrustRegistryPhase]; phase != "" { streamJob(setup.RegistryJobName(sid, phase)) } else { diff --git a/internal/handler/setup_trust_registry_test.go b/internal/handler/setup_trust_registry_test.go index 5174eec..5079dd7 100644 --- a/internal/handler/setup_trust_registry_test.go +++ b/internal/handler/setup_trust_registry_test.go @@ -1,6 +1,7 @@ package handler import ( + "encoding/json" "net/http" "net/http/httptest" "strings" @@ -66,3 +67,33 @@ func TestVtaOnlyRegistryEnablementIsRejectedWithoutInfrastructure(t *testing.T) t.Fatalf("status=%d", w.Code) } } + +func TestRegistryReturnsPersistedJobError(t *testing.T) { + db := testutil.Postgres(t) + session := testutil.SetupSession(t, db, model.ModeFullStack) + const failure = "identity job failed (Job has reached the specified backoff limit)\n\n--- Job Logs ---\nError: an unknown failure\nCaused by:\n storage unavailable" + if err := db.Model(&session).Updates(map[string]any{ + "trust_registry_enabled": true, + "trust_registry_state": "failed", + "trust_registry_phase": "provision_identity", + "trust_registry_error": failure, + }).Error; err != nil { + t.Fatal(err) + } + h := &SetupHandler{db: db} + r := gin.New() + r.Use(func(c *gin.Context) { c.Set(middleware.ContextUserID, session.UserID) }) + r.GET("/setup/:id/trust-registry", h.GetTrustRegistry) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/setup/"+session.VtaName+"/trust-registry", nil)) + var response struct { + State string `json:"state"` + Error string `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if w.Code != http.StatusOK || response.State != "failed" || response.Error != failure { + t.Fatalf("stored Job error was not returned intact: %s", w.Body.String()) + } +} diff --git a/internal/k8s/fullstack_names.go b/internal/k8s/fullstack_names.go index a7b26c2..1cb50ee 100644 --- a/internal/k8s/fullstack_names.go +++ b/internal/k8s/fullstack_names.go @@ -110,6 +110,9 @@ func allFSJobNames(sessionID uint) []string { FSJobConfigRollback(sessionID), fmt.Sprintf("fs-%d-tr-preflight", sessionID), fmt.Sprintf("fs-%d-tr-identity", sessionID), + fmt.Sprintf("fs-%d-tr-setup-key", sessionID), + fmt.Sprintf("fs-%d-tr-acl-grant", sessionID), + fmt.Sprintf("fs-%d-tr-setup", sessionID), fmt.Sprintf("fs-%d-tr-referral", sessionID), fmt.Sprintf("fs-%d-tr-config", sessionID), fmt.Sprintf("fs-%d-tr-restore", sessionID), diff --git a/internal/k8s/setup_jobs.go b/internal/k8s/setup_jobs.go index 85f9dcd..7433fcf 100644 --- a/internal/k8s/setup_jobs.go +++ b/internal/k8s/setup_jobs.go @@ -168,7 +168,7 @@ func (c *Client) WaitForJob(ctx context.Context, ns, jobName string) (succeeded } } -// JobLogs returns the full combined stdout/stderr of the job's pod. +// JobLogs returns the full combined stdout/stderr of the job's latest pod. // Returns an error if no pod exists or logs cannot be fetched. func (c *Client) JobLogs(ctx context.Context, ns, jobName string) (string, error) { pods, err := c.kube.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{ @@ -181,10 +181,16 @@ func (c *Client) JobLogs(ctx context.Context, ns, jobName string) (string, error return "", fmt.Errorf("no pods found for job %s", jobName) } - req := c.kube.CoreV1().Pods(ns).GetLogs(pods.Items[0].Name, &corev1.PodLogOptions{}) + pod := &pods.Items[0] + for i := range pods.Items { + if pod.CreationTimestamp.Before(&pods.Items[i].CreationTimestamp) { + pod = &pods.Items[i] + } + } + req := c.kube.CoreV1().Pods(ns).GetLogs(pod.Name, &corev1.PodLogOptions{}) stream, err := req.Stream(ctx) if err != nil { - return "", fmt.Errorf("get logs for pod %s: %w", pods.Items[0].Name, err) + return "", fmt.Errorf("get logs for pod %s: %w", pod.Name, err) } defer stream.Close() diff --git a/internal/k8s/trust_registry.go b/internal/k8s/trust_registry.go index b75b495..f49ece4 100644 --- a/internal/k8s/trust_registry.go +++ b/internal/k8s/trust_registry.go @@ -94,7 +94,7 @@ func (c *Client) FinishRegistryJobs(ctx context.Context, ns string, names []stri } // RegistryEnvironment exports only the non-secret settings set by the Farm. -// The credential is supplied through a memory volume, never through this list. +// The credential stays in its PVC file, never in this list. func (c *Client) RegistryEnvironment(ctx context.Context, ns, selector string) (string, error) { deployments, err := c.kube.AppsV1().Deployments(ns).List(ctx, metav1.ListOptions{LabelSelector: selector}) if err != nil { @@ -103,7 +103,7 @@ func (c *Client) RegistryEnvironment(ctx context.Context, ns, selector string) ( if len(deployments.Items) != 1 || len(deployments.Items[0].Spec.Template.Spec.Containers) != 1 { return "", fmt.Errorf("registry deployment is missing or ambiguous") } - allowed := map[string]bool{"MEDIATOR_DID": true, "ADMIN_DIDS": true, "ACL_MODE": true, "LISTEN_ADDRESS": true, "ENABLE_DIDCOMM": true, "ENABLE_TSP": true, "TR_VTA_CONTEXT_ID": true, "TR_STORAGE_BACKEND": true, "TR_FJALL_PATH": true, "TR_SECRETS_VAULT_ADDR": true, "TR_SECRETS_VAULT_KV_MOUNT": true, "TR_SECRETS_VAULT_SECRET_PATH": true, "TR_SECRETS_VAULT_SECRET_KEY": true, "TR_SECRETS_VAULT_AUTH_METHOD": true, "TR_SECRETS_VAULT_K8S_ROLE": true, "TR_SECRETS_VAULT_K8S_MOUNT": true, "TR_SECRETS_VAULT_SKIP_VERIFY": true} + allowed := map[string]bool{"MEDIATOR_DID": true, "ADMIN_DIDS": true, "ACL_MODE": true, "LISTEN_ADDRESS": true, "ENABLE_REST": true, "ENABLE_DIDCOMM": true, "ENABLE_TSP": true, "TR_VTA_CONTEXT_ID": true, "TR_STORAGE_BACKEND": true, "TR_FJALL_PATH": true, "TR_SECRETS_VAULT_ADDR": true, "TR_SECRETS_VAULT_KV_MOUNT": true, "TR_SECRETS_VAULT_SECRET_PATH": true, "TR_SECRETS_VAULT_SECRET_KEY": true, "TR_SECRETS_VAULT_AUTH_METHOD": true, "TR_SECRETS_VAULT_K8S_ROLE": true, "TR_SECRETS_VAULT_K8S_MOUNT": true, "TR_SECRETS_VAULT_SKIP_VERIFY": true} var lines []string for _, env := range deployments.Items[0].Spec.Template.Spec.Containers[0].Env { if env.ValueFrom != nil { diff --git a/internal/setup/orchestrator_vtc.go b/internal/setup/orchestrator_vtc.go index 8881efb..9b38007 100644 --- a/internal/setup/orchestrator_vtc.go +++ b/internal/setup/orchestrator_vtc.go @@ -201,6 +201,9 @@ func (o *Orchestrator) fsStepVtcAclGrant(ctx context.Context, ns string, s *mode // registered "dids" server, and writes the key bundle to Vault (kubernetes // auth — hence SA vta). Captures the terse completion block. func (o *Orchestrator) fsStepVtcSetup(ctx context.Context, ns string, s *model.SetupSession) (VtcSetupOutcome, error) { + if err := o.prepareRegistryForVtc(ctx, ns, s); err != nil { + return VtcSetupOutcome{}, err + } toml, err := RenderVtcSetupTOML(s, VtcVaultSecrets{ Addr: o.vaultAddr, KVMount: o.vault.KVMount(), diff --git a/internal/setup/templates_vtc.go b/internal/setup/templates_vtc.go index 58dcc57..16453e8 100644 --- a/internal/setup/templates_vtc.go +++ b/internal/setup/templates_vtc.go @@ -32,6 +32,8 @@ vta_did = "{{ .VtaDid }}" context = "{{ .VtcName }}" setup_key_file = "setup-key.json" single_admin_mode = true +{{ if .RegistryDid }}registry_did = {{ printf "%q" .RegistryDid }} +{{ end }} [webvh] server_id = "dids" @@ -69,6 +71,7 @@ type vtcSetupData struct { VtaDid string VtcName string VtcDidPath string + RegistryDid string MediatorDid string MediatorURL string Vault VtcVaultSecrets @@ -86,6 +89,7 @@ func RenderVtcSetupTOML(s *model.SetupSession, vault VtcVaultSecrets) (string, e VtaDid: s.VtaDid, VtcName: s.VtcName, VtcDidPath: VtcDidPath(s.VtcName), + RegistryDid: s.TrustRegistryDid, MediatorDid: s.MediatorDid, MediatorURL: "https://" + s.MediatorFQDN() + "/mediator/v1", Vault: vault, diff --git a/internal/setup/templates_vtc_test.go b/internal/setup/templates_vtc_test.go index df9bd1c..3ad16e3 100644 --- a/internal/setup/templates_vtc_test.go +++ b/internal/setup/templates_vtc_test.go @@ -32,3 +32,21 @@ func TestRenderVtcSetupTOMLEnablesSingleAdminMode(t *testing.T) { t.Fatalf("single_admin_mode = false, want true\n%s", rendered) } } + +func TestVtcSetupPublishesOnlyProvisionedRegistryReferral(t *testing.T) { + for _, did := range []string{"", "did:webvh:registry"} { + s := &model.SetupSession{VtcName: "community", TrustRegistryEnabled: did != "", TrustRegistryDid: did} + rendered, err := RenderVtcSetupTOML(s, VtcVaultSecrets{}) + if err != nil { + t.Fatal(err) + } + var recipe map[string]any + if err := toml.Unmarshal([]byte(rendered), &recipe); err != nil { + t.Fatal(err) + } + value, exists := recipe["registry_did"] + if did == "" && exists || did != "" && value != did { + t.Fatalf("registry_did = %v, expected %q (omitted when empty)", value, did) + } + } +} diff --git a/internal/setup/trust_registry.go b/internal/setup/trust_registry.go index ba56a53..7bcb9c7 100644 --- a/internal/setup/trust_registry.go +++ b/internal/setup/trust_registry.go @@ -9,9 +9,9 @@ import ( "regexp" "strings" "time" + "unicode/utf8" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" "github.com/ic3software/vtafarm-api/internal/k8s" "github.com/ic3software/vtafarm-api/internal/model" @@ -22,22 +22,6 @@ import ( const RegistryPVCSize = "200Mi" -// Match the Farm's Vault image; credential delivery needs only the Vault CLI. -const registryVaultInitImage = "hashicorp/vault:2.0.2" - -const registryCredentialInit = `set -eu -umask 077 -credential_file=/run/trust-registry/credential.json -mkdir -p /run/trust-registry -VAULT_TOKEN="$(vault write -field=token "auth/$TR_SECRETS_VAULT_K8S_MOUNT/login" \ - "role=$TR_SECRETS_VAULT_K8S_ROLE" jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token)" -export VAULT_TOKEN -trap 'rm -f "$credential_file.tmp"; vault token revoke -self >/dev/null 2>&1 || true' EXIT -vault kv get -mount="$TR_SECRETS_VAULT_KV_MOUNT" -field=credential \ - "$TR_CREDENTIAL_VAULT_PATH" > "$credential_file.tmp" -mv "$credential_file.tmp" "$credential_file" -` - func (o *Orchestrator) RegistryAvailable() bool { return o != nil && o.k8s != nil && o.vault != nil } @@ -51,7 +35,7 @@ func RegistryJobName(id uint, phase string) string { return fmt.Sprintf("fs-%d-t func registryJobNames(id uint) []string { var names []string - for _, phase := range []string{"preflight", "identity", "referral", "config", "restore"} { + for _, phase := range []string{"preflight", "identity", "setup-key", "acl-grant", "setup", "referral", "config", "restore"} { names = append(names, RegistryJobName(id, phase)) } return names @@ -63,40 +47,67 @@ func (o *Orchestrator) CleanupFailedRegistryJobs(s *model.SetupSession) error { return o.k8s.CleanupFailedRegistryJobs(ctx, o.k8s.UserNamespace(fmt.Sprint(s.UserID)), registryJobNames(s.ID)) } +var ( + jobPrivateKeyBlock = regexp.MustCompile(`(?s)-----BEGIN [^-]*PRIVATE KEY-----.*?(?:-----END [^-]*PRIVATE KEY-----|$)`) + jobSensitiveLine = regexp.MustCompile(`(?i)(?:private[ _-]?key(?:multibase|hex|jwk)?(?:[ \t]*\([^\r\n)]*\))?|password|passwd|(?:client[ _-]?)?secret|secret[ _-]?id|(?:(?:access|refresh|vault|api)[ _-]?)?token|api[ _-]?key|authorization|cookie|credential)["']?[ \t]*[:=]|bearer[ \t]+[a-z0-9._~+/-]{16,}=*|://[^\s/@]+:[^\s/@]+@`) +) + func registryJobFailure(phase, reason, output string) error { - // Only retain known diagnostic text; arbitrary CLI output can contain keys - // or configuration values and must not be copied into the session response. - for _, diagnostic := range []string{ - "could not sign the DID-management task with this VTA's operational key", - "Registry image needs the vta feature", - "Registry image needs Vault Kubernetes authentication", - } { - if strings.Contains(output, diagnostic) { - return fmt.Errorf("%s job failed: %s (%s)", phase, diagnostic, reason) + // Keep arbitrary diagnostics, but redact credential output before bounding + // the stored tail so truncation cannot split a private-key block. + output = jobPrivateKeyBlock.ReplaceAllString(stripANSI(output), "[redacted private key]") + lines := strings.Split(output, "\n") + for i, line := range lines { + if jobSensitiveLine.MatchString(line) { + lines[i] = "[redacted sensitive log line]" + } + } + output = strings.TrimSpace(strings.Join(lines, "\n")) + const maxBytes = 16 * 1024 + if len(output) > maxBytes { + output = output[len(output)-maxBytes:] + for len(output) > 0 && !utf8.RuneStart(output[0]) { + output = output[1:] } + output = "[earlier job output truncated]\n" + output + } + message := fmt.Sprintf("%s job failed (%s)", phase, reason) + if output != "" { + message += "\n\n--- Job Logs ---\n" + output } - return fmt.Errorf("%s job failed (%s)", phase, reason) + return errors.New(message) } -func (o *Orchestrator) registryJob(ctx context.Context, ns string, s *model.SetupSession, phase, image, dir, command string, mounts []k8s.PVCMount, inputs map[string][]byte) (string, error) { +func (o *Orchestrator) registryJob(ctx context.Context, ns string, s *model.SetupSession, phase, image, dir string, command []string, mounts []k8s.PVCMount, inputs map[string][]byte) (string, error) { name := RegistryJobName(s.ID, phase) - if err := o.k8s.PrepareRegistryJob(ctx, ns, name, phase == "identity"); err != nil { + if err := o.k8s.PrepareRegistryJob(ctx, ns, name, phase == "identity" || phase == "setup-key" || phase == "acl-grant" || phase == "setup"); err != nil { return "", err } - if err := o.k8s.CreateComponentJob(ctx, ns, k8s.ComponentJobSpec{Name: name, Image: image, Command: []string{"sh", "-c", command}, WorkingDir: dir, ServiceAccount: k8s.VtaServiceAccount, PVCMounts: mounts, SecretData: inputs, Env: fsNoColorEnv(), RetainForRecovery: true}); err != nil { + if err := o.k8s.CreateComponentJob(ctx, ns, k8s.ComponentJobSpec{Name: name, Image: image, Command: command, WorkingDir: dir, ServiceAccount: k8s.VtaServiceAccount, PVCMounts: mounts, SecretData: inputs, Env: fsNoColorEnv(), RetainForRecovery: true}); err != nil { return "", err } ok, reason, err := o.k8s.WaitForJob(ctx, ns, name) - if err != nil { - cleanup, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - defer cancel() - o.k8s.DeleteComponentJob(cleanup, ns, name) - stopErr := o.k8s.WaitForComponentPodsGone(cleanup, ns, "job-name="+name, time.Minute) - return "", errors.Join(err, stopErr) - } - if !ok { - output, _ := o.k8s.JobLogs(ctx, ns, name) - return "", registryJobFailure(phase, reason, output) + if err != nil || !ok { + // A timed-out job still has useful logs; read them before cleanup using + // a fresh context because the operation context may already be expired. + logsCtx, cancelLogs := context.WithTimeout(context.Background(), 15*time.Second) + output, logsErr := o.k8s.JobLogs(logsCtx, ns, name) + cancelLogs() + if err != nil { + reason = err.Error() + } + failure := registryJobFailure(phase, reason, output) + if logsErr != nil { + failure = errors.Join(failure, fmt.Errorf("could not read job logs: %w", logsErr)) + } + if err != nil { + cleanup, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + o.k8s.DeleteComponentJob(cleanup, ns, name) + stopErr := o.k8s.WaitForComponentPodsGone(cleanup, ns, "job-name="+name, time.Minute) + return "", errors.Join(failure, stopErr) + } + return "", failure } return o.k8s.JobLogs(ctx, ns, name) } @@ -176,24 +187,6 @@ func (o *Orchestrator) EnableTrustRegistry(ctx context.Context, s *model.SetupSe if _, err := MergeRegistryConfig(current, placeholder); err != nil { return err } - _, err = o.registryJob(ctx, ns, s, "preflight", s.TrustRegistryImage, "/tmp", `set -eu -binary=$(command -v trust-registry) -grep -aq TR_VTA_CREDENTIAL "$binary" || { echo 'Registry image needs the vta feature'; exit 1; } -grep -aq TR_SECRETS_VAULT_K8S_ROLE "$binary" || { echo 'Registry image needs Vault Kubernetes authentication'; exit 1; } -echo 'Registry feature preflight passed'`, nil, nil) - if err != nil { - return fmt.Errorf("registry image preflight failed; use a build with vta, secrets-vault, storage-fjall and standalone: %w", err) - } - // --help never opens the live keystore. Older images fail before downtime. - pod, container, err := o.k8s.RunningPod(ctx, ns, fmt.Sprintf("app=fs-vta,session-id=%d", s.ID)) - if err != nil { - return err - } - for _, command := range [][]string{{"vta", "did-mgmt", "dids", "create", "--help"}, {"vta", "did-mgmt", "dids", "edit", "--help"}} { - if _, err = o.k8s.ExecCapture(ctx, ns, pod, container, command); err != nil { - return errors.New("VTA image needs non-interactive DID creation/editing; upgrade VTA before enabling the registry") - } - } document, err := webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) if err != nil { return fmt.Errorf("cannot validate existing VTC DID history: %w", err) @@ -201,60 +194,16 @@ echo 'Registry feature preflight passed'`, nil, nil) if _, _, err = RegistryReferral(document, s.VtcDid, placeholder); err != nil { return err } - if err = o.vault.EnsureUserAccess(ctx, s.UserID, ns, k8s.VtaServiceAccount); err != nil { - return err - } - credential, err := o.vault.RegistryCredential(ctx, s.UserID, s.ID, s.VtaDid, s.PublicURL()) - if err != nil { - return errors.New("cannot persist registry credential in Vault; check the API registry Vault policy") - } - contextID := fmt.Sprintf("trust-registry-%d", s.ID) - path := s.VtcName + "-trust-registry" - if err = o.RegistryPhase(s, "provision_identity"); err != nil { + if err = o.provisionRegistry(ctx, ns, s); err != nil { return err } - if s.TrustRegistryDid == "" { - var output string - err = o.registryStopped(ctx, ns, s, "vta", func() error { - cmd := fmt.Sprintf(`set -eu -vta contexts get %s >/dev/null 2>&1 || vta contexts create --id %s --name 'Trust Registry' -vta acl get %s >/dev/null 2>&1 || vta import-did --did %s --role admin --context %s --label trust-registry -existing=$(vta did-mgmt dids list --context %s 2>&1) -if printf '%%s' "$existing" | grep -q 'DID:'; then - printf '%%s\n' "$existing" -else - vta did-mgmt dids create --context %s --server dids --path %s --mediator-service --pre-rotation 1 -fi -vta did-mgmt dids list --context %s`, shellQuote(contextID), shellQuote(contextID), shellQuote(credential.Did), shellQuote(credential.Did), shellQuote(contextID), shellQuote(contextID), shellQuote(contextID), shellQuote(path), shellQuote(contextID)) - var e error - output, e = o.registryJob(ctx, ns, s, "identity", s.VtaImage, "/work/vta", cmd, []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, nil) - return e - }) - if err != nil { - return err - } - matches := regexp.MustCompile(`DID:\s+(did:webvh:\S+)`).FindAllStringSubmatch(stripANSI(output), -1) - identities := map[string]bool{} - for _, match := range matches { - identities[match[1]] = true - } - if len(identities) != 1 { - return errors.New("registry context must contain exactly one hosted DID; inspect identity job before retrying") - } - for did := range identities { - s.TrustRegistryDid = did - } - if err = o.db.Model(s).Update("trust_registry_did", s.TrustRegistryDid).Error; err != nil { - return err - } - } if err = webvh.NewResolver(webvh.DefaultTimeout).ResolveDID(ctx, s.TrustRegistryDid); err != nil { return fmt.Errorf("registry DID did not resolve: %w", err) } if err = o.RegistryPhase(s, "deploy_registry"); err != nil { return err } - if err = o.deployTrustRegistry(ctx, ns, s, contextID); err != nil { + if err = o.deployTrustRegistry(ctx, ns, s); err != nil { return err } if err = o.RegistryPhase(s, "publish_referral"); err != nil { @@ -271,7 +220,7 @@ vta did-mgmt dids list --context %s`, shellQuote(contextID), shellQuote(contextI } if changed { err = o.registryStopped(ctx, ns, s, "vta", func() error { - _, e := o.registryJob(ctx, ns, s, "referral", s.VtaImage, "/work/vta", "vta did-mgmt dids edit --did "+shellQuote(s.VtcDid)+" --document /config/document.json --no-confirm", []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, map[string][]byte{"document.json": updated}) + _, e := o.registryJob(ctx, ns, s, "referral", s.VtaImage, "/work/vta", []string{"vta", "did-mgmt", "dids", "edit", "--did", s.VtcDid, "--document", "/config/document.json", "--no-confirm"}, []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, map[string][]byte{"document.json": updated}) return e }) if err != nil { @@ -291,11 +240,11 @@ vta did-mgmt dids list --context %s`, shellQuote(contextID), shellQuote(contextI } if current != merged { err = o.registryStopped(ctx, ns, s, "vtc", func() error { - _, e := o.registryJob(ctx, ns, s, "config", s.VtcImage, "/work/vtc", `set -eu; umask 077 + _, e := o.registryJob(ctx, ns, s, "config", s.VtcImage, "/work/vtc", []string{"sh", "-ec", `set -eu; umask 077 cp -p config.toml config.toml.vtafarm-registry-backup cp -p config.toml config.toml.vtafarm-registry-next cat /config/vtc.toml > config.toml.vtafarm-registry-next -mv config.toml.vtafarm-registry-next config.toml`, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, map[string][]byte{"vtc.toml": []byte(merged)}) +mv config.toml.vtafarm-registry-next config.toml`}, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, map[string][]byte{"vtc.toml": []byte(merged)}) return e }) if err != nil { @@ -320,12 +269,12 @@ func (o *Orchestrator) restoreRegistryVtc(ns string, s *model.SetupSession) erro ctx, cancel := context.WithTimeout(context.Background(), 4*time.Minute) defer cancel() return o.registryStopped(ctx, ns, s, "vtc", func() error { - _, err := o.registryJob(ctx, ns, s, "restore", s.VtcImage, "/work/vtc", `set -eu; test -f config.toml.vtafarm-registry-backup; cp -p config.toml.vtafarm-registry-backup config.toml`, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, nil) + _, err := o.registryJob(ctx, ns, s, "restore", s.VtcImage, "/work/vtc", []string{"sh", "-ec", `set -eu; test -f config.toml.vtafarm-registry-backup; cp -p config.toml.vtafarm-registry-backup config.toml`}, []k8s.PVCMount{{Name: "vtc-data", ClaimName: k8s.FSVtcName(s.ID), MountPath: "/work/vtc"}}, nil) return err }) } -func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *model.SetupSession, contextID string) error { +func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *model.SetupSession) error { name := k8s.FSTrustRegistryName(s.ID) if err := o.k8s.CreateComponentPVC(ctx, ns, name, RegistryPVCSize); err != nil { return err @@ -335,17 +284,19 @@ func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *mo return err } values := map[string]string{ + // REST must stay on for /health. Without TR_PUBLIC_URL it adds no + // public DID service, matching the setup recipe's DIDComm-only DID. + "ENABLE_REST": "true", "ENABLE_DIDCOMM": "true", "ENABLE_TSP": "false", "MEDIATOR_DID": s.MediatorDid, "ADMIN_DIDS": s.VtcDid, "ACL_MODE": "ExplicitDeny", - "TR_VTA_CREDENTIAL": "file:///run/trust-registry/credential.json", - "TR_VTA_CONTEXT_ID": contextID, + "TR_VTA_CREDENTIAL": "file://" + registryCredentialPath, + "TR_VTA_CONTEXT_ID": registryContextID(s.ID), "TR_STORAGE_BACKEND": "fjall", "TR_FJALL_PATH": "/work/trust-registry/data", "TR_SECRETS_VAULT_ADDR": o.vaultAddr, - "VAULT_ADDR": o.vaultAddr, "TR_SECRETS_VAULT_KV_MOUNT": o.vault.KVMount(), "TR_SECRETS_VAULT_SECRET_PATH": vault.TrustRegistryPrefix(s.UserID, s.ID) + "/identity-cache", "TR_SECRETS_VAULT_SECRET_KEY": "bundle", @@ -353,8 +304,6 @@ func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *mo "TR_SECRETS_VAULT_K8S_MOUNT": o.vault.K8sAuthMount(), "TR_SECRETS_VAULT_K8S_ROLE": vault.UserName(s.UserID), "TR_SECRETS_VAULT_SKIP_VERIFY": "true", - "VAULT_SKIP_VERIFY": "true", - "TR_CREDENTIAL_VAULT_PATH": vault.TrustRegistryPrefix(s.UserID, s.ID) + "/credential", "LISTEN_ADDRESS": "0.0.0.0:3232", "RUST_LOG": "info", } @@ -366,10 +315,6 @@ func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *mo if err != nil { return err } - mount := corev1.VolumeMount{Name: "registry-credential", MountPath: "/run/trust-registry", ReadOnly: true} - initMount := mount - initMount.ReadOnly = false - root := int64(0) if err = o.k8s.CreateComponentDeployment(ctx, ns, k8s.ComponentDeploymentSpec{ Name: name, Image: s.TrustRegistryImage, Command: []string{"trust-registry"}, WorkingDir: "/work/trust-registry", @@ -377,21 +322,7 @@ func (o *Orchestrator) deployTrustRegistry(ctx context.Context, ns string, s *mo Port: 3232, HealthCheckPath: "/health", Labels: labels, Resources: profiles.Requirements(k8s.ComponentTrustRegistry), PVCMounts: []k8s.PVCMount{{Name: "registry-data", ClaimName: name, MountPath: "/work/trust-registry"}}, - ExtraVolumes: []corev1.Volume{{ - Name: mount.Name, - VolumeSource: corev1.VolumeSource{EmptyDir: &corev1.EmptyDirVolumeSource{ - Medium: corev1.StorageMediumMemory, - SizeLimit: resource.NewQuantity(64<<10, resource.BinarySI), - }}, - }}, - ExtraMounts: []corev1.VolumeMount{mount}, Recreate: true, - InitContainers: []corev1.Container{{ - Name: "vault-credential", Image: registryVaultInitImage, - Command: []string{"sh", "-ec", registryCredentialInit}, Env: env, - VolumeMounts: []corev1.VolumeMount{initMount}, - Resources: k8s.ComponentResources("10m", "16Mi", "128Mi"), - SecurityContext: &corev1.SecurityContext{RunAsUser: &root}, - }}, + Recreate: true, }); err != nil { return err } @@ -429,7 +360,7 @@ func (o *Orchestrator) TeardownTrustRegistryVault(ctx context.Context, userID, s } func (o *Orchestrator) finishRegistrySetup(ctx context.Context, s *model.SetupSession) bool { - if !s.TrustRegistryEnabled || s.TrustRegistryState == "active" { + if !s.TrustRegistryEnabled || s.TrustRegistryState == "active" || s.TrustRegistryState == "failed" { return true } o.fsSetStatus(s.ID, "step_trust_registry") diff --git a/internal/setup/trust_registry_deployment_test.go b/internal/setup/trust_registry_deployment_test.go index 3999033..c4e98f4 100644 --- a/internal/setup/trust_registry_deployment_test.go +++ b/internal/setup/trust_registry_deployment_test.go @@ -15,7 +15,6 @@ import ( "gorm.io/driver/postgres" "gorm.io/gorm" appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes/scheme" "github.com/ic3software/vtafarm-api/internal/config" @@ -93,23 +92,24 @@ users: } o := &Orchestrator{db: db, k8s: client, vault: vaultClient, vaultAddr: server.URL} session := &model.SetupSession{ID: 9, UserID: 7, VtcDid: "did:webvh:community", MediatorDid: "did:webvh:mediator", TrustRegistryImage: "registry:test-build"} - err = o.deployTrustRegistry(context.Background(), "test-7", session, "trust-registry-9") + err = o.deployTrustRegistry(context.Background(), "test-7", session) if err == nil || !strings.Contains(err.Error(), "captured deployment") { t.Fatalf("did not reach deployment capture: %v", err) } pod := deployment.Spec.Template.Spec - if len(pod.Containers) != 1 || len(pod.InitContainers) != 1 { - t.Fatal("missing registry or Vault credential init container") + if len(pod.Containers) != 1 || len(pod.InitContainers) != 0 { + t.Fatal("registry should use its setup credential without a helper init container") } env := map[string]string{} for _, value := range pod.Containers[0].Env { env[value.Name] = value.Value } for key, expected := range map[string]string{ - "TR_VTA_CONTEXT_ID": "trust-registry-9", - "TR_VTA_CREDENTIAL": "file:///run/trust-registry/credential.json", + "TR_VTA_CONTEXT_ID": "trust-registry-9-headless", + "TR_VTA_CREDENTIAL": "file:///work/trust-registry/vta-credential.json", "ADMIN_DIDS": "did:webvh:community", "MEDIATOR_DID": "did:webvh:mediator", + "ENABLE_REST": "true", "ENABLE_DIDCOMM": "true", "ENABLE_TSP": "false", "TR_SECRETS_VAULT_K8S_MOUNT": "farm-auth", @@ -118,13 +118,13 @@ users: t.Errorf("runtime %s = %q, want %q", key, env[key], expected) } } - if pod.InitContainers[0].Image != registryVaultInitImage || deployment.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType { - t.Fatal("wrong credential init image or concurrent storage writers") + if deployment.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType { + t.Fatal("concurrent storage writers") } - for _, volume := range pod.Volumes { - if volume.Name == "registry-credential" && volume.EmptyDir != nil && volume.EmptyDir.Medium == corev1.StorageMediumMemory { + for _, mount := range pod.Containers[0].VolumeMounts { + if mount.Name == "registry-data" && mount.MountPath == registryWorkDir { return } } - t.Fatal("credential volume is not memory-backed") + t.Fatal("registry cannot read the credential written by its setup Job") } diff --git a/internal/setup/trust_registry_failure_test.go b/internal/setup/trust_registry_failure_test.go index f580136..cf9cf6c 100644 --- a/internal/setup/trust_registry_failure_test.go +++ b/internal/setup/trust_registry_failure_test.go @@ -1,23 +1,149 @@ package setup import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" "strings" + "sync/atomic" "testing" + "unicode/utf8" + + "github.com/ic3software/vtafarm-api/internal/config" + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" ) -func TestRegistryJobFailureRetainsUsefulDiagnosticsWithoutRawLogs(t *testing.T) { - const diagnostic = "could not sign the DID-management task with this VTA's operational key" +func TestRegistryJobFailurePreservesArbitraryOutput(t *testing.T) { + const reason = "Job has reached the specified backoff limit" for _, output := range []string{ - "Error: " + diagnostic + "\nprivateKeyMultibase=test-key-never-persist", - "privateKeyMultibase=test-key-never-persist", - "", + "Error: no transport in common with `did:webvh:test:example.com`: it advertises [tsp, didcomm] and this agent can initiate [rest]", + "Error: a previously unknown failure\n\nCaused by:\n 0: could not write database\n 1: disk full", + "sh: custom-tool: not found", + "last operation never finished", + "Error: password authentication failed for user test", + "Error: could not load private key from the credential file", } { - failure := registryJobFailure("identity", "Job has reached the specified backoff limit", output).Error() - if !strings.Contains(failure, "identity job failed") || strings.Contains(failure, "test-key-never-persist") || strings.Contains(failure, "check this setup job's logs") { - t.Fatal("failure summary leaked raw logs or required a deleted Pod") + got := registryJobFailure("identity", reason, "\x1b[31m"+output+"\x1b[0m\n").Error() + want := "identity job failed (" + reason + ")\n\n--- Job Logs ---\n" + output + if got != want { + t.Fatalf("failure = %q, want %q", got, want) } - if strings.Contains(output, diagnostic) && !strings.Contains(failure, diagnostic) { - t.Fatal("failure summary lost the known signing error") + } + if got := registryJobFailure("identity", reason, " \n").Error(); got != "identity job failed ("+reason+")" { + t.Fatalf("empty log fallback = %q", got) + } +} + +func TestRegistryJobFailureRedactsCredentials(t *testing.T) { + const diagnostic = "Error: could not complete the last operation" + for _, secretOutput := range []string{ + "privateKeyMultibase=test-key-never-persist", + `{"private_key": "test-key-never-persist"}`, + "Private key (multibase): test-key-never-persist", + "VAULT_TOKEN=test-key-never-persist", + `{"token": "test-key-never-persist"}`, + "Authorization: Bearer test-key-never-persist", + "database: postgres://user:test-key-never-persist@localhost/db", + "-----BEGIN PRIVATE KEY-----\ntest-key-never-persist\n-----END PRIVATE KEY-----", + } { + got := registryJobFailure("identity", "failed", secretOutput+"\n"+diagnostic).Error() + if strings.Contains(got, "test-key-never-persist") || !strings.Contains(got, diagnostic) { + t.Fatalf("failure did not preserve the diagnostic safely: %q", got) } } } + +func TestRegistryJobFailureRetainsBoundedLogTail(t *testing.T) { + const finalError = "Error: final operation failed" + output := "old output\n" + strings.Repeat("測", 10000) + "\n" + finalError + got := registryJobFailure("identity", "failed", output).Error() + if !strings.HasSuffix(got, finalError) || strings.Contains(got, "old output") || len(got) > 17*1024 || !utf8.ValidString(got) { + t.Fatal("failure did not retain a bounded UTF-8 log tail") + } +} + +func TestRegistryJobCapturesLatestLogsBeforeCleanup(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "") + for _, scenario := range []string{"failed", "cancelled", "logs unavailable"} { + t.Run(scenario, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var created, deleted, logsRead atomic.Bool + const output = "Error: last job failed for an unrecognized reason\nCaused by:\n storage unavailable" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/jobs"): + created.Store(true) + fmt.Fprint(w, `{"kind":"Job","apiVersion":"batch/v1","metadata":{"name":"fs-9-tr-identity"}}`) + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/jobs/"): + if !created.Load() { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","reason":"NotFound","code":404}`) + } else if scenario == "cancelled" { + cancel() + fmt.Fprint(w, `{"kind":"Job","apiVersion":"batch/v1","status":{"active":1}}`) + } else { + fmt.Fprint(w, `{"kind":"Job","apiVersion":"batch/v1","status":{"conditions":[{"type":"Failed","status":"True","message":"Job has reached the specified backoff limit"}]}}`) + } + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/pods"): + if r.URL.Query().Get("labelSelector") != "job-name=fs-9-tr-identity" { + t.Error("log lookup did not target the failing Job") + } + if deleted.Load() { + fmt.Fprint(w, `{"kind":"PodList","apiVersion":"v1","items":[]}`) + } else { + fmt.Fprint(w, `{"kind":"PodList","apiVersion":"v1","items":[{"metadata":{"name":"old-attempt","creationTimestamp":"2026-01-01T00:00:00Z"}},{"metadata":{"name":"latest-attempt","creationTimestamp":"2026-01-01T00:01:00Z"}}]}`) + } + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/latest-attempt/log"): + logsRead.Store(true) + if scenario == "logs unavailable" { + w.WriteHeader(http.StatusForbidden) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","reason":"Forbidden","code":403}`) + } else { + w.Header().Set("Content-Type", "text/plain") + fmt.Fprint(w, output) + } + case r.Method == http.MethodDelete: + if !logsRead.Load() { + t.Error("Job was deleted before its logs were read") + } + deleted.Store(true) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Success"}`) + default: + t.Errorf("unexpected request: %s %s", r.Method, r.URL.Path) + http.NotFound(w, r) + } + })) + defer server.Close() + kubeconfig := filepath.Join(t.TempDir(), "config") + contents := fmt.Sprintf(`{"apiVersion":"v1","kind":"Config","clusters":[{"name":"test","cluster":{"server":%q}}],"contexts":[{"name":"test","context":{"cluster":"test","user":"test"}}],"current-context":"test","users":[{"name":"test","user":{}}]}`, server.URL) + if err := os.WriteFile(kubeconfig, []byte(contents), 0600); err != nil { + t.Fatal(err) + } + client, err := k8s.NewClient(&config.Config{K8s: config.K8sConfig{Kubeconfig: kubeconfig}}) + if err != nil { + t.Fatal(err) + } + o := &Orchestrator{k8s: client} + _, err = o.registryJob(ctx, "test", &model.SetupSession{ID: 9}, "identity", "test:latest", "/tmp", []string{"false"}, nil, nil) + if err == nil || !logsRead.Load() { + t.Fatalf("failure did not read the latest Job logs: %v", err) + } + if scenario == "logs unavailable" { + if !strings.Contains(err.Error(), "could not read job logs") || !strings.Contains(err.Error(), "backoff limit") { + t.Fatalf("lost log retrieval error or Job failure reason: %v", err) + } + } else if !strings.Contains(err.Error(), output) { + t.Fatalf("lost the final Job output: %v", err) + } + if deleted.Load() != (scenario == "cancelled") { + t.Fatal("terminal Job must remain until the caller saves its error; cancelled Job must stop") + } + }) + } +} diff --git a/internal/setup/trust_registry_init_test.go b/internal/setup/trust_registry_init_test.go index f58129b..e061e29 100644 --- a/internal/setup/trust_registry_init_test.go +++ b/internal/setup/trust_registry_init_test.go @@ -1,14 +1,9 @@ package setup import ( - "os" - "os/exec" - "path/filepath" - "strings" - "testing" - "github.com/ic3software/vtafarm-api/internal/k8s" "github.com/ic3software/vtafarm-api/internal/vault" + "testing" ) func TestRegistryAvailableWithoutHelperImageConfiguration(t *testing.T) { @@ -21,86 +16,3 @@ func TestRegistryAvailableWithoutHelperImageConfiguration(t *testing.T) { t.Fatal("Registry must still require Vault") } } - -func TestRegistryCredentialInitKeepsSecretsPrivateAndCleansUpFailures(t *testing.T) { - for _, fail := range []bool{false, true} { - name := "success" - if fail { - name = "failed read" - } - t.Run(name, func(t *testing.T) { - dir := t.TempDir() - bin := filepath.Join(dir, "vault") - fakeCLI := `#!/bin/sh -set -eu -case "$1" in -write) - test "$2" = -field=token - test "$3" = auth/farm-auth/login - test "$4" = role=vtafarm-user-7 - test "$5" = jwt=@/var/run/secrets/kubernetes.io/serviceaccount/token - printf '%s' "$TEST_TOKEN" - ;; -kv) - test "$VAULT_TOKEN" = "$TEST_TOKEN" - test "$2" = get - test "$3" = -mount=secret - test "$4" = -field=credential - test "$5" = trust-registry/user-7/session-9/credential - printf '%s' "$TEST_CREDENTIAL" - test "$TEST_FAIL" = false - ;; -token) - test "$2" = revoke - test "$3" = -self - touch "$TEST_REVOKED" - ;; -*) exit 99 ;; -esac -` - if err := os.WriteFile(bin, []byte(fakeCLI), 0700); err != nil { - t.Fatal(err) - } - outputDir := filepath.Join(dir, "credential") - credentialFile := filepath.Join(outputDir, "credential.json") - const token = "test-token-never-log" - const credential = `{"did":"did:key:test","privateKeyMultibase":"test-key-never-log"}` - failed := "false" - if fail { - failed = "true" - } - cmd := exec.Command("sh", "-ec", strings.ReplaceAll(registryCredentialInit, "/run/trust-registry", outputDir)) - cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), - "TR_SECRETS_VAULT_K8S_MOUNT=farm-auth", "TR_SECRETS_VAULT_K8S_ROLE=vtafarm-user-7", - "TR_SECRETS_VAULT_KV_MOUNT=secret", "TR_CREDENTIAL_VAULT_PATH=trust-registry/user-7/session-9/credential", - "TEST_TOKEN="+token, "TEST_CREDENTIAL="+credential, "TEST_FAIL="+failed, "TEST_REVOKED="+filepath.Join(dir, "revoked")) - output, err := cmd.CombinedOutput() - if (err != nil) != fail { - t.Fatalf("unexpected init result: %v", err) - } - if strings.Contains(string(output), token) || strings.Contains(string(output), "test-key-never-log") { - t.Fatal("init leaked credentials to container logs") - } - if _, err := os.Stat(filepath.Join(dir, "revoked")); err != nil { - t.Fatal("init did not revoke its Vault token") - } - if _, err := os.Stat(credentialFile + ".tmp"); !os.IsNotExist(err) { - t.Fatal("init retained temporary credential material") - } - if fail { - if _, err := os.Stat(credentialFile); !os.IsNotExist(err) { - t.Fatal("failed init published a partial credential") - } - return - } - content, err := os.ReadFile(credentialFile) - if err != nil || string(content) != credential { - t.Fatal("init did not write the credential") - } - info, err := os.Stat(credentialFile) - if err != nil || info.Mode().Perm() != 0600 { - t.Fatal("credential permissions are not 0600") - } - }) - } -} diff --git a/internal/setup/trust_registry_setup.go b/internal/setup/trust_registry_setup.go new file mode 100644 index 0000000..4dea700 --- /dev/null +++ b/internal/setup/trust_registry_setup.go @@ -0,0 +1,170 @@ +package setup + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/pelletier/go-toml/v2" +) + +const registryWorkDir = "/work/trust-registry" +const registryCredentialPath = registryWorkDir + "/vta-credential.json" + +// The earlier offline flow created contexts without the one-time admin handoff. +// Use a distinct context; never repurpose or delete a legacy context's identity. +func registryContextID(id uint) string { return fmt.Sprintf("trust-registry-%d-headless", id) } + +func renderRegistrySetupTOML(s *model.SetupSession) ([]byte, error) { + return toml.Marshal(struct { + VTADID string `toml:"vta_did"` + Context string `toml:"context"` + SetupKeyFile string `toml:"setup_key_file"` + CredentialOut string `toml:"credential_out"` + EnvOut string `toml:"env_out"` + Transports []string `toml:"transports"` + MediatorDID string `toml:"mediator_did"` + WebVH struct { + ServerID string `toml:"server_id"` + Path string `toml:"path"` + } `toml:"webvh"` + }{ + VTADID: s.VtaDid, Context: registryContextID(s.ID), + SetupKeyFile: "setup-key.json", CredentialOut: registryCredentialPath, EnvOut: "vta.env", + Transports: []string{"didcomm"}, MediatorDID: s.MediatorDid, + WebVH: struct { + ServerID string `toml:"server_id"` + Path string `toml:"path"` + }{ServerID: "dids", Path: s.VtcName + "-trust-registry"}, + }) +} + +func parseRegistrySetupOutput(output string, s *model.SetupSession) (string, error) { + values := map[string]string{} + for _, line := range strings.Split(stripANSI(output), "\n") { + key, value, ok := strings.Cut(strings.TrimSpace(line), "=") + if !ok { + continue + } + switch key { + case "registry_did", "admin_did", "vta_did", "context_id", "credential_path": + if _, exists := values[key]; exists { + return "", fmt.Errorf("duplicate %s in registry setup output", key) + } + values[key] = value + } + } + if !strings.HasPrefix(values["registry_did"], "did:webvh:") || !strings.HasPrefix(values["admin_did"], "did:key:") || + values["vta_did"] != s.VtaDid || values["context_id"] != registryContextID(s.ID) || values["credential_path"] != registryCredentialPath { + return "", errors.New("registry setup output is incomplete or does not match this session") + } + return values["registry_did"], nil +} + +// Provision before VTC setup on new stacks, or during opt-in on existing stacks. +// Successful Jobs are receipts: retries reuse their key, grant and output rather +// than replacing credentials or minting another registry DID. +func (o *Orchestrator) provisionRegistry(ctx context.Context, ns string, s *model.SetupSession) error { + if !o.RegistryAvailable() { + return errors.New("Trust Registry requires Kubernetes and Vault") + } + if s.TrustRegistryDid != "" { + output, err := o.k8s.JobLogs(ctx, ns, RegistryJobName(s.ID, "setup")) + if err != nil { + return fmt.Errorf("cannot verify the headless setup receipt for this Registry; reconcile its existing credential and context before retrying: %w", err) + } + did, err := parseRegistrySetupOutput(output, s) + if err != nil || did != s.TrustRegistryDid { + return errors.New("existing Registry does not match its headless setup receipt; migrate legacy credentials and context before retrying") + } + return nil + } + if s.VtaDid == "" || s.MediatorDid == "" { + return errors.New("VTA and mediator identities must already exist") + } + if err := o.k8s.FinishRegistryJobs(ctx, ns, registryJobNames(s.ID)); err != nil { + return err + } + if err := o.RegistryPhase(s, "preflight"); err != nil { + return err + } + output, err := o.registryJob(ctx, ns, s, "preflight", s.TrustRegistryImage, "/tmp", + []string{"trust-registry", "setup", "--help"}, nil, nil) + if err != nil { + return fmt.Errorf("registry image needs headless setup built with the vta feature: %w", err) + } + if !strings.Contains(output, "--setup-key-out") || !strings.Contains(output, "--from") { + return errors.New("registry image does not support the two-phase headless setup") + } + if err := o.vault.EnsureUserAccess(ctx, s.UserID, ns, k8s.VtaServiceAccount); err != nil { + return err + } + if err := o.k8s.CreateComponentPVC(ctx, ns, k8s.FSTrustRegistryName(s.ID), RegistryPVCSize); err != nil { + return err + } + mounts := []k8s.PVCMount{{Name: "registry-data", ClaimName: k8s.FSTrustRegistryName(s.ID), MountPath: registryWorkDir}} + if err := o.RegistryPhase(s, "setup_key"); err != nil { + return err + } + output, err = o.registryJob(ctx, ns, s, "setup-key", s.TrustRegistryImage, registryWorkDir, + []string{"trust-registry", "setup", "--setup-key-out", "setup-key.json", "--context", registryContextID(s.ID)}, mounts, nil) + if err != nil { + return err + } + setupDID, err := ParseVtcSetupKeyDid(stripANSI(output)) + if err != nil { + return err + } + if err := o.RegistryPhase(s, "grant_access"); err != nil { + return err + } + if err := o.registryStopped(ctx, ns, s, "vta", func() error { + _, err := o.registryJob(ctx, ns, s, "acl-grant", s.VtaImage, "/work/vta", + []string{"vta", "contexts", "create", "--id", registryContextID(s.ID), "--name", "Trust Registry", "--admin-did", setupDID, "--admin-expires", "1h", "--admin-handoff"}, + []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, nil) + return err + }); err != nil { + return err + } + recipe, err := renderRegistrySetupTOML(s) + if err != nil { + return err + } + if err := o.RegistryPhase(s, "provision_identity"); err != nil { + return err + } + output, err = o.registryJob(ctx, ns, s, "setup", s.TrustRegistryImage, registryWorkDir, + []string{"trust-registry", "setup", "--from", "/config/tr-setup.toml"}, mounts, map[string][]byte{"tr-setup.toml": recipe}) + if err != nil { + return err + } + did, err := parseRegistrySetupOutput(output, s) + if err != nil { + return err + } + if err := o.db.Model(s).Update("trust_registry_did", did).Error; err != nil { + return err + } + s.TrustRegistryDid = did + return nil +} + +func (o *Orchestrator) prepareRegistryForVtc(ctx context.Context, ns string, s *model.SetupSession) error { + if !s.TrustRegistryEnabled || s.TrustRegistryState == "failed" || s.VtcDid != "" { + return nil + } + if err := o.provisionRegistry(ctx, ns, s); err != nil { + if ctx.Err() != nil { + return err + } + if saveErr := o.db.Model(s).Updates(map[string]any{"trust_registry_state": "failed", "trust_registry_error": err.Error()}).Error; saveErr != nil { + return saveErr + } + s.TrustRegistryState, s.TrustRegistryError = "failed", err.Error() + return o.CleanupFailedRegistryJobs(s) + } + return nil +} diff --git a/internal/setup/trust_registry_setup_test.go b/internal/setup/trust_registry_setup_test.go new file mode 100644 index 0000000..888b6be --- /dev/null +++ b/internal/setup/trust_registry_setup_test.go @@ -0,0 +1,253 @@ +package setup + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/ic3software/vtafarm-api/internal/config" + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/testutil" + "github.com/ic3software/vtafarm-api/internal/vault" + "github.com/pelletier/go-toml/v2" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/scheme" +) + +func TestRegistryHeadlessRecipeAndCompletion(t *testing.T) { + s := &model.SetupSession{ID: 9, VtaDid: "did:webvh:vta", MediatorDid: "did:webvh:mediator", VtcName: "community"} + recipe, err := renderRegistrySetupTOML(s) + if err != nil { + t.Fatal(err) + } + var values map[string]any + if err := toml.Unmarshal(recipe, &values); err != nil { + t.Fatal(err) + } + for key, want := range map[string]string{ + "vta_did": s.VtaDid, "context": "trust-registry-9-headless", "setup_key_file": "setup-key.json", + "credential_out": "/work/trust-registry/vta-credential.json", "env_out": "vta.env", "mediator_did": s.MediatorDid, + } { + if values[key] != want { + t.Errorf("recipe %s = %v, want %s", key, values[key], want) + } + } + if fmt.Sprint(values["transports"]) != "[didcomm]" || values["public_url"] != nil { + t.Fatal("recipe advertises a transport without a reachable public endpoint") + } + webvh := values["webvh"].(map[string]any) + if webvh["server_id"] != "dids" || webvh["path"] != "community-trust-registry" { + t.Fatal("recipe does not use the existing registered DID host") + } + output := "Trust Registry setup complete.\nregistry_did=did:webvh:registry\nadmin_did=did:key:admin\nvta_did=did:webvh:vta\ncontext_id=trust-registry-9-headless\ncredential_path=/work/trust-registry/vta-credential.json\nenv_path=vta.env\n" + if did, err := parseRegistrySetupOutput(output, s); err != nil || did != "did:webvh:registry" { + t.Fatalf("completion: did=%q err=%v", did, err) + } + for _, invalid := range []string{ + strings.ReplaceAll(output, "vta_did=did:webvh:vta", "vta_did=did:webvh:other"), + strings.ReplaceAll(output, "context_id=trust-registry-9-headless", "context_id=another-context"), + strings.ReplaceAll(output, "credential_path=/work/trust-registry/vta-credential.json", "credential_path=/tmp/credential.json"), + strings.ReplaceAll(output, "admin_did=did:key:admin\n", ""), + output + "registry_did=did:webvh:other\n", + } { + if _, err := parseRegistrySetupOutput(invalid, s); err == nil { + t.Fatal("accepted an incomplete or mismatched setup receipt") + } + } +} + +func TestRegistryPreparationLeavesOptedOutAndExistingCommunitiesAlone(t *testing.T) { + o := &Orchestrator{} + for _, s := range []*model.SetupSession{ + {}, + {TrustRegistryEnabled: true, TrustRegistryState: "failed"}, + {TrustRegistryEnabled: true, VtcDid: "did:webvh:existing-community"}, + } { + if err := o.prepareRegistryForVtc(context.Background(), "unused", s); err != nil { + t.Fatal(err) + } + } +} + +func TestHeadlessRegistryProvisioningAndRecovery(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "") + for _, fails := range []bool{false, true} { + t.Run(fmt.Sprintf("setup failure=%t", fails), func(t *testing.T) { + db := testutil.Postgres(t) + s := testutil.SetupSession(t, db, model.ModeFullStack) + s.VtaDid, s.MediatorDid, s.VtcName = "did:webvh:vta", "did:webvh:mediator", "community" + s.TrustRegistryEnabled, s.TrustRegistryState = true, "enabling" + s.TrustRegistryImage, s.VtaImage = "registry:test", "vta:test" + if err := db.Save(&s).Error; err != nil { + t.Fatal(err) + } + var mu sync.Mutex + jobs := map[string]*batchv1.Job{} + creations := map[string]int{} + var order []string + one := int32(1) + deployment := &appsv1.Deployment{TypeMeta: metav1.TypeMeta{APIVersion: "apps/v1", Kind: "Deployment"}, + ObjectMeta: metav1.ObjectMeta{Name: k8s.FSVtaName(s.ID)}, Spec: appsv1.DeploymentSpec{Replicas: &one}, Status: appsv1.DeploymentStatus{ReadyReplicas: 1}} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch { + case r.URL.Path == "/v1/auth/approle/login": + fmt.Fprint(w, `{"auth":{"client_token":"test-only-token","lease_duration":3600}}`) + case strings.HasPrefix(r.URL.Path, "/v1/"): + if strings.Contains(r.URL.Path, "/credential") { + t.Error("Farm must not generate or transfer a Registry credential") + } + w.WriteHeader(http.StatusNoContent) + case r.Method == http.MethodDelete: + delete(jobs, filepath.Base(r.URL.Path)) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","status":"Success"}`) + case r.Method == http.MethodPost || r.Method == http.MethodPut: + body, _ := io.ReadAll(r.Body) + object, _, err := scheme.Codecs.UniversalDeserializer().Decode(body, nil, nil) + if err != nil { + t.Error(err) + w.WriteHeader(http.StatusBadRequest) + return + } + switch obj := object.(type) { + case *batchv1.Job: + if jobs[obj.Name] != nil { + w.WriteHeader(http.StatusConflict) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","reason":"AlreadyExists","code":409}`) + return + } + jobs[obj.Name], creations[obj.Name] = obj, creations[obj.Name]+1 + order = append(order, obj.Name) + if strings.HasSuffix(obj.Name, "-acl-grant") && *deployment.Spec.Replicas != 0 { + t.Error("offline context grant ran with the VTA still running") + } + if strings.HasSuffix(obj.Name, "-setup") && *deployment.Spec.Replicas != 1 { + t.Error("headless setup ran before the VTA restarted") + } + obj.Status = batchv1.JobStatus{Succeeded: 1, Conditions: []batchv1.JobCondition{{Type: batchv1.JobComplete, Status: corev1.ConditionTrue}}} + if fails && strings.HasSuffix(obj.Name, "-setup") { + obj.Status = batchv1.JobStatus{Failed: 1, Conditions: []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue, Message: "backoff limit"}}} + } + case *appsv1.Deployment: + deployment = obj + } + _ = json.NewEncoder(w).Encode(object) + case strings.Contains(r.URL.Path, "/jobs/"): + if job := jobs[filepath.Base(r.URL.Path)]; job != nil { + _ = json.NewEncoder(w).Encode(job) + } else { + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, `{"kind":"Status","apiVersion":"v1","reason":"NotFound","code":404}`) + } + case strings.Contains(r.URL.Path, "/deployments/"): + _ = json.NewEncoder(w).Encode(deployment) + case strings.HasSuffix(r.URL.Path, "/pods"): + list := corev1.PodList{TypeMeta: metav1.TypeMeta{APIVersion: "v1", Kind: "PodList"}} + selector := r.URL.Query().Get("labelSelector") + if name, ok := strings.CutPrefix(selector, "job-name="); ok { + if jobs[name] != nil { + list.Items = []corev1.Pod{{ObjectMeta: metav1.ObjectMeta{Name: name}}} + } + } else if *deployment.Spec.Replicas == 1 { + list.Items = []corev1.Pod{{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "vta"}}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "vta", Ready: true}}}}} + } + _ = json.NewEncoder(w).Encode(list) + case strings.HasSuffix(r.URL.Path, "/log"): + w.Header().Set("Content-Type", "text/plain") + name := filepath.Base(filepath.Dir(r.URL.Path)) + switch { + case strings.HasSuffix(name, "-preflight"): + fmt.Fprint(w, "--setup-key-out --from") + case strings.HasSuffix(name, "-setup-key"): + fmt.Fprint(w, "Setup DID (ephemeral):\n did:key:setup") + case strings.HasSuffix(name, "-setup"): + if fails { + fmt.Fprint(w, "Error: headless provisioning failed at the DID host") + } else { + fmt.Fprintf(w, "registry_did=did:webvh:registry\nadmin_did=did:key:admin\nvta_did=%s\ncontext_id=%s\ncredential_path=%s\n", s.VtaDid, registryContextID(s.ID), registryCredentialPath) + } + } + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + path := filepath.Join(t.TempDir(), "config") + cfg := fmt.Sprintf(`{"apiVersion":"v1","kind":"Config","clusters":[{"name":"test","cluster":{"server":%q}}],"contexts":[{"name":"test","context":{"cluster":"test","user":"test"}}],"current-context":"test","users":[{"name":"test","user":{}}]}`, server.URL) + if err := os.WriteFile(path, []byte(cfg), 0600); err != nil { + t.Fatal(err) + } + client, err := k8s.NewClient(&config.Config{K8s: config.K8sConfig{Kubeconfig: path}}) + if err != nil { + t.Fatal(err) + } + v, err := vault.New(vault.Config{Addr: server.URL, RoleID: "unused", SecretID: "unused"}) + if err != nil { + t.Fatal(err) + } + o := &Orchestrator{db: db, k8s: client, vault: v} + if err := o.prepareRegistryForVtc(context.Background(), "test", &s); err != nil { + t.Fatal(err) + } + var stored model.SetupSession + if err := db.First(&stored, s.ID).Error; err != nil { + t.Fatal(err) + } + if fails { + if stored.TrustRegistryState != "failed" || !strings.Contains(stored.TrustRegistryError, "headless provisioning failed") || stored.Status != s.Status || !o.finishRegistrySetup(context.Background(), &s) { + t.Fatal("Registry failure was not isolated and persisted for the VTC to continue") + } + return + } + if stored.TrustRegistryDid != "did:webvh:registry" || stored.VtcDid != "" { + t.Fatal("Registry identity was not saved before VTC creation") + } + // Simulate a lost DB write after the setup Job completed. + if err := db.Model(&s).Update("trust_registry_did", "").Error; err != nil { + t.Fatal(err) + } + s.TrustRegistryDid = "" + if err := o.provisionRegistry(context.Background(), "test", &s); err != nil { + t.Fatal(err) + } + if err := o.provisionRegistry(context.Background(), "test", &s); err != nil { + t.Fatalf("persisted identity could not reuse its setup receipt: %v", err) + } + s.TrustRegistryDid = "did:webvh:legacy" + if err := o.provisionRegistry(context.Background(), "test", &s); err == nil { + t.Fatal("accepted a legacy identity with a different headless credential") + } + mu.Lock() + for _, phase := range []string{"setup-key", "acl-grant", "setup"} { + job := jobs[RegistryJobName(s.ID, phase)] + if creations[job.Name] != 1 || job.Spec.TTLSecondsAfterFinished != nil || job.Spec.Template.Spec.Containers[0].Command[0] == "sh" { + t.Errorf("%s did not preserve a direct-CLI setup receipt", phase) + } + } + grant := strings.Join(jobs[RegistryJobName(s.ID, "acl-grant")].Spec.Template.Spec.Containers[0].Command, " ") + if !strings.Contains(grant, "--admin-did did:key:setup --admin-expires 1h --admin-handoff") || len(order) < 4 || order[1] != RegistryJobName(s.ID, "setup-key") || order[2] != RegistryJobName(s.ID, "acl-grant") || order[3] != RegistryJobName(s.ID, "setup") { + t.Error("headless handshake order or temporary admin grant is incorrect") + } + delete(jobs, RegistryJobName(s.ID, "setup")) + mu.Unlock() + if err := o.provisionRegistry(context.Background(), "test", &s); err == nil || !strings.Contains(err.Error(), "reconcile") { + t.Fatal("missing headless receipt did not prevent overwriting a legacy deployment") + } + }) + } +} diff --git a/internal/vault/trust_registry.go b/internal/vault/trust_registry.go index 12df36f..8ce4b4f 100644 --- a/internal/vault/trust_registry.go +++ b/internal/vault/trust_registry.go @@ -2,81 +2,13 @@ package vault import ( "context" - "crypto/ed25519" - "crypto/rand" - "encoding/json" - "errors" "fmt" - "net/http" - - "github.com/ic3software/vtafarm-api/internal/didkey" - "github.com/ic3software/vtafarm-api/internal/didresolver" ) func TrustRegistryPrefix(userID, sessionID uint) string { return fmt.Sprintf("trust-registry/user-%d/session-%d", userID, sessionID) } -type RegistryCredential struct { - Did string `json:"did"` - PrivateKeyMultibase string `json:"privateKeyMultibase"` - VtaDid string `json:"vtaDid"` - VtaURL string `json:"vtaUrl"` -} - -// RegistryCredential creates once with Vault CAS=0 so retries keep the ACL identity. -func (c *Client) RegistryCredential(ctx context.Context, userID, sessionID uint, vtaDid, vtaURL string) (RegistryCredential, error) { - token, err := c.login(ctx) - if err != nil { - return RegistryCredential{}, err - } - path := "/v1/" + c.cfg.KVMount + "/data/" + TrustRegistryPrefix(userID, sessionID) + "/credential" - read := func() (RegistryCredential, error) { - var response struct { - Data struct { - Data map[string]string `json:"data"` - } `json:"data"` - } - if err := c.do(ctx, http.MethodGet, path, token, nil, &response); err != nil { - return RegistryCredential{}, err - } - var credential RegistryCredential - if err := json.Unmarshal([]byte(response.Data.Data["credential"]), &credential); err != nil { - return credential, errors.New("invalid registry credential in Vault") - } - if credential.Did == "" || credential.PrivateKeyMultibase == "" || credential.VtaDid != vtaDid { - return credential, errors.New("registry credential does not match this VTA") - } - return credential, nil - } - credential, err := read() - if err == nil { - return credential, nil - } - var response *responseError - if !errors.As(err, &response) || response.statusCode != http.StatusNotFound { - return credential, err - } - pub, priv, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - return credential, err - } - did, err := didkey.FromPublicKey(pub) - if err != nil { - return credential, err - } - credential = RegistryCredential{Did: did, PrivateKeyMultibase: "z" + didresolver.EncodeBase58BTC(append([]byte{0x80, 0x26}, priv.Seed()...)), VtaDid: vtaDid, VtaURL: vtaURL} - encoded, err := json.Marshal(credential) - if err != nil { - return RegistryCredential{}, err - } - if err := c.do(ctx, http.MethodPost, path, token, map[string]any{"options": map[string]int{"cas": 0}, "data": map[string]string{"credential": string(encoded)}}, nil); err != nil { - // Another worker may have won the CAS; read its credential rather than rotating it. - return read() - } - return credential, nil -} - func (c *Client) DeleteTrustRegistrySecrets(ctx context.Context, userID, sessionID uint) error { token, err := c.login(ctx) if err != nil { diff --git a/internal/vault/trust_registry_test.go b/internal/vault/trust_registry_test.go deleted file mode 100644 index 1049c09..0000000 --- a/internal/vault/trust_registry_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package vault - -import ( - "context" - "crypto/ed25519" - "encoding/json" - "net/http" - "strings" - "testing" - - "github.com/ic3software/vtafarm-api/internal/didkey" - "github.com/ic3software/vtafarm-api/internal/didresolver" -) - -func TestRegistryCredentialIsScopedStableAndCASProtected(t *testing.T) { - stored := "" - writes := 0 - client, closeServer := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/v1/secret/data/trust-registry/user-7/session-9/credential" { - t.Error("wrong secret scope") - http.Error(w, "not found", 404) - return - } - if r.Method == http.MethodGet { - if stored == "" { - http.Error(w, "missing", 404) - return - } - _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{"data": map[string]string{"credential": stored}}}) - return - } - var request struct { - Options struct { - CAS int `json:"cas"` - } `json:"options"` - Data map[string]string `json:"data"` - } - if err := json.NewDecoder(r.Body).Decode(&request); err != nil { - t.Fatal(err) - } - if request.Options.CAS != 0 { - t.Error("credential creation must use CAS=0") - } - stored = request.Data["credential"] - writes++ - w.WriteHeader(http.StatusNoContent) - }) - defer closeServer() - first, err := client.RegistryCredential(context.Background(), 7, 9, "did:webvh:vta", "https://vta.example") - if err != nil { - t.Fatal(err) - } - second, err := client.RegistryCredential(context.Background(), 7, 9, "did:webvh:vta", "https://vta.example") - if err != nil { - t.Fatal(err) - } - if first != second || writes != 1 { - t.Fatal("retry rotated the registry access credential") - } - key, err := didresolver.DecodeBase58BTC(strings.TrimPrefix(first.PrivateKeyMultibase, "z")) - if err != nil { - t.Fatal(err) - } - if len(key) != 34 || key[0] != 0x80 || key[1] != 0x26 { - t.Fatal("invalid Ed25519 secret multicodec") - } - derived, _ := didkey.FromPublicKey(ed25519.NewKeyFromSeed(key[2:]).Public().(ed25519.PublicKey)) - if derived != first.Did { - t.Fatal("credential DID does not match the private key") - } - if _, err = client.RegistryCredential(context.Background(), 7, 9, "did:webvh:other", "https://other.example"); err == nil { - t.Fatal("accepted credential for another VTA") - } -} From f260b2dd580cae52770359adff61b0660ad6a7c5 Mon Sep 17 00:00:00 2001 From: Geoff Turk Date: Fri, 23 Oct 2026 11:17:30 +0200 Subject: [PATCH 4/7] feat: publish the registry referral through the running VTA The existing-stack enablement appended the TrustRegistry referral to the VTC's DID document with the offline `vta did-mgmt dids edit`, run in a Job on the VTA's PVC. That command holds no mediator session, so the only transport it can initiate toward a DID host is REST, and a Full Stack `dids` daemon, provisioned through the VTA with a mediator, advertises TSP and DIDComm alone. The Job failed with "no transport in common ... it advertises [tsp, didcomm] and this agent can initiate [rest]" after the VTA had already committed the entry locally. The referral is now published through the running VTA, which holds the mediator session and publishes over either transport, and which re-publishes an unconfirmed local head before applying the next update, so a stack left behind by the offline attempt converges too. internal/vta is the farm's first client for a VTA: a did:key minted for one call, the two login tasks (auth/challenge/0.1, auth/authenticate/0.2) over /trust-tasks for a bearer token, then vta/webvh/dids/get/1.0 for the VTA's head of the log and vta/webvh/dids/update/1.0 with the document and expectedVersionId. Refusals come back typed (TaskError, with the reason the VTA gives, "conflict" among them). vta/webvh/servers/domains/0.1, a read-only task the VTA answers only by reaching the host, is the probe that waits out the gap after a restart in which the VTA serves HTTPS but has no mediator session yet; an update in that gap would commit and then fail to publish. publishRegistryReferral (internal/setup/trust_registry_referral.go) replaces the referral Job: it skips everything when the published document already carries the referral; otherwise it grants the key as an admin of the VTC's context for an hour (`vta acl create`) in the same stopped-VTA window pattern as the registry context grant, logs in, waits for the host to be reachable, reads the head, and sends the update. When the VTA's head carries the referral but the host does not, it sends the document unchanged, which makes the VTA reconcile the host and append a version. The verify phase still reads the host. The envelope, proof and reply checks the DID hosting client carried move to internal/trusttaskclient so the two clients share one implementation of eddsa-jcs-2022; didhosting keeps its names as aliases and its tests are unchanged. docs/optional-trust-registry-design.md: step 6 and the paragraph on the offline edit now describe this path, and name the host-side change that would let offline publishes work (a TrustTaskHTTPS entry on the daemon's own DID), which is separate from registry enablement. Verified with gofmt, go vet ./... and go test on internal/vta, internal/trusttaskclient, internal/didhosting and internal/setup, all in a golang:1.26 container. EnableTrustRegistry itself still has no end-to-end test; the new step is behind the newVtaClient seam for one. Signed-off-by: Geoff Turk --- docs/optional-trust-registry-design.md | 51 ++- internal/didhosting/client.go | 135 +------ internal/setup/trust_registry.go | 21 +- internal/setup/trust_registry_referral.go | 119 ++++++ internal/trusttaskclient/trusttaskclient.go | 187 +++++++++ .../trusttaskclient/trusttaskclient_test.go | 93 +++++ internal/vta/client.go | 367 ++++++++++++++++++ internal/vta/client_test.go | 342 ++++++++++++++++ 8 files changed, 1165 insertions(+), 150 deletions(-) create mode 100644 internal/setup/trust_registry_referral.go create mode 100644 internal/trusttaskclient/trusttaskclient.go create mode 100644 internal/trusttaskclient/trusttaskclient_test.go create mode 100644 internal/vta/client.go create mode 100644 internal/vta/client_test.go diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md index 5b27be0..7d73350 100644 --- a/docs/optional-trust-registry-design.md +++ b/docs/optional-trust-registry-design.md @@ -218,10 +218,18 @@ existing PVCs, or re-enroll the existing administrators. Reuse successful Job receipts on retry. 5. Set `ADMIN_DIDS` to the existing `session.VtcDid`, start the registry, and verify its advertised transport and write path. -6. Append a signed update to the **existing** VTC DID history through the VTA, - adding the referral while preserving the DID, keys, other services, and - history. Publish the extended log through the existing DID host. A self-hosted - VTC log, if present, must also be updated using the supported install path. +6. Append a signed update to the **existing** VTC DID history through the + **running** VTA, adding the referral while preserving the DID, keys, other + services, and history. The farm mints a did:key for this one operation, grants + it as a context-scoped admin of the VTC's context in the same stopped-VTA + window pattern as step 3 (`vta acl create --role admin --contexts + --expires 1h`), logs in to the VTA over HTTPS as that key + (`auth/challenge/0.1`, `auth/authenticate/0.2`), reads the VTA's head of the + log (`vta/webvh/dids/get/1.0`), and sends `vta/webvh/dids/update/1.0` with the + document plus `expectedVersionId`. The VTA signs the new entry and publishes + it to the DID host itself, over TSP or DIDComm. The key is never stored; the + grant expires on its own. A self-hosted VTC log, if present, must also be + updated using the supported install path. 7. Read the current VTC runtime config immediately before writing, merge only `[registry].did`, and restart the VTC using the existing component-config backup/write/start/rollback pattern. Do not render a fresh setup config over @@ -236,17 +244,34 @@ Upstream explicitly supports changing the referral after initial mint via VTA-side DID editing ([setup schema][vtc-setup]). This requires a signed DID-log update, not a change to a locally served JSON file alone. -The existing Farm referral update still uses offline `vta did-mgmt dids edit`. -That CLI has only REST available for publishing to an external DID host. A host -advertising only DIDComm/TSP therefore still blocks **existing-stack referral -updates**. Headless Registry provisioning does not fix this separate path; -it needs an authenticated online VTA edit or upstream offline transport support. -New stacks avoid that edit by passing `registry_did` to VTC setup. Do not report -existing-stack activation as verified until the referral publication succeeds. +**Why the edit goes through the running VTA and not the offline CLI.** The +offline `vta did-mgmt dids edit` holds no mediator session, so the only +transport it can initiate toward a DID host is REST. A host provisioned through +a VTA with a mediator, which is every Full Stack `dids` daemon, advertises only +`TSPTransport` and `DIDCommMessaging`, so the offline command stops with +`no transport in common … it advertises [tsp, didcomm] and this agent can +initiate [rest]`, after it has already committed the new entry locally. The +running VTA holds the mediator session and publishes over TSP or DIDComm; it +also re-publishes an unconfirmed local head before applying the next update, +which is how a stack left in that state by the earlier offline attempt heals. +`internal/vta` is the client; `publishRegistryReferral` in +`internal/setup/trust_registry_referral.go` is the step. It skips the grant and +the edit when the published document already carries the referral, and sends an +otherwise unchanged document when the VTA's head carries it but the host does +not, which makes the VTA reconcile and publish. New stacks avoid the edit by +passing `registry_did` to VTC setup. Do not report existing-stack activation as +verified until the referral publication succeeds. + +A mediator-only host also blocks every other offline publish (`dids create`, +`dids register`). The durable fix on the host side is one more online edit, of +the daemon's own DID, adding `{"id":"#trust-tasks","type":"TrustTaskHTTPS","serviceEndpoint":"https:///api"}`; +the daemon's readiness check logs that exact entry, and its control plane reloads +on publish. That is a separate change, not part of registry enablement. **Downtime:** the VTC needs a restart for the new client/syncer. Provisioning -authorization may also need a brief VTA maintenance window: current Farm ACL -operations use `scale to 0 -> offline Job -> restart`. Prefer compatible live +authorization needs two brief VTA maintenance windows on an existing stack, the +registry context grant and the referral grant, each `scale to 0 -> offline Job -> +restart`; a retry that finds the referral already published takes neither. Prefer compatible live APIs where a suitable credential already exists, but do not promise zero VTA downtime or bypass the fjall lock. DID publication should use the live hosting interface. Always restart a temporarily stopped component on error. Preserve diff --git a/internal/didhosting/client.go b/internal/didhosting/client.go index 7e41bde..dfe55f5 100644 --- a/internal/didhosting/client.go +++ b/internal/didhosting/client.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/ed25519" - "crypto/sha256" "encoding/base64" "encoding/json" "errors" @@ -14,11 +13,11 @@ import ( "strings" "time" - "github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer" "github.com/google/uuid" "github.com/ic3software/vtafarm-api/internal/didkey" "github.com/ic3software/vtafarm-api/internal/didresolver" + "github.com/ic3software/vtafarm-api/internal/trusttaskclient" "github.com/ic3software/vtafarm-api/internal/webvh" ) @@ -30,28 +29,21 @@ const ( aclGrantType = "https://trusttasks.org/spec/acl/grant/0.1" aclRevokeType = "https://trusttasks.org/spec/acl/revoke/0.1" - maxTrustTaskResponseBytes = 1 << 20 + maxTrustTaskResponseBytes = trusttaskclient.MaxResponseBytes ) -type trustTask struct { - ID string `json:"id"` - Type string `json:"type"` - ThreadID string `json:"threadId,omitempty"` - Issuer string `json:"issuer,omitempty"` - Recipient string `json:"recipient,omitempty"` - IssuedAt string `json:"issuedAt,omitempty"` - Payload json.RawMessage `json:"payload"` - Proof *proof `json:"proof,omitempty"` -} +// The envelope, the proof and the cryptosuite steps are the shared outbound +// client's; the names below keep this package's own vocabulary. +type ( + trustTask = trusttaskclient.Document + proof = trusttaskclient.Proof +) -type proof struct { - Type string `json:"type"` - Cryptosuite string `json:"cryptosuite"` - VerificationMethod string `json:"verificationMethod"` - Created string `json:"created"` - ProofPurpose string `json:"proofPurpose"` - ProofValue string `json:"proofValue,omitempty"` -} +var ( + signTrustTask = trusttaskclient.Sign + proofMessage = trusttaskclient.ProofMessage + decodeProofValue = trusttaskclient.DecodeProofValue +) // Client talks to one DID Hosting control plane exclusively through signed // Trust Tasks. serverDid is supplied by provisioning and is both the request @@ -222,106 +214,9 @@ func (c *Client) call(ctx context.Context, typeURI string, payload any) (json.Ra return reply.Payload, nil } -func signTrustTask(doc *trustTask, kid string, privateKey ed25519.PrivateKey, created string) error { - doc.Proof = nil - proofConfig := proof{ - Type: "DataIntegrityProof", - Cryptosuite: "eddsa-jcs-2022", - VerificationMethod: kid, - Created: created, - ProofPurpose: "authentication", - } - message, err := proofMessage(doc, proofConfig) - if err != nil { - return err - } - proofConfig.ProofValue = "z" + didresolver.EncodeBase58BTC(ed25519.Sign(privateKey, message)) - doc.Proof = &proofConfig - return nil -} - func (c *Client) verifyReply(ctx context.Context, reply, request *trustTask) error { - if reply.ThreadID != request.ID { - return errors.New("DID hosting Trust Task response is not threaded to the request") - } - if reply.Issuer != c.serverDid { - return errors.New("DID hosting Trust Task response issuer does not match the service DID") - } - if reply.Recipient != c.clientDid { - return errors.New("DID hosting Trust Task response recipient does not match DID_HOSTING_DID") - } - issuedAt, err := time.Parse(time.RFC3339, reply.IssuedAt) - if err != nil || issuedAt.Before(time.Now().Add(-5*time.Minute)) || issuedAt.After(time.Now().Add(didresolver.DefaultClockSkew)) { - return errors.New("DID hosting Trust Task response is outside the freshness window") - } - if reply.Proof == nil { - return errors.New("DID hosting Trust Task response has no proof") - } - p := *reply.Proof - if p.Type != "DataIntegrityProof" || p.Cryptosuite != "eddsa-jcs-2022" || p.ProofPurpose != "authentication" { - return errors.New("DID hosting Trust Task response has an unsupported proof") - } - proofDid, _, ok := strings.Cut(p.VerificationMethod, "#") - if !ok || proofDid != c.serverDid { - return errors.New("DID hosting Trust Task response proof does not belong to the service DID") - } - created, err := time.Parse(time.RFC3339, p.Created) - if err != nil || created.After(time.Now().Add(didresolver.DefaultClockSkew)) { - return errors.New("DID hosting Trust Task response proof has an invalid creation time") - } - signature, err := decodeProofValue(p.ProofValue) - if err != nil { - return err - } - publicKey, err := c.resolver.ResolveAuthenticationKey(ctx, c.serverDid, p.VerificationMethod) - if err != nil { - return fmt.Errorf("resolve DID hosting response key: %w", err) - } - reply.Proof = nil - message, err := proofMessage(reply, p) - reply.Proof = &p - if err != nil { - return fmt.Errorf("canonicalize Trust Task response: %w", err) - } - if len(signature) != ed25519.SignatureSize || !ed25519.Verify(publicKey, message, signature) { - return errors.New("DID hosting Trust Task response proof signature is invalid") + if err := trusttaskclient.VerifyReply(ctx, c.resolver, reply, request, trusttaskclient.Reply{Service: c.serverDid, Client: c.clientDid}); err != nil { + return fmt.Errorf("DID hosting %w", err) } return nil } - -func proofMessage(doc *trustTask, p proof) ([]byte, error) { - p.ProofValue = "" - proofCanonical, err := canonicalJSON(p) - if err != nil { - return nil, fmt.Errorf("canonicalize proof options: %w", err) - } - documentCanonical, err := canonicalJSON(doc) - if err != nil { - return nil, fmt.Errorf("canonicalize document: %w", err) - } - proofHash := sha256.Sum256(proofCanonical) - documentHash := sha256.Sum256(documentCanonical) - message := make([]byte, 0, len(proofHash)+len(documentHash)) - message = append(message, proofHash[:]...) - message = append(message, documentHash[:]...) - return message, nil -} - -func canonicalJSON(value any) ([]byte, error) { - encoded, err := json.Marshal(value) - if err != nil { - return nil, err - } - return jsoncanonicalizer.Transform(encoded) -} - -func decodeProofValue(value string) ([]byte, error) { - if !strings.HasPrefix(value, "z") { - return nil, errors.New("DID hosting Trust Task proofValue is not base58btc multibase") - } - decoded, err := didresolver.DecodeBase58BTC(value[1:]) - if err != nil || didresolver.EncodeBase58BTC(decoded) != value[1:] { - return nil, errors.New("DID hosting Trust Task proofValue is not canonical base58btc") - } - return decoded, nil -} diff --git a/internal/setup/trust_registry.go b/internal/setup/trust_registry.go index 7bcb9c7..c06b371 100644 --- a/internal/setup/trust_registry.go +++ b/internal/setup/trust_registry.go @@ -209,24 +209,11 @@ func (o *Orchestrator) EnableTrustRegistry(ctx context.Context, s *model.SetupSe if err = o.RegistryPhase(s, "publish_referral"); err != nil { return err } - // Resolve again immediately before editing; never apply the preflight snapshot. - document, err = webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) - if err != nil { - return err - } - updated, changed, err := RegistryReferral(document, s.VtcDid, s.TrustRegistryDid) - if err != nil { + // Through the running VTA, which reads its own head immediately before + // editing; the preflight snapshot above is never applied. + if err = o.publishRegistryReferral(ctx, ns, s); err != nil { return err } - if changed { - err = o.registryStopped(ctx, ns, s, "vta", func() error { - _, e := o.registryJob(ctx, ns, s, "referral", s.VtaImage, "/work/vta", []string{"vta", "did-mgmt", "dids", "edit", "--did", s.VtcDid, "--document", "/config/document.json", "--no-confirm"}, []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, map[string][]byte{"document.json": updated}) - return e - }) - if err != nil { - return err - } - } if err = o.RegistryPhase(s, "configure_vtc"); err != nil { return err } @@ -258,7 +245,7 @@ mv config.toml.vtafarm-registry-next config.toml`}, []k8s.PVCMount{{Name: "vtc-d if err != nil { return err } - if _, changed, err = RegistryReferral(document, s.VtcDid, s.TrustRegistryDid); err != nil || changed { + if _, changed, err := RegistryReferral(document, s.VtcDid, s.TrustRegistryDid); err != nil || changed { return errors.New("published VTC referral has not converged; retry after DID hosting is updated") } s.TrustRegistryState = "active" diff --git a/internal/setup/trust_registry_referral.go b/internal/setup/trust_registry_referral.go new file mode 100644 index 0000000..fcb4da3 --- /dev/null +++ b/internal/setup/trust_registry_referral.go @@ -0,0 +1,119 @@ +package setup + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/ic3software/vtafarm-api/internal/k8s" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/vta" + "github.com/ic3software/vtafarm-api/internal/webvh" +) + +const ( + // referralGrantLabel names the ACL entry the referral edit runs under. + referralGrantLabel = "trust-registry-referral" + // referralGrantTTL bounds that entry; the VTA's sweeper removes it. The + // key itself lives only in this process, for the length of one call. + referralGrantTTL = "1h" + // referralHostWait is how long to give the restarted VTA to bring its + // mediator session up before the edit, whose publish needs it. + referralHostWait = 3 * time.Minute + // referralDidsServer is the id the session's DID hosting daemon is + // registered under at the VTA (`vta did-mgmt servers add --id dids`). + referralDidsServer = "dids" +) + +// newVtaClient is the seam a test replaces to stand in a VTA. +var newVtaClient = vta.NewEphemeral + +// publishRegistryReferral adds the TrustRegistry referral to the VTC's +// published DID document, through the running VTA. +// +// Why the running VTA: the offline `vta did-mgmt dids edit` this step used to +// run holds no mediator session, so it can publish only to a DID host that +// advertises an HTTPS endpoint. A host provisioned through a VTA with a +// mediator, which is every Full Stack `dids` daemon, advertises TSP and +// DIDComm alone, and the offline edit stopped with `no transport in common` +// after committing the entry locally. The running VTA holds the mediator +// session, publishes over either, and re-publishes an unconfirmed local head +// before applying the next update, which is also how a stack left behind by +// that offline attempt converges. +// +// The farm has no standing credential at a user's VTA, and does not want one. +// It mints a did:key for this call, grants it as an admin of the VTC's context +// for an hour in the same stopped-VTA window pattern the registry context +// grant uses, logs in over HTTPS, and sends the update signed by that key. On +// return the key is garbage; the entry expires on its own. +// +// Idempotent by observation, not by record: a published document that +// already carries the referral means nothing to do, with no grant and no +// restart; a VTA head that carries it while the host does not gets an update +// with an unchanged document, which the VTA answers by reconciling the host +// and then appending a version. Either way the verify phase that follows reads +// the host, not this function's belief. +func (o *Orchestrator) publishRegistryReferral(ctx context.Context, ns string, s *model.SetupSession) error { + published, err := webvh.NewResolver(webvh.DefaultTimeout).ResolveDocument(ctx, s.VtcDid) + if err != nil { + return fmt.Errorf("resolve the VTC DID before publishing the referral: %w", err) + } + if _, changed, err := RegistryReferral(published, s.VtcDid, s.TrustRegistryDid); err != nil { + return err + } else if !changed { + log.Printf("[orchestrator] fs session %d: %s already refers to the registry; nothing to publish", s.ID, s.VtcDid) + return nil + } + + client, err := newVtaClient(s.PublicURL(), s.VtaDid) + if err != nil { + return err + } + grant := []string{"vta", "acl", "create", "--did", client.DID(), "--role", "admin", "--contexts", s.VtcName, "--expires", referralGrantTTL, "--label", referralGrantLabel} + err = o.registryStopped(ctx, ns, s, "vta", func() error { + _, e := o.registryJob(ctx, ns, s, "referral", s.VtaImage, "/work/vta", grant, []k8s.PVCMount{{Name: "vta-data", ClaimName: k8s.FSVtaName(s.ID), MountPath: "/work/vta"}}, nil) + return e + }) + if err != nil { + return err + } + + if err = client.Login(ctx); err != nil { + return fmt.Errorf("log in to the VTA as the referral grant %s: %w", client.DID(), err) + } + if err = client.WaitUntilHostReachable(ctx, referralDidsServer, referralHostWait); err != nil { + return fmt.Errorf("the restarted VTA has no route to the DID host, so it could not publish: %w", err) + } + didLog, err := client.DIDLog(ctx, s.VtcDid) + if err != nil { + return err + } + head, state, err := vta.LatestLogEntry(didLog) + if err != nil { + return fmt.Errorf("read the VTA's head of %s: %w", s.VtcDid, err) + } + updated, changed, err := RegistryReferral(state, s.VtcDid, s.TrustRegistryDid) + if err != nil { + return err + } + if !changed { + // The VTA's head has the referral and the host does not: an earlier + // publish failed after the local commit. An update that changes + // nothing in the document still makes the VTA re-publish that head + // first, then append a version the host takes. + log.Printf("[orchestrator] fs session %d: the VTA's head of %s (%s) carries the referral but the host does not; publishing through a new version", s.ID, s.VtcDid, head) + updated = state + } + newVersion, err := client.UpdateDIDDocument(ctx, s.VtcDid, updated, head) + if err != nil { + var refusal *vta.TaskError + if errors.As(err, &refusal) && refusal.Conflict() { + return fmt.Errorf("the VTC DID advanced past %s while the referral was being published; retry registry enablement: %w", head, err) + } + return err + } + log.Printf("[orchestrator] fs session %d: TrustRegistry referral published; %s is now version %s", s.ID, s.VtcDid, newVersion) + return nil +} diff --git a/internal/trusttaskclient/trusttaskclient.go b/internal/trusttaskclient/trusttaskclient.go new file mode 100644 index 0000000..56c2649 --- /dev/null +++ b/internal/trusttaskclient/trusttaskclient.go @@ -0,0 +1,187 @@ +// Package trusttaskclient holds what an outbound Trust Task client needs and +// nothing a server needs: the document envelope, the eddsa-jcs-2022 proof a +// did:key makes over it, and the checks a signed reply has to pass before its +// payload is believed. The DID hosting client and the VTA client are both +// built on it, so the two never disagree about what a proof covers. +// +// The proof follows the Data Integrity eddsa-jcs-2022 cryptosuite: the signed +// message is sha256(JCS(proof options without proofValue)) followed by +// sha256(JCS(document without proof)), and proofValue is the base58btc +// multibase ("z…") of the Ed25519 signature. Both the DID hosting daemon and +// the VTA verify the JSON exactly as received, so a document is signed after +// its final marshalling shape is fixed and never re-encoded afterwards. +package trusttaskclient + +import ( + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/cyberphone/json-canonicalization/go/src/webpki.org/jsoncanonicalizer" + + "github.com/ic3software/vtafarm-api/internal/didresolver" +) + +// MaxResponseBytes caps what a reply may carry; anything over it is refused +// unread. Both services cap their own bodies at the same size. +const MaxResponseBytes = 1 << 20 + +// Document is the Trust Task envelope as sent and received. Fields the spec +// leaves optional are omitted when empty so the signed bytes carry no nulls. +type Document struct { + ID string `json:"id"` + Type string `json:"type"` + ThreadID string `json:"threadId,omitempty"` + Issuer string `json:"issuer,omitempty"` + Recipient string `json:"recipient,omitempty"` + IssuedAt string `json:"issuedAt,omitempty"` + Payload json.RawMessage `json:"payload"` + Proof *Proof `json:"proof,omitempty"` +} + +// Proof is a Data Integrity proof. Only the members the cryptosuite signs are +// carried: a verifier refuses a proof with members outside this set. +type Proof struct { + Type string `json:"type"` + Cryptosuite string `json:"cryptosuite"` + VerificationMethod string `json:"verificationMethod"` + Created string `json:"created"` + ProofPurpose string `json:"proofPurpose"` + ProofValue string `json:"proofValue,omitempty"` +} + +// Sign attaches an eddsa-jcs-2022 proof with purpose `authentication`, made +// by privateKey, naming kid as the verification method. Any proof already on +// the document is discarded first. +func Sign(doc *Document, kid string, privateKey ed25519.PrivateKey, created string) error { + doc.Proof = nil + options := Proof{ + Type: "DataIntegrityProof", + Cryptosuite: "eddsa-jcs-2022", + VerificationMethod: kid, + Created: created, + ProofPurpose: "authentication", + } + message, err := ProofMessage(doc, options) + if err != nil { + return err + } + options.ProofValue = "z" + didresolver.EncodeBase58BTC(ed25519.Sign(privateKey, message)) + doc.Proof = &options + return nil +} + +// ProofMessage is the 64 bytes the cryptosuite signs for doc under the proof +// options p (its ProofValue is ignored). doc.Proof must be nil. +func ProofMessage(doc *Document, p Proof) ([]byte, error) { + p.ProofValue = "" + proofCanonical, err := CanonicalJSON(p) + if err != nil { + return nil, fmt.Errorf("canonicalize proof options: %w", err) + } + documentCanonical, err := CanonicalJSON(doc) + if err != nil { + return nil, fmt.Errorf("canonicalize document: %w", err) + } + proofHash := sha256.Sum256(proofCanonical) + documentHash := sha256.Sum256(documentCanonical) + message := make([]byte, 0, len(proofHash)+len(documentHash)) + message = append(message, proofHash[:]...) + message = append(message, documentHash[:]...) + return message, nil +} + +// CanonicalJSON is RFC 8785 JCS of value's JSON encoding. +func CanonicalJSON(value any) ([]byte, error) { + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + return jsoncanonicalizer.Transform(encoded) +} + +// DecodeProofValue reads a canonical base58btc multibase proofValue. +func DecodeProofValue(value string) ([]byte, error) { + if !strings.HasPrefix(value, "z") { + return nil, errors.New("Trust Task proofValue is not base58btc multibase") + } + decoded, err := didresolver.DecodeBase58BTC(value[1:]) + if err != nil || didresolver.EncodeBase58BTC(decoded) != value[1:] { + return nil, errors.New("Trust Task proofValue is not canonical base58btc") + } + return decoded, nil +} + +// Reply is the identity a signed reply must prove and whom it must address. +type Reply struct { + // Service is the DID the reply must be issued and signed by. + Service string + // Client is the DID the reply must be addressed to. + Client string + // Now is the clock the freshness checks use; nil means time.Now. + Now func() time.Time +} + +// VerifyReply checks that reply answers request and was signed by the +// service: threaded to the request id, issued by the service to the client, +// issued within the last five minutes (plus clock skew), carrying an +// eddsa-jcs-2022 `authentication` proof under a key the service's DID +// document lists for authentication, which verifies over the reply as +// received. resolver fetches that key. +func VerifyReply(ctx context.Context, resolver didresolver.AuthenticationKeyResolver, reply, request *Document, expect Reply) error { + now := time.Now + if expect.Now != nil { + now = expect.Now + } + if reply.ThreadID != request.ID { + return errors.New("Trust Task response is not threaded to the request") + } + if reply.Issuer != expect.Service { + return errors.New("Trust Task response issuer does not match the service DID") + } + if reply.Recipient != expect.Client { + return errors.New("Trust Task response recipient does not match the client DID") + } + issuedAt, err := time.Parse(time.RFC3339, reply.IssuedAt) + if err != nil || issuedAt.Before(now().Add(-5*time.Minute)) || issuedAt.After(now().Add(didresolver.DefaultClockSkew)) { + return errors.New("Trust Task response is outside the freshness window") + } + if reply.Proof == nil { + return errors.New("Trust Task response has no proof") + } + p := *reply.Proof + if p.Type != "DataIntegrityProof" || p.Cryptosuite != "eddsa-jcs-2022" || p.ProofPurpose != "authentication" { + return errors.New("Trust Task response has an unsupported proof") + } + proofDid, _, ok := strings.Cut(p.VerificationMethod, "#") + if !ok || proofDid != expect.Service { + return errors.New("Trust Task response proof does not belong to the service DID") + } + created, err := time.Parse(time.RFC3339, p.Created) + if err != nil || created.After(now().Add(didresolver.DefaultClockSkew)) { + return errors.New("Trust Task response proof has an invalid creation time") + } + signature, err := DecodeProofValue(p.ProofValue) + if err != nil { + return err + } + publicKey, err := resolver.ResolveAuthenticationKey(ctx, expect.Service, p.VerificationMethod) + if err != nil { + return fmt.Errorf("resolve Trust Task response key: %w", err) + } + reply.Proof = nil + message, err := ProofMessage(reply, p) + reply.Proof = &p + if err != nil { + return fmt.Errorf("canonicalize Trust Task response: %w", err) + } + if len(signature) != ed25519.SignatureSize || !ed25519.Verify(publicKey, message, signature) { + return errors.New("Trust Task response proof signature is invalid") + } + return nil +} diff --git a/internal/trusttaskclient/trusttaskclient_test.go b/internal/trusttaskclient/trusttaskclient_test.go new file mode 100644 index 0000000..1663959 --- /dev/null +++ b/internal/trusttaskclient/trusttaskclient_test.go @@ -0,0 +1,93 @@ +package trusttaskclient + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "strings" + "testing" + "time" + + "github.com/ic3software/vtafarm-api/internal/didkey" + "github.com/ic3software/vtafarm-api/internal/didresolver" +) + +func keyPair(t *testing.T, fill byte) (did, kid string, private ed25519.PrivateKey) { + t.Helper() + seed := make([]byte, ed25519.SeedSize) + for i := range seed { + seed[i] = fill + } + private = ed25519.NewKeyFromSeed(seed) + did, err := didkey.FromPublicKey(private.Public().(ed25519.PublicKey)) + if err != nil { + t.Fatal(err) + } + return did, did + "#" + strings.TrimPrefix(did, "did:key:"), private +} + +func TestSignedReplyRoundTrip(t *testing.T) { + clientDid, _, _ := keyPair(t, 1) + serviceDid, serviceKid, servicePrivate := keyPair(t, 2) + now := time.Date(2026, 10, 23, 12, 0, 0, 0, time.UTC) + created := now.Format(time.RFC3339) + + request := Document{ID: "urn:uuid:req", Type: "https://example.org/task/0.1", Issuer: clientDid, Recipient: serviceDid, IssuedAt: created, Payload: json.RawMessage(`{"a":1}`)} + reply := Document{ID: "urn:uuid:rep", Type: request.Type + "#response", ThreadID: request.ID, Issuer: serviceDid, Recipient: clientDid, IssuedAt: created, Payload: json.RawMessage(`{"ok":true}`)} + if err := Sign(&reply, serviceKid, servicePrivate, created); err != nil { + t.Fatal(err) + } + if reply.Proof.ProofPurpose != "authentication" || reply.Proof.Cryptosuite != "eddsa-jcs-2022" || !strings.HasPrefix(reply.Proof.ProofValue, "z") { + t.Fatalf("proof %+v", reply.Proof) + } + + // The proof survives a trip through JSON, which is how it is received. + encoded, _ := json.Marshal(reply) + var received Document + if err := json.Unmarshal(encoded, &received); err != nil { + t.Fatal(err) + } + expect := Reply{Service: serviceDid, Client: clientDid, Now: func() time.Time { return now }} + if err := VerifyReply(context.Background(), didresolver.DIDKeyResolver{}, &received, &request, expect); err != nil { + t.Fatal(err) + } + if received.Proof == nil { + t.Fatal("verification stripped the proof") + } + + // One byte of payload changed after signing fails, and so does a reply + // threaded to another request, addressed elsewhere, or stale. + tampered := received + tampered.Payload = json.RawMessage(`{"ok":false}`) + if err := VerifyReply(context.Background(), didresolver.DIDKeyResolver{}, &tampered, &request, expect); err == nil || !strings.Contains(err.Error(), "signature is invalid") { + t.Fatalf("tampered payload: %v", err) + } + other := request + other.ID = "urn:uuid:other" + if err := VerifyReply(context.Background(), didresolver.DIDKeyResolver{}, &received, &other, expect); err == nil || !strings.Contains(err.Error(), "threaded") { + t.Fatalf("wrong thread: %v", err) + } + if err := VerifyReply(context.Background(), didresolver.DIDKeyResolver{}, &received, &request, Reply{Service: serviceDid, Client: "did:key:z6MkSomeoneElse", Now: expect.Now}); err == nil || !strings.Contains(err.Error(), "recipient") { + t.Fatalf("wrong recipient: %v", err) + } + late := Reply{Service: serviceDid, Client: clientDid, Now: func() time.Time { return now.Add(6 * time.Minute) }} + if err := VerifyReply(context.Background(), didresolver.DIDKeyResolver{}, &received, &request, late); err == nil || !strings.Contains(err.Error(), "freshness") { + t.Fatalf("stale reply: %v", err) + } +} + +func TestDecodeProofValueWantsCanonicalMultibase(t *testing.T) { + if _, err := DecodeProofValue("abc"); err == nil { + t.Fatal("a value without the z prefix decoded") + } + if _, err := DecodeProofValue("z0OIl"); err == nil { + t.Fatal("non-alphabet characters decoded") + } + signature := make([]byte, ed25519.SignatureSize) + signature[0] = 7 + encoded := "z" + didresolver.EncodeBase58BTC(signature) + decoded, err := DecodeProofValue(encoded) + if err != nil || len(decoded) != ed25519.SignatureSize || decoded[0] != 7 { + t.Fatalf("round trip %v %d", err, len(decoded)) + } +} diff --git a/internal/vta/client.go b/internal/vta/client.go new file mode 100644 index 0000000..819e6e9 --- /dev/null +++ b/internal/vta/client.go @@ -0,0 +1,367 @@ +// Package vta is the farm's client for a running VTA, over HTTPS, as a +// short-lived did:key the farm minted for one operation. +// +// A VTA has one authenticated surface: Trust Task documents POSTed to +// /trust-tasks. Logging in is two of them, `auth/challenge/0.1` (unsigned) +// and `auth/authenticate/0.2` (signed by the did:key), which yields a bearer +// token. Every later document carries that token and is signed by the same +// key, because the VTA checks that the proof's signer, the document's issuer +// and the session's subject are one DID. +// +// The farm uses this where an operation has to happen inside the running VTA +// rather than against its stopped store: a did:webvh update is one. The +// offline `vta did-mgmt dids edit` holds no mediator session, so it can +// publish only to a DID host that advertises an HTTPS endpoint; a host that +// advertises TSP or DIDComm alone, which is what a host provisioned through a +// VTA with a mediator gets, is reached by the running VTA alone. +package vta + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/google/uuid" + + "github.com/ic3software/vtafarm-api/internal/didkey" + "github.com/ic3software/vtafarm-api/internal/didresolver" + "github.com/ic3software/vtafarm-api/internal/trusttaskclient" + "github.com/ic3software/vtafarm-api/internal/webvh" +) + +const ( + trustTasksPath = "/trust-tasks" + + challengeType = "https://trusttasks.org/spec/auth/challenge/0.1" + authenticateType = "https://trusttasks.org/spec/auth/authenticate/0.2" + didGetType = "https://trusttasks.org/spec/vta/webvh/dids/get/1.0" + didUpdateType = "https://trusttasks.org/spec/vta/webvh/dids/update/1.0" + domainsType = "https://trusttasks.org/spec/vta/webvh/servers/domains/0.1" + errorType = "https://trusttasks.org/spec/trust-task-error/0.5" + + // ReasonConflict is `details.reason` when `expectedVersionId` no longer + // names the DID's local head. + ReasonConflict = "conflict" + // ReasonNotFound is `details.reason` for a DID the VTA does not hold, or + // one the caller may not update; the VTA does not say which. + ReasonNotFound = "not_found" +) + +// Client is one did:key's session with one VTA. +type Client struct { + baseURL string + vtaDid string + did string + kid string + key ed25519.PrivateKey + hc *http.Client + resolver didresolver.AuthenticationKeyResolver + now func() time.Time + token string + // probeInterval paces WaitUntilHostReachable. + probeInterval time.Duration +} + +// NewEphemeral mints a fresh Ed25519 did:key, held in memory only, for a +// VTA reached at baseURL whose DID is vtaDid. The key has no standing until +// the VTA's ACL names it; the caller grants that, scoped and expiring, and +// discards the client when the operation is done. +func NewEphemeral(baseURL, vtaDid string) (*Client, error) { + _, key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, fmt.Errorf("mint ephemeral VTA client key: %w", err) + } + return newClient(baseURL, vtaDid, key) +} + +func newClient(baseURL, vtaDid string, key ed25519.PrivateKey) (*Client, error) { + baseURL = strings.TrimRight(baseURL, "/") + if baseURL == "" { + return nil, errors.New("VTA URL is required") + } + if vtaDid == "" { + return nil, errors.New("VTA DID is required") + } + did, err := didkey.FromPublicKey(key.Public().(ed25519.PublicKey)) + if err != nil { + return nil, err + } + return &Client{ + baseURL: baseURL, + vtaDid: vtaDid, + did: did, + kid: did + "#" + strings.TrimPrefix(did, "did:key:"), + key: key, + hc: &http.Client{ + Timeout: 60 * time.Second, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }, + resolver: didresolver.MethodResolver{WebVH: webvh.NewResolver(5 * time.Second)}, + now: time.Now, + probeInterval: 5 * time.Second, + }, nil +} + +// DID is the client's did:key, the subject of the ACL entry it needs. +func (c *Client) DID() string { return c.did } + +// Login runs the challenge and authenticate tasks and keeps the access token +// for the calls that follow. The VTA hands a challenge to any DID and refuses +// the authenticate step for one its ACL does not name, so a missing or +// expired grant surfaces here as a `permissionDenied` TaskError. +func (c *Client) Login(ctx context.Context) error { + var challenge struct { + Challenge string `json:"challenge"` + SessionID string `json:"sessionId"` + } + reply, err := c.send(ctx, challengeType, map[string]any{"subject": c.did}, false, false) + if err != nil { + return fmt.Errorf("VTA challenge: %w", err) + } + if err := json.Unmarshal(reply, &challenge); err != nil || challenge.Challenge == "" || challenge.SessionID == "" { + return errors.New("VTA challenge response is missing the challenge or session id") + } + + var tokens struct { + Tokens struct { + AccessToken string `json:"accessToken"` + } `json:"tokens"` + } + reply, err = c.send(ctx, authenticateType, map[string]any{"challenge": challenge.Challenge, "sessionId": challenge.SessionID}, true, false) + if err != nil { + return fmt.Errorf("VTA authenticate: %w", err) + } + if err := json.Unmarshal(reply, &tokens); err != nil || tokens.Tokens.AccessToken == "" { + return errors.New("VTA authenticate response carries no access token") + } + c.token = tokens.Tokens.AccessToken + return nil +} + +// DIDLog returns the did.jsonl the VTA holds for did, one log entry per +// line. The last line is the local head, which may be ahead of what the DID +// host serves if an earlier publish failed; the VTA re-publishes it before +// its next update. +func (c *Client) DIDLog(ctx context.Context, did string) (string, error) { + reply, err := c.send(ctx, didGetType, map[string]any{"did": did, "includeLog": true}, true, true) + if err != nil { + return "", fmt.Errorf("read the DID log from the VTA: %w", err) + } + var got struct { + Log string `json:"log"` + } + if err := json.Unmarshal(reply, &got); err != nil || strings.TrimSpace(got.Log) == "" { + return "", errors.New("the VTA returned no DID log") + } + return got.Log, nil +} + +// UpdateDIDDocument replaces did's document and has the VTA publish the new +// log entry to the DID host over whichever transport the two share. The VTA +// rotates the DID's update key as part of any document change. With +// expectedVersionID set, the VTA refuses (ReasonConflict) if its head has +// moved since the caller read it. It returns the new versionId. +func (c *Client) UpdateDIDDocument(ctx context.Context, did string, document json.RawMessage, expectedVersionID string) (string, error) { + payload := map[string]any{"did": did, "document": document} + if expectedVersionID != "" { + payload["expectedVersionId"] = expectedVersionID + } + reply, err := c.send(ctx, didUpdateType, payload, true, true) + if err != nil { + return "", fmt.Errorf("update the DID document at the VTA: %w", err) + } + var got struct { + NewVersionID string `json:"newVersionId"` + Serverless bool `json:"serverless"` + } + if err := json.Unmarshal(reply, &got); err != nil || got.NewVersionID == "" { + return "", errors.New("the VTA reported no new version id") + } + if got.Serverless { + return "", fmt.Errorf("the VTA holds %s as a serverless DID; nothing published it", did) + } + return got.NewVersionID, nil +} + +// HostReachable asks the VTA for the hosting domains it may mint into on the +// DID host registered as serverID (`vta/webvh/servers/domains/0.1`), a +// read-only task the VTA can answer only by reaching that host over a +// transport the two share. It is the probe for the one state a caller cannot +// see from outside: right after a restart the VTA answers HTTPS before its +// mediator session is up, and in that window it can initiate REST alone, so a +// host that advertises TSP or DIDComm only is out of its reach and a document +// update would commit locally and then fail to publish. +func (c *Client) HostReachable(ctx context.Context, serverID string) error { + _, err := c.send(ctx, domainsType, map[string]any{"serverId": serverID}, true, true) + return err +} + +// WaitUntilHostReachable polls HostReachable until it succeeds, the timeout +// passes, or ctx ends. The last refusal is returned on timeout. +func (c *Client) WaitUntilHostReachable(ctx context.Context, serverID string, timeout time.Duration) error { + deadline := c.now().Add(timeout) + for { + err := c.HostReachable(ctx, serverID) + if err == nil { + return nil + } + if !c.now().Before(deadline) { + return fmt.Errorf("the VTA could not reach DID host %q within %s: %w", serverID, timeout, err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(c.probeInterval): + } + } +} + +// TaskError is a Trust Task refusal, as the VTA's `trust-task-error/0.5` +// document carries it. +type TaskError struct { + Status int + Code string + Reason string + Message string +} + +func (e *TaskError) Error() string { + text := fmt.Sprintf("the VTA refused the task: %s", e.Code) + if e.Reason != "" { + text += " (" + e.Reason + ")" + } + if e.Message != "" { + text += ": " + e.Message + } + return text +} + +// Conflict reports an `expectedVersionId` that no longer named the head. +func (e *TaskError) Conflict() bool { return e.Reason == ReasonConflict } + +// send POSTs one Trust Task and returns the verified reply's payload. signed +// attaches the did:key's proof; bearer sends the session token, which the +// login tasks must not carry. +func (c *Client) send(ctx context.Context, typeURI string, payload any, signed, bearer bool) (json.RawMessage, error) { + payloadJSON, err := json.Marshal(payload) + if err != nil { + return nil, fmt.Errorf("encode Trust Task payload: %w", err) + } + now := c.now().UTC().Format(time.RFC3339) + doc := trusttaskclient.Document{ + ID: "urn:uuid:" + uuid.New().String(), + Type: typeURI, + Issuer: c.did, + Recipient: c.vtaDid, + IssuedAt: now, + Payload: payloadJSON, + } + if signed { + if err := trusttaskclient.Sign(&doc, c.kid, c.key, now); err != nil { + return nil, fmt.Errorf("sign Trust Task: %w", err) + } + } + body, err := json.Marshal(doc) + if err != nil { + return nil, fmt.Errorf("encode Trust Task: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, c.baseURL+trustTasksPath, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("create Trust Task request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Trust-Task", typeURI) + if bearer { + if c.token == "" { + return nil, errors.New("not logged in to the VTA") + } + req.Header.Set("Authorization", "Bearer "+c.token) + } + resp, err := c.hc.Do(req) + if err != nil { + return nil, fmt.Errorf("send Trust Task: %w", err) + } + defer resp.Body.Close() + responseBody, err := io.ReadAll(io.LimitReader(resp.Body, trusttaskclient.MaxResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("read Trust Task response: %w", err) + } + if len(responseBody) > trusttaskclient.MaxResponseBytes { + return nil, errors.New("VTA Trust Task response exceeds 1 MiB") + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return nil, refusal(resp.StatusCode, responseBody) + } + + var reply trusttaskclient.Document + if err := json.Unmarshal(responseBody, &reply); err != nil { + return nil, fmt.Errorf("decode Trust Task response: %w", err) + } + if err := trusttaskclient.VerifyReply(ctx, c.resolver, &reply, &doc, trusttaskclient.Reply{Service: c.vtaDid, Client: c.did, Now: c.now}); err != nil { + return nil, fmt.Errorf("VTA %w", err) + } + if reply.Type != typeURI+"#response" { + return nil, fmt.Errorf("unexpected Trust Task response type %q", reply.Type) + } + return reply.Payload, nil +} + +// refusal reads a non-2xx body: a `trust-task-error/0.5` document becomes a +// TaskError; anything else (a flat `{"error":…}`, a proxy page) is quoted. +func refusal(status int, body []byte) error { + var doc struct { + Type string `json:"type"` + Payload struct { + Code string `json:"code"` + Message string `json:"message"` + Details struct { + Reason string `json:"reason"` + } `json:"details"` + } `json:"payload"` + } + if err := json.Unmarshal(body, &doc); err == nil && doc.Type == errorType && doc.Payload.Code != "" { + return &TaskError{Status: status, Code: doc.Payload.Code, Reason: doc.Payload.Details.Reason, Message: doc.Payload.Message} + } + text := strings.TrimSpace(string(body)) + if len(text) > 512 { + text = text[:512] + "…" + } + return fmt.Errorf("VTA returned HTTP %d: %s", status, text) +} + +// LatestLogEntry reads the head of a did.jsonl: its versionId, and the DID +// document (`state`) it establishes. +func LatestLogEntry(didLog string) (versionID string, state json.RawMessage, err error) { + lines := strings.Split(didLog, "\n") + var last string + for i := len(lines) - 1; i >= 0; i-- { + if strings.TrimSpace(lines[i]) != "" { + last = lines[i] + break + } + } + if last == "" { + return "", nil, errors.New("the DID log is empty") + } + var entry struct { + VersionID string `json:"versionId"` + State json.RawMessage `json:"state"` + } + if err := json.Unmarshal([]byte(last), &entry); err != nil { + return "", nil, fmt.Errorf("parse the DID log head: %w", err) + } + if entry.VersionID == "" || len(entry.State) == 0 { + return "", nil, errors.New("the DID log head has no versionId or state") + } + return entry.VersionID, entry.State, nil +} diff --git a/internal/vta/client_test.go b/internal/vta/client_test.go new file mode 100644 index 0000000..3760434 --- /dev/null +++ b/internal/vta/client_test.go @@ -0,0 +1,342 @@ +package vta + +import ( + "context" + "crypto/ed25519" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/ic3software/vtafarm-api/internal/didkey" + "github.com/ic3software/vtafarm-api/internal/didresolver" + "github.com/ic3software/vtafarm-api/internal/trusttaskclient" +) + +type identity struct { + did string + kid string + private ed25519.PrivateKey +} + +func newIdentity(t *testing.T, fill byte) identity { + t.Helper() + seed := make([]byte, ed25519.SeedSize) + for i := range seed { + seed[i] = fill + } + private := ed25519.NewKeyFromSeed(seed) + did, err := didkey.FromPublicKey(private.Public().(ed25519.PublicKey)) + if err != nil { + t.Fatal(err) + } + return identity{did: did, kid: did + "#" + strings.TrimPrefix(did, "did:key:"), private: private} +} + +// fakeVTA answers the four tasks the client sends the way a VTA does: a +// signed, threaded `#response` from the VTA's own DID (a did:key here, so the +// client's resolver needs no network), the login tasks without a bearer, and +// everything else only with the token the authenticate step handed out. +type fakeVTA struct { + t *testing.T + self identity + client identity + token string + log string + updates []map[string]json.RawMessage + refuse *TaskError + // domainsRefusals is how many domains probes to refuse before answering, + // the shape of a VTA whose mediator session is not up yet. + domainsRefusals int + domainsProbes int +} + +func (f *fakeVTA) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != trustTasksPath || r.Method != http.MethodPost { + http.NotFound(w, r) + return + } + var doc trusttaskclient.Document + if err := json.NewDecoder(r.Body).Decode(&doc); err != nil { + f.t.Errorf("decode request: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + if doc.Recipient != f.self.did || doc.Issuer != f.client.did || !strings.HasPrefix(doc.ID, "urn:uuid:") { + f.t.Errorf("unexpected envelope: %+v", doc) + } + if r.Header.Get("Trust-Task") != doc.Type { + f.t.Errorf("Trust-Task header %q does not name the document type %q", r.Header.Get("Trust-Task"), doc.Type) + } + bearer := r.Header.Get("Authorization") + login := doc.Type == challengeType || doc.Type == authenticateType + if login && bearer != "" { + f.t.Errorf("%s carried a bearer token", doc.Type) + } + if !login && bearer != "Bearer "+f.token { + f.t.Errorf("%s carried authorization %q, want the session token", doc.Type, bearer) + } + if doc.Type != challengeType { + f.verifyProof(&doc) + } + if f.refuse != nil && doc.Type == didUpdateType { + f.writeRefusal(w, &doc, f.refuse) + return + } + if doc.Type == domainsType { + f.domainsProbes++ + if f.domainsRefusals > 0 { + f.domainsRefusals-- + f.writeRefusal(w, &doc, &TaskError{Status: http.StatusInternalServerError, Code: "internalError", Message: "no transport in common with `did:webvh:QmHost:dids.example`: it advertises [tsp, didcomm] and this agent can initiate [rest]"}) + return + } + f.writeReply(w, &doc, doc.Type+"#response", map[string]any{"domains": []any{}, "default": ""}) + return + } + var payload any + switch doc.Type { + case challengeType: + var req struct { + Subject string `json:"subject"` + } + _ = json.Unmarshal(doc.Payload, &req) + if req.Subject != f.client.did { + f.t.Errorf("challenge subject %q", req.Subject) + } + payload = map[string]any{"challenge": "6f1a" + strings.Repeat("ab", 30), "sessionId": "sess-1", "expiresAt": "2026-10-23T12:05:00Z"} + case authenticateType: + var req struct { + Challenge string `json:"challenge"` + SessionID string `json:"sessionId"` + } + _ = json.Unmarshal(doc.Payload, &req) + if req.SessionID != "sess-1" || !strings.HasPrefix(req.Challenge, "6f1a") { + f.t.Errorf("authenticate payload %s", doc.Payload) + } + f.token = "jwt-for-" + f.client.did + payload = map[string]any{"session": map[string]any{"subject": f.client.did}, "tokens": map[string]any{"accessToken": f.token, "tokenType": "Bearer", "expiresIn": 900}} + case didGetType: + payload = map[string]any{"record": map[string]any{"serverId": "dids"}, "log": f.log} + case didUpdateType: + var req map[string]json.RawMessage + _ = json.Unmarshal(doc.Payload, &req) + f.updates = append(f.updates, req) + payload = map[string]any{"did": "did:webvh:QmVtc:dids.example:vtc", "newVersionId": "3-QmNew", "serverless": false} + default: + f.t.Errorf("unexpected task %s", doc.Type) + w.WriteHeader(http.StatusUnprocessableEntity) + return + } + f.writeReply(w, &doc, doc.Type+"#response", payload) + } +} + +func (f *fakeVTA) verifyProof(doc *trusttaskclient.Document) { + f.t.Helper() + if doc.Proof == nil { + f.t.Errorf("%s carried no proof", doc.Type) + return + } + p := *doc.Proof + if p.ProofPurpose != "authentication" || p.VerificationMethod != f.client.kid { + f.t.Errorf("proof %+v", p) + } + signature, err := trusttaskclient.DecodeProofValue(p.ProofValue) + if err != nil { + f.t.Error(err) + return + } + doc.Proof = nil + message, err := trusttaskclient.ProofMessage(doc, p) + doc.Proof = &p + if err != nil { + f.t.Error(err) + return + } + key, err := (didresolver.DIDKeyResolver{}).ResolveAuthenticationKey(context.Background(), f.client.did, f.client.kid) + if err != nil { + f.t.Error(err) + return + } + if !ed25519.Verify(key, message, signature) { + f.t.Errorf("%s proof did not verify", doc.Type) + } +} + +func (f *fakeVTA) writeReply(w http.ResponseWriter, request *trusttaskclient.Document, typeURI string, payload any) { + body, _ := json.Marshal(payload) + now := time.Now().UTC().Format(time.RFC3339) + reply := trusttaskclient.Document{ + ID: "urn:uuid:reply", + Type: typeURI, + ThreadID: request.ID, + Issuer: f.self.did, + Recipient: f.client.did, + IssuedAt: now, + Payload: body, + } + if err := trusttaskclient.Sign(&reply, f.self.kid, f.self.private, now); err != nil { + f.t.Fatal(err) + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(reply) +} + +func (f *fakeVTA) writeRefusal(w http.ResponseWriter, request *trusttaskclient.Document, refusal *TaskError) { + details := map[string]any{} + if refusal.Reason != "" { + details["reason"] = refusal.Reason + } + body, _ := json.Marshal(map[string]any{ + "code": refusal.Code, + "message": refusal.Message, + "retryable": false, + "inResponseTo": map[string]any{"id": request.ID, "typeUri": request.Type}, + "details": details, + }) + reply := trusttaskclient.Document{ID: "urn:uuid:err", Type: errorType, ThreadID: request.ID, Issuer: f.self.did, Recipient: f.client.did, IssuedAt: time.Now().UTC().Format(time.RFC3339), Payload: body} + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(refusal.Status) + _ = json.NewEncoder(w).Encode(reply) +} + +const testLog = `{"versionId":"1-QmOne","state":{"id":"did:webvh:QmVtc:dids.example:vtc","service":[]}} +{"versionId":"2-QmTwo","state":{"id":"did:webvh:QmVtc:dids.example:vtc","service":[{"id":"did:webvh:QmVtc:dids.example:vtc#tsp","type":"TSPTransport","serviceEndpoint":"did:webvh:QmMed:dids.example:mediator"}]}} +` + +func newTestClient(t *testing.T, fake *fakeVTA) *Client { + t.Helper() + server := httptest.NewServer(fake.handler()) + t.Cleanup(server.Close) + client, err := newClient(server.URL+"/", fake.self.did, fake.client.private) + if err != nil { + t.Fatal(err) + } + client.resolver = didresolver.DIDKeyResolver{} + return client +} + +func TestLoginReadAndUpdateOverOneEndpoint(t *testing.T) { + fake := &fakeVTA{t: t, self: newIdentity(t, 1), client: newIdentity(t, 2), log: testLog} + client := newTestClient(t, fake) + ctx := context.Background() + + if client.DID() != fake.client.did { + t.Fatalf("client DID %q", client.DID()) + } + if _, err := client.DIDLog(ctx, "did:webvh:QmVtc:dids.example:vtc"); err == nil { + t.Fatal("an authenticated task went out before login") + } + if err := client.Login(ctx); err != nil { + t.Fatal(err) + } + if client.token != "jwt-for-"+fake.client.did { + t.Fatalf("token %q", client.token) + } + + didLog, err := client.DIDLog(ctx, "did:webvh:QmVtc:dids.example:vtc") + if err != nil { + t.Fatal(err) + } + version, state, err := LatestLogEntry(didLog) + if err != nil { + t.Fatal(err) + } + if version != "2-QmTwo" || !strings.Contains(string(state), "TSPTransport") { + t.Fatalf("head %s %s", version, state) + } + + newVersion, err := client.UpdateDIDDocument(ctx, "did:webvh:QmVtc:dids.example:vtc", state, version) + if err != nil { + t.Fatal(err) + } + if newVersion != "3-QmNew" { + t.Fatalf("new version %q", newVersion) + } + if len(fake.updates) != 1 { + t.Fatalf("%d updates sent", len(fake.updates)) + } + sent := fake.updates[0] + if string(sent["did"]) != `"did:webvh:QmVtc:dids.example:vtc"` || string(sent["expectedVersionId"]) != `"2-QmTwo"` { + t.Fatalf("update payload %v", sent) + } + if !json.Valid(sent["document"]) || !strings.Contains(string(sent["document"]), "TSPTransport") { + t.Fatalf("update document %s", sent["document"]) + } + if _, has := sent["preRotationCount"]; has { + t.Fatal("update payload carried a field the caller did not set") + } +} + +func TestARefusalBecomesATaskError(t *testing.T) { + fake := &fakeVTA{t: t, self: newIdentity(t, 3), client: newIdentity(t, 4), log: testLog, + refuse: &TaskError{Status: http.StatusUnprocessableEntity, Code: "taskFailed", Reason: ReasonConflict, Message: "expectedVersionId does not match"}} + client := newTestClient(t, fake) + ctx := context.Background() + if err := client.Login(ctx); err != nil { + t.Fatal(err) + } + _, err := client.UpdateDIDDocument(ctx, "did:webvh:QmVtc:dids.example:vtc", json.RawMessage(`{"id":"did:webvh:QmVtc:dids.example:vtc"}`), "1-QmOne") + var refusal *TaskError + if !errors.As(err, &refusal) { + t.Fatalf("error %v is not a TaskError", err) + } + if !refusal.Conflict() || refusal.Status != http.StatusUnprocessableEntity || refusal.Code != "taskFailed" { + t.Fatalf("refusal %+v", refusal) + } + if !strings.Contains(err.Error(), "taskFailed (conflict): expectedVersionId does not match") { + t.Fatalf("message %q", err.Error()) + } +} + +func TestAReplyFromAnotherDIDIsRefused(t *testing.T) { + fake := &fakeVTA{t: t, self: newIdentity(t, 5), client: newIdentity(t, 6), log: testLog} + client := newTestClient(t, fake) + // The client expects the VTA to be someone else: every reply now fails + // verification before its payload is read. + client.vtaDid = newIdentity(t, 7).did + fake.self = identity{did: client.vtaDid, kid: fake.self.kid, private: fake.self.private} + err := client.Login(context.Background()) + if err == nil || !strings.Contains(err.Error(), "proof") { + t.Fatalf("login error %v", err) + } +} + +func TestWaitUntilHostReachableOutlastsTheMediatorGap(t *testing.T) { + fake := &fakeVTA{t: t, self: newIdentity(t, 8), client: newIdentity(t, 9), log: testLog, domainsRefusals: 2} + client := newTestClient(t, fake) + client.probeInterval = time.Millisecond + ctx := context.Background() + if err := client.Login(ctx); err != nil { + t.Fatal(err) + } + if err := client.WaitUntilHostReachable(ctx, "dids", time.Second); err != nil { + t.Fatal(err) + } + if fake.domainsProbes != 3 { + t.Fatalf("%d probes", fake.domainsProbes) + } + + fake.domainsRefusals = 1 << 20 + err := client.WaitUntilHostReachable(ctx, "dids", 20*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "no transport in common") { + t.Fatalf("timeout error %v", err) + } +} + +func TestLatestLogEntry(t *testing.T) { + if _, _, err := LatestLogEntry("\n\n"); err == nil { + t.Fatal("an empty log parsed") + } + if _, _, err := LatestLogEntry(`{"versionId":"1-x"}`); err == nil { + t.Fatal("an entry without state parsed") + } + version, state, err := LatestLogEntry("{\"versionId\":\"1-a\",\"state\":{\"id\":\"did:webvh:a\"}}\n{\"versionId\":\"2-b\",\"state\":{\"id\":\"did:webvh:a\",\"service\":[]}}\n\n") + if err != nil || version != "2-b" || string(state) != `{"id":"did:webvh:a","service":[]}` { + t.Fatalf("%s %s %v", version, state, err) + } +} From b84b4e67f5736b799e96271c725429ffb57e17c9 Mon Sep 17 00:00:00 2001 From: vthwang Date: Fri, 23 Oct 2026 22:55:14 -0700 Subject: [PATCH 5/7] fix: shorten referral ACL grant lifetime Reduce the exposure window for the ephemeral VTA administrator grant while preserving enough time for the bounded registry workflow. - Change the referral ACL expiry from one hour to fifteen minutes - Keep the design documentation aligned with the implementation Signed-off-by: vthwang --- docs/optional-trust-registry-design.md | 2 +- internal/setup/trust_registry_referral.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md index 7d73350..b389102 100644 --- a/docs/optional-trust-registry-design.md +++ b/docs/optional-trust-registry-design.md @@ -223,7 +223,7 @@ existing PVCs, or re-enroll the existing administrators. services, and history. The farm mints a did:key for this one operation, grants it as a context-scoped admin of the VTC's context in the same stopped-VTA window pattern as step 3 (`vta acl create --role admin --contexts - --expires 1h`), logs in to the VTA over HTTPS as that key + --expires 15m`), logs in to the VTA over HTTPS as that key (`auth/challenge/0.1`, `auth/authenticate/0.2`), reads the VTA's head of the log (`vta/webvh/dids/get/1.0`), and sends `vta/webvh/dids/update/1.0` with the document plus `expectedVersionId`. The VTA signs the new entry and publishes diff --git a/internal/setup/trust_registry_referral.go b/internal/setup/trust_registry_referral.go index fcb4da3..2561276 100644 --- a/internal/setup/trust_registry_referral.go +++ b/internal/setup/trust_registry_referral.go @@ -18,7 +18,7 @@ const ( referralGrantLabel = "trust-registry-referral" // referralGrantTTL bounds that entry; the VTA's sweeper removes it. The // key itself lives only in this process, for the length of one call. - referralGrantTTL = "1h" + referralGrantTTL = "15m" // referralHostWait is how long to give the restarted VTA to bring its // mediator session up before the edit, whose publish needs it. referralHostWait = 3 * time.Minute @@ -45,7 +45,7 @@ var newVtaClient = vta.NewEphemeral // // The farm has no standing credential at a user's VTA, and does not want one. // It mints a did:key for this call, grants it as an admin of the VTC's context -// for an hour in the same stopped-VTA window pattern the registry context +// for fifteen minutes in the same stopped-VTA window pattern the registry context // grant uses, logs in over HTTPS, and sends the update signed by that key. On // return the key is garbage; the entry expires on its own. // From 5bfb6d06bd8551321c73e35ab46b37658fee6f74 Mon Sep 17 00:00:00 2001 From: vthwang Date: Fri, 23 Oct 2026 23:43:04 -0700 Subject: [PATCH 6/7] fix: make trust registry teardown resumable Cancel Registry workers before acquiring the teardown lease and clear abandoned Registry maintenance leases under the advisory lock. Apply the protocol to owner, admin, pool, and load-test cleanup paths. Signed-off-by: vthwang --- internal/handler/admin_load_testing.go | 20 +++-- .../handler/admin_platform_stack_admins.go | 19 +++++ internal/handler/setup.go | 30 ++++--- internal/handler/setup_delete_test.go | 80 +++++++++++++++++++ internal/setup/initial_provision_test.go | 38 +++++++++ internal/setup/orchestrator.go | 16 +++- 6 files changed, 184 insertions(+), 19 deletions(-) create mode 100644 internal/handler/setup_delete_test.go diff --git a/internal/handler/admin_load_testing.go b/internal/handler/admin_load_testing.go index 8e9366b..7dbbe62 100644 --- a/internal/handler/admin_load_testing.go +++ b/internal/handler/admin_load_testing.go @@ -397,14 +397,20 @@ func (h *SetupHandler) deleteLoadTest(runID uint, userID *uint) { errorsSeen := make([]string, 0) for i := range sessions { s := &sessions[i] - if h.orch != nil { - if err := h.orch.Cancel(s.ID); err != nil { - errorsSeen = append(errorsSeen, fmt.Sprintf("%s: unable to stop provisioning", s.VtaName)) - continue - } - } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) - _, err := h.teardownVtaOnlySession(ctx, s) + if err := h.prepareSessionTeardown(ctx, s.ID); err != nil { + cancel() + errorsSeen = append(errorsSeen, fmt.Sprintf("%s: unable to stop provisioning", s.VtaName)) + continue + } + release, err := h.acquireTeardownMaintenance(s.ID) + if err != nil { + cancel() + errorsSeen = append(errorsSeen, fmt.Sprintf("%s: maintenance still in progress", s.VtaName)) + continue + } + _, err = h.teardownVtaOnlySession(ctx, s) + release() cancel() if err != nil { errorsSeen = append(errorsSeen, fmt.Sprintf("%s: %v", s.VtaName, err)) diff --git a/internal/handler/admin_platform_stack_admins.go b/internal/handler/admin_platform_stack_admins.go index bde4005..3596cc9 100644 --- a/internal/handler/admin_platform_stack_admins.go +++ b/internal/handler/admin_platform_stack_admins.go @@ -121,6 +121,25 @@ func (h *SetupHandler) acquireSessionMaintenance(sessionID uint) (func(), error) return release, nil } +// acquireTeardownMaintenance is the destructive counterpart to +// acquireSessionMaintenance. The caller must first durably move the session to +// deleting and quiesce its workers with prepareSessionTeardown. At that point +// an enabling Registry is work being cancelled, not a reason to block cleanup. +// +// Keep the order prepareSessionTeardown -> acquireTeardownMaintenance. The +// Registry worker takes its advisory lock before this stack lease; reversing +// that order here would let deletion and enablement deadlock each other. +func (h *SetupHandler) acquireTeardownMaintenance(sessionID uint) (func(), error) { + var session model.SetupSession + if err := h.db.Select("status").First(&session, sessionID).Error; err != nil { + return nil, err + } + if session.Status != "deleting" { + return nil, errors.New("session must be marked deleting before teardown maintenance") + } + return h.acquireStackMaintenance(sessionID) +} + func (h *SetupHandler) acquireStackMaintenance(sessionID uint) (func(), error) { if !aclJobLocks.TryLock(sessionID) { return nil, errAclJobBusy diff --git a/internal/handler/setup.go b/internal/handler/setup.go index 539a4b2..b8c9b81 100644 --- a/internal/handler/setup.go +++ b/internal/handler/setup.go @@ -840,13 +840,11 @@ func (h *SetupHandler) teardownSession(c *gin.Context, session *model.SetupSessi c.JSON(http.StatusConflict, gin.H{"error": "Use pool inventory cleanup for unclaimed VTAs. Confirmed claims are protected.", "reason": "pooled_session"}) return } - if h.orch != nil { - if err := h.orch.Cancel(session.ID); err != nil { - c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Unable to stop provisioning before deletion. Please retry."}) - return - } + if err := h.prepareSessionTeardown(c.Request.Context(), session.ID); err != nil { + c.JSON(http.StatusServiceUnavailable, gin.H{"error": "Unable to stop provisioning before deletion. Please retry."}) + return } - release, err := h.acquireSessionMaintenance(session.ID) + release, err := h.acquireTeardownMaintenance(session.ID) if err != nil { c.JSON(http.StatusConflict, gin.H{"error": "A VTA maintenance operation is still running. Retry deletion after it finishes."}) return @@ -864,6 +862,18 @@ func (h *SetupHandler) teardownSession(c *gin.Context, session *model.SetupSessi c.Status(http.StatusNoContent) } +// prepareSessionTeardown establishes the durable half of the deletion +// protocol before any resource is removed. Orchestrator.Cancel also cancels +// local workers and waits for a Registry worker on another replica. Without an +// orchestrator (tests and deliberately reduced deployments), preserve the same +// state transition directly so teardown never relies on an in-memory owner. +func (h *SetupHandler) prepareSessionTeardown(ctx context.Context, sessionID uint) error { + if h.orch != nil { + return h.orch.Cancel(sessionID) + } + return connection.Stop(ctx, h.db, sessionID) +} + // teardownVtaOnlySession removes one VTA-only session without writing an HTTP // response. The user/admin delete handlers and load-test batch cleanup share it // so all three remove the same DNS, DID-hosting, Kubernetes and Vault state. @@ -1012,12 +1022,10 @@ func (h *SetupHandler) cleanupPoolSession(ctx context.Context, sessionID uint) e if confirmed != 0 { return errors.New("confirmed pool claim cannot be cleaned as unused inventory") } - if h.orch != nil { - if err := h.orch.Cancel(sessionID); err != nil { - return err - } + if err := h.prepareSessionTeardown(ctx, sessionID); err != nil { + return err } - release, err := h.acquireSessionMaintenance(sessionID) + release, err := h.acquireTeardownMaintenance(sessionID) if err != nil { return err } diff --git a/internal/handler/setup_delete_test.go b/internal/handler/setup_delete_test.go new file mode 100644 index 0000000..de6ea7b --- /dev/null +++ b/internal/handler/setup_delete_test.go @@ -0,0 +1,80 @@ +package handler + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/ic3software/vtafarm-api/internal/middleware" + "github.com/ic3software/vtafarm-api/internal/model" + "github.com/ic3software/vtafarm-api/internal/testutil" + "gorm.io/gorm" +) + +func TestDeleteCancelsEnablingRegistry(t *testing.T) { + for _, initialStatus := range []string{"running", "deleting"} { + for _, admin := range []bool{false, true} { + name := initialStatus + "/owner" + if admin { + name = initialStatus + "/admin" + } + t.Run(name, func(t *testing.T) { + db := testutil.Postgres(t) + session := testutil.SetupSession(t, db, model.ModeFullStack) + if err := db.Model(&session).Updates(map[string]any{ + "status": initialStatus, + "trust_registry_enabled": true, + "trust_registry_state": "enabling", + "trust_registry_phase": "grant_access", + "trust_registry_image": "registry:test", + "trust_registry_did": "did:webvh:registry.example", + }).Error; err != nil { + t.Fatal(err) + } + + h := &SetupHandler{db: db} + r := gin.New() + path := "/setup/" + session.VtaName + if admin { + r.DELETE("/setup/:id", h.AdminDeleteSession) + } else { + r.Use(func(c *gin.Context) { c.Set(middleware.ContextUserID, session.UserID) }) + r.DELETE("/setup/:id", h.Delete) + } + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, path, nil)) + if w.Code != http.StatusNoContent { + t.Fatalf("status=%d body=%s", w.Code, w.Body.String()) + } + if err := db.First(&model.SetupSession{}, session.ID).Error; !errors.Is(err, gorm.ErrRecordNotFound) { + t.Fatalf("session still exists after deletion: %v", err) + } + }) + } + } +} + +func TestTeardownMaintenanceRequiresDeletingState(t *testing.T) { + db := testutil.Postgres(t) + session := testutil.SetupSession(t, db, model.ModeFullStack) + h := &SetupHandler{db: db} + + if release, err := h.acquireTeardownMaintenance(session.ID); err == nil { + release() + t.Fatal("teardown maintenance accepted a session that was not deleting") + } + if err := db.Model(&session).Updates(map[string]any{ + "status": "deleting", + "trust_registry_enabled": true, + "trust_registry_state": "enabling", + }).Error; err != nil { + t.Fatal(err) + } + release, err := h.acquireTeardownMaintenance(session.ID) + if err != nil { + t.Fatalf("teardown maintenance rejected cancelled Registry work: %v", err) + } + release() +} diff --git a/internal/setup/initial_provision_test.go b/internal/setup/initial_provision_test.go index 359520d..eb65c0d 100644 --- a/internal/setup/initial_provision_test.go +++ b/internal/setup/initial_provision_test.go @@ -144,3 +144,41 @@ func TestTeardownStopsWorkerOnAnotherReplica(t *testing.T) { t.Error("teardown work restarted") }) } + +func TestTeardownClearsAbandonedRegistryMaintenanceLease(t *testing.T) { + db := testutil.Postgres(t) + s := testutil.SetupSession(t, db, model.ModeFullStack) + started := time.Now().Add(-time.Minute) + if err := db.Model(&s).Updates(map[string]any{ + "status": "running", + "trust_registry_enabled": true, + "trust_registry_state": "enabling", + }).Error; err != nil { + t.Fatal(err) + } + if err := db.Create(&model.VtaAclSnapshot{ + SessionID: s.ID, MaintenanceStartedAt: &started, + }).Error; err != nil { + t.Fatal(err) + } + + o := &Orchestrator{db: db} + if err := o.Cancel(s.ID); err != nil { + t.Fatal(err) + } + + var got model.SetupSession + if err := db.First(&got, s.ID).Error; err != nil { + t.Fatal(err) + } + if got.Status != "deleting" { + t.Fatalf("status = %q, want deleting", got.Status) + } + var snapshot model.VtaAclSnapshot + if err := db.First(&snapshot, "session_id = ?", s.ID).Error; err != nil { + t.Fatal(err) + } + if snapshot.MaintenanceStartedAt != nil { + t.Fatalf("maintenance lease was not cleared: %v", snapshot.MaintenanceStartedAt) + } +} diff --git a/internal/setup/orchestrator.go b/internal/setup/orchestrator.go index 85430d4..9162803 100644 --- a/internal/setup/orchestrator.go +++ b/internal/setup/orchestrator.go @@ -162,8 +162,22 @@ func (o *Orchestrator) Cancel(sessionID uint) error { o.mu.Unlock() // A worker on another replica observes the deleting state on its next // heartbeat. Wait for its lock before deleting the resources it can create. + // Once this transaction owns the advisory lock, no Registry worker can + // still own the stack-maintenance lease. Clear a lease it abandoned so a + // resumed deletion does not have to wait for the generic stale timeout. return o.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error { - return tx.Exec("SELECT pg_advisory_xact_lock($1, $2)", 565441, int64(sessionID)).Error + if err := tx.Exec("SELECT pg_advisory_xact_lock($1, $2)", 565441, int64(sessionID)).Error; err != nil { + return err + } + var session model.SetupSession + if err := tx.Select("status", "trust_registry_state").First(&session, sessionID).Error; err != nil { + return err + } + if session.Status != "deleting" || session.TrustRegistryState != "enabling" { + return nil + } + return tx.Model(&model.VtaAclSnapshot{}).Where("session_id = ?", sessionID). + Update("maintenance_started_at", nil).Error }) } From 7ce37571fe66b9a462cebf5fadc744b35ce4d689 Mon Sep 17 00:00:00 2001 From: vthwang Date: Sat, 24 Oct 2026 01:39:28 -0700 Subject: [PATCH 7/7] fix: fail setup when trust registry fails Prevent an opted-in new stack from being reported as running after Registry setup fails. Persist the Registry failure as the session error, pin it to the Registry phase, and document and test the behavior. Signed-off-by: vthwang --- docs/optional-trust-registry-design.md | 21 ++++++++++++--------- internal/setup/trust_registry.go | 14 +++++++++++++- internal/setup/trust_registry_setup_test.go | 10 ++++++++-- 3 files changed, 33 insertions(+), 12 deletions(-) diff --git a/docs/optional-trust-registry-design.md b/docs/optional-trust-registry-design.md index b389102..bb0002a 100644 --- a/docs/optional-trust-registry-design.md +++ b/docs/optional-trust-registry-design.md @@ -185,7 +185,7 @@ existing bootstrap and admin gate -> deploy registry and check its messaging/write-path status -> merge [registry].did into the VTC runtime config and restart VTC -> verify the published referral and finish add-on enablement - -> running (Registry has its own active/failed state) + -> running only after Registry activation succeeds ``` Follow [upstream headless setup][tr-headless], using Registry Jobs for both setup @@ -194,10 +194,11 @@ phases and a single offline VTA context grant. No PNM credential or offline headless setup plus `vta`, `secrets-vault`, `storage-fjall` and `standalone`; the VTA must include [the trust-registry template][vta-tr-template]. -A Registry setup failure is recorded independently and lets VTC setup proceed -without a referral. It is not immediately retried at the end of that same run. -The user can retry enablement after correcting the cause. Preserve the VTC's -existing claim/install output; its setup now follows Registry provisioning. +A Registry identity setup failure is recorded while VTC setup proceeds without +a referral, so the baseline components can finish provisioning safely. The +session then stops at the Trust Registry step and reports the Registry error; +it must not be marked running. Preserve the VTC's existing claim/install output; +its setup now follows Registry provisioning. ## 6. Add to an existing stack without rebuilding @@ -323,8 +324,9 @@ add-on state (`disabled`, `enabling`, `active`, `failed`), resumable phase, timestamps, and a sanitized error. No credentials/private keys in these columns. For an existing stack, add-on failure must not mark the parent setup `failed`. For new opted-in stacks, finish the add-on attempt before returning the baseline -stack to running. An add-on failure stays visible and retryable without failing -the already deployed four-component stack. +stack to running. An add-on failure marks the new setup failed at the Trust +Registry step, while an in-place enablement failure stays visible and retryable +without failing the already-running four-component stack. After persisting an add-on failure, remove its failed Jobs and their Pods, ConfigMaps, and temporary input Secrets. Keep the phase and sanitized error in @@ -410,8 +412,9 @@ Required acceptance coverage: VTA-offline case without a plaintext identity-cache copy. - Failures/restarts at credential handoff, DID publication, config update, and deployment resume without duplicate contexts/DIDs or a stuck stopped VTC/VTA. -- Add-on failure remains visible without invalidating the existing parent stack; - config restoration and incomplete-referral restoration are accurately reported. +- New-stack add-on failure stops completion and displays the Registry error; + in-place failure remains visible without invalidating the existing parent + stack. Config restoration and incomplete-referral restoration are accurately reported. - Upgrades use a single writer; full teardown cleans only this session's registry resources/secrets. Capacity and concurrent-operation guards cover add-on work. diff --git a/internal/setup/trust_registry.go b/internal/setup/trust_registry.go index c06b371..875060f 100644 --- a/internal/setup/trust_registry.go +++ b/internal/setup/trust_registry.go @@ -347,9 +347,17 @@ func (o *Orchestrator) TeardownTrustRegistryVault(ctx context.Context, userID, s } func (o *Orchestrator) finishRegistrySetup(ctx context.Context, s *model.SetupSession) bool { - if !s.TrustRegistryEnabled || s.TrustRegistryState == "active" || s.TrustRegistryState == "failed" { + if !s.TrustRegistryEnabled || s.TrustRegistryState == "active" { return true } + if s.TrustRegistryState == "failed" { + message := s.TrustRegistryError + if message == "" { + message = "Trust Registry setup failed" + } + o.markFailedAt(s.ID, "step_trust_registry", message) + return false + } o.fsSetStatus(s.ID, "step_trust_registry") if err := o.EnableTrustRegistry(ctx, s); err != nil { if ctx.Err() != nil { @@ -357,11 +365,15 @@ func (o *Orchestrator) finishRegistrySetup(ctx context.Context, s *model.SetupSe } if saveErr := o.db.Model(s).Updates(map[string]any{"trust_registry_state": "failed", "trust_registry_error": err.Error()}).Error; saveErr != nil { log.Printf("[orchestrator] registry failure for session %d could not be recorded; retaining Job", s.ID) + o.markFailedAt(s.ID, "step_trust_registry", "Trust Registry setup failed and its error could not be saved") return false } + s.TrustRegistryState, s.TrustRegistryError = "failed", err.Error() if cleanupErr := o.CleanupFailedRegistryJobs(s); cleanupErr != nil { log.Printf("[orchestrator] failed registry Job cleanup for session %d: %v", s.ID, cleanupErr) } + o.markFailedAt(s.ID, "step_trust_registry", err.Error()) + return false } return true } diff --git a/internal/setup/trust_registry_setup_test.go b/internal/setup/trust_registry_setup_test.go index 888b6be..dbba5dc 100644 --- a/internal/setup/trust_registry_setup_test.go +++ b/internal/setup/trust_registry_setup_test.go @@ -209,8 +209,14 @@ func TestHeadlessRegistryProvisioningAndRecovery(t *testing.T) { t.Fatal(err) } if fails { - if stored.TrustRegistryState != "failed" || !strings.Contains(stored.TrustRegistryError, "headless provisioning failed") || stored.Status != s.Status || !o.finishRegistrySetup(context.Background(), &s) { - t.Fatal("Registry failure was not isolated and persisted for the VTC to continue") + if stored.TrustRegistryState != "failed" || !strings.Contains(stored.TrustRegistryError, "headless provisioning failed") || stored.Status != s.Status || o.finishRegistrySetup(context.Background(), &s) { + t.Fatal("Registry failure was not persisted or did not stop session completion") + } + if err := db.First(&stored, s.ID).Error; err != nil { + t.Fatal(err) + } + if stored.Status != "failed" || stored.FailedStage != "step_trust_registry" || !strings.Contains(stored.ErrorMsg, "headless provisioning failed") { + t.Fatal("Registry failure did not surface as a failed Trust Registry setup step") } return }