From 40aa62bebde4547a866be0528d524ba7c0aa8e9c Mon Sep 17 00:00:00 2001 From: success-OG Date: Tue, 30 Jun 2026 00:16:26 +0100 Subject: [PATCH] feat(backend): API snapshots, error boundary, governance export, idempotency retention --- backend/README.md | 20 ++ backend/openapi.json | 79 ++++++- backend/package-lock.json | 25 ++- .../get-_api_v1_transactions.json | 110 ++++++++++ .../get-_api_v1_vault_summary.json | 24 +++ backend/scripts/check-schema-snapshots.ts | 9 + .../governanceSnapshotExport.test.ts | 75 +++++++ .../__tests__/idempotencyRetention.test.ts | 54 +++++ backend/src/__tests__/issues711.test.ts | 42 +++- backend/src/apiContractSnapshots.ts | 60 ++++++ backend/src/governanceSnapshotExport.ts | 203 ++++++++++++++++++ backend/src/idempotency.ts | 56 +++++ backend/src/idempotencyRetention.ts | 97 +++++++++ backend/src/index.ts | 110 ++++++++++ backend/src/listEndpoints.ts | 5 + backend/src/swagger.ts | 20 +- 16 files changed, 965 insertions(+), 24 deletions(-) create mode 100644 backend/schema-snapshots/get-_api_v1_transactions.json create mode 100644 backend/schema-snapshots/get-_api_v1_vault_summary.json create mode 100644 backend/src/__tests__/governanceSnapshotExport.test.ts create mode 100644 backend/src/__tests__/idempotencyRetention.test.ts create mode 100644 backend/src/governanceSnapshotExport.ts create mode 100644 backend/src/idempotencyRetention.ts diff --git a/backend/README.md b/backend/README.md index 5b7b97bf..05e42b30 100644 --- a/backend/README.md +++ b/backend/README.md @@ -226,6 +226,26 @@ npm test -- --watch npm test -- --coverage ``` +### API Contract Schema Snapshots (Issue #711) + +Committed JSON snapshots under `schema-snapshots/` describe the response shape of critical public endpoints. CI fails when a required field is removed or changes type. + +**Guarded endpoints:** `GET /health`, `GET /ready`, `GET /api/v1/vault/summary`, `GET /api/v1/transactions` + +```bash +# Verify snapshots are backward-compatible (CI) +npm run snapshots:check + +# Regenerate after an intentional breaking API change +npm run snapshots:write +``` + +When bumping snapshots intentionally: + +1. Update the Zod schema in `src/apiContractSnapshots.ts` +2. Run `npm run snapshots:write` and commit `schema-snapshots/*.json` +3. Align OpenAPI annotations and run `npm run generate:openapi` + ## Issues Addressed ### Issue #145: Rate Limiting diff --git a/backend/openapi.json b/backend/openapi.json index 49241c08..0cb381ca 100644 --- a/backend/openapi.json +++ b/backend/openapi.json @@ -35,20 +35,28 @@ "count": { "type": "integer" }, - "total": { + "limit": { "type": "integer" }, + "total": { + "type": "integer", + "nullable": true + }, "nextCursor": { - "type": "string" + "type": "string", + "nullable": true }, "prevCursor": { - "type": "string" + "type": "string", + "nullable": true }, "currentPage": { - "type": "integer" + "type": "integer", + "nullable": true }, "totalPages": { - "type": "integer" + "type": "integer", + "nullable": true }, "hasNextPage": { "type": "boolean" @@ -58,6 +66,24 @@ } } }, + "VaultSummary": { + "type": "object", + "properties": { + "totalAssets": { + "type": "number" + }, + "totalShares": { + "type": "number" + }, + "apy": { + "type": "number" + }, + "timestamp": { + "type": "string", + "format": "date-time" + } + } + }, "Error": { "type": "object", "properties": { @@ -74,6 +100,16 @@ }, "Transaction": { "type": "object", + "required": [ + "id", + "type", + "status", + "amount", + "asset", + "timestamp", + "transactionHash", + "walletAddress" + ], "properties": { "id": { "type": "string" @@ -85,6 +121,14 @@ "withdrawal" ] }, + "status": { + "type": "string", + "enum": [ + "pending", + "completed", + "failed" + ] + }, "amount": { "type": "string" }, @@ -341,6 +385,10 @@ }, "pagination": { "$ref": "#/components/schemas/PaginationMeta" + }, + "timestamp": { + "type": "string", + "format": "date-time" } } } @@ -489,6 +537,27 @@ } } } + }, + "/vault/summary": { + "get": { + "summary": "Vault summary", + "description": "Returns high-level vault metrics including total assets, shares, and APY.", + "tags": [ + "Vault" + ], + "responses": { + "200": { + "description": "Vault summary metrics", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VaultSummary" + } + } + } + } + } + } } }, "tags": [] diff --git a/backend/package-lock.json b/backend/package-lock.json index 66ac98e7..0b892b09 100644 --- a/backend/package-lock.json +++ b/backend/package-lock.json @@ -34,7 +34,7 @@ "rate-limit-redis": "^4.3.1", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", - "zod": "^3.23.8" + "zod": "^4.3.6" }, "devDependencies": { "@types/cors": "^2.8.19", @@ -57,6 +57,17 @@ "typescript": "^5.1.0" } }, + "../packages/api-schemas": { + "name": "@yieldvault/api-schemas", + "version": "1.0.0", + "dependencies": { + "zod": "^4.3.6" + }, + "devDependencies": { + "typescript": "~5.9.3", + "vitest": "^4.1.5" + } + }, "node_modules/@apidevtools/json-schema-ref-parser": { "version": "14.0.1", "resolved": "https://registry.npmjs.org/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-14.0.1.tgz", @@ -6035,7 +6046,6 @@ "version": "2.3.3", "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, "hasInstallScript": true, "license": "MIT", "optional": true, @@ -9949,17 +9959,6 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } - }, - "../packages/api-schemas": { - "name": "@yieldvault/api-schemas", - "version": "1.0.0", - "dependencies": { - "zod": "^4.3.6" - }, - "devDependencies": { - "typescript": "~5.9.3", - "vitest": "^4.1.5" - } } } } diff --git a/backend/schema-snapshots/get-_api_v1_transactions.json b/backend/schema-snapshots/get-_api_v1_transactions.json new file mode 100644 index 00000000..21034640 --- /dev/null +++ b/backend/schema-snapshots/get-_api_v1_transactions.json @@ -0,0 +1,110 @@ +{ + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "type": { + "type": "string", + "enum": [ + "deposit", + "withdrawal" + ] + }, + "status": { + "type": "string", + "enum": [ + "pending", + "completed", + "failed" + ] + }, + "amount": { + "type": "string" + }, + "asset": { + "type": "string" + }, + "timestamp": { + "type": "string" + }, + "transactionHash": { + "type": "string" + }, + "walletAddress": { + "type": "string" + } + }, + "required": [ + "id", + "type", + "status", + "amount", + "asset", + "timestamp", + "transactionHash", + "walletAddress" + ], + "additionalProperties": false + } + }, + "pagination": { + "type": "object", + "properties": { + "count": { + "type": "number" + }, + "limit": { + "type": "number" + }, + "total": { + "type": "number" + }, + "nextCursor": { + "type": "string" + }, + "prevCursor": { + "type": "string" + }, + "currentPage": { + "type": "number" + }, + "totalPages": { + "type": "number" + }, + "hasNextPage": { + "type": "boolean" + }, + "hasPrevPage": { + "type": "boolean" + } + }, + "required": [ + "count", + "limit", + "total", + "nextCursor", + "prevCursor", + "currentPage", + "totalPages", + "hasNextPage", + "hasPrevPage" + ], + "additionalProperties": false + }, + "timestamp": { + "type": "string" + } + }, + "required": [ + "data", + "pagination", + "timestamp" + ], + "additionalProperties": false +} diff --git a/backend/schema-snapshots/get-_api_v1_vault_summary.json b/backend/schema-snapshots/get-_api_v1_vault_summary.json new file mode 100644 index 00000000..090a1be9 --- /dev/null +++ b/backend/schema-snapshots/get-_api_v1_vault_summary.json @@ -0,0 +1,24 @@ +{ + "type": "object", + "properties": { + "totalAssets": { + "type": "number" + }, + "totalShares": { + "type": "number" + }, + "apy": { + "type": "number" + }, + "timestamp": { + "type": "string" + } + }, + "required": [ + "totalAssets", + "totalShares", + "apy", + "timestamp" + ], + "additionalProperties": false +} diff --git a/backend/scripts/check-schema-snapshots.ts b/backend/scripts/check-schema-snapshots.ts index 365ae112..3a62ea84 100644 --- a/backend/scripts/check-schema-snapshots.ts +++ b/backend/scripts/check-schema-snapshots.ts @@ -2,9 +2,18 @@ /** * Verify API contract schema snapshots remain backward-compatible (Issue #711). * + * Snapshots guard high-traffic public endpoints (health, readiness, vault summary, + * and transaction list) against silent response-shape regressions. + * * Usage: * tsx scripts/check-schema-snapshots.ts # fail on breaking changes * tsx scripts/check-schema-snapshots.ts --write # regenerate snapshot files + * + * Intentional breaking changes: + * 1. Update the Zod schema in src/apiContractSnapshots.ts + * 2. Run: npm run snapshots:write + * 3. Commit the updated files under schema-snapshots/ + * 4. Align OpenAPI annotations and run: npm run generate:openapi */ import { diff --git a/backend/src/__tests__/governanceSnapshotExport.test.ts b/backend/src/__tests__/governanceSnapshotExport.test.ts new file mode 100644 index 00000000..0afcf958 --- /dev/null +++ b/backend/src/__tests__/governanceSnapshotExport.test.ts @@ -0,0 +1,75 @@ +import { + exportGovernanceSnapshots, + listGovernanceSnapshots, +} from '../governanceSnapshotExport'; + +jest.mock('../prismaClient', () => ({ + getPrismaClient: () => ({ + reconciliationSnapshot: { + findMany: jest.fn().mockResolvedValue([ + { + id: 'rec-1', + generatedAt: new Date('2026-01-01T00:00:00Z'), + traceId: 'trace-1', + status: 'CLEAN', + windowFrom: new Date('2025-12-31T00:00:00Z'), + windowTo: new Date('2026-01-01T00:00:00Z'), + summaryJson: JSON.stringify({ counts: { drifted: 0 } }), + driftCount: 0, + }, + ]), + }, + exportManifest: { + findMany: jest.fn().mockResolvedValue([]), + }, + }), +})); + +jest.mock('../adminConfigChangeAudit', () => ({ + listAdminConfigChanges: jest.fn().mockResolvedValue([ + { + id: 'cfg-1', + configType: 'feature-flag', + action: 'update', + actor: 'admin@test', + preChangeSnapshot: { enabled: false }, + postChangeSnapshot: { enabled: true }, + metadata: {}, + createdAt: '2026-01-02T00:00:00Z', + }, + ]), +})); + +jest.mock('../exportManifest', () => ({ + createExportManifest: jest.fn().mockResolvedValue({ + id: 'exp-test', + reportType: 'governance-snapshots', + checksum: 'abc', + rowCount: 2, + }), +})); + +jest.mock('../middleware/structuredLogging', () => ({ + logger: { log: jest.fn(), configure: jest.fn() }, +})); + +describe('governanceSnapshotExport', () => { + it('lists reconciliation and config-change snapshots', async () => { + const result = await listGovernanceSnapshots({ limit: 10 }); + + expect(result.total).toBeGreaterThanOrEqual(2); + expect(result.data.some((row) => row.type === 'reconciliation')).toBe(true); + expect(result.data.some((row) => row.type === 'config-change')).toBe(true); + }); + + it('exports governance snapshots with manifest', async () => { + const result = await exportGovernanceSnapshots({ + requester: 'admin@test', + types: ['reconciliation', 'config-change'], + limit: 10, + }); + + expect(result.rows.length).toBeGreaterThanOrEqual(2); + expect(result.manifest.reportType).toBe('governance-snapshots'); + }); +}); diff --git a/backend/src/__tests__/idempotencyRetention.test.ts b/backend/src/__tests__/idempotencyRetention.test.ts new file mode 100644 index 00000000..6b06d591 --- /dev/null +++ b/backend/src/__tests__/idempotencyRetention.test.ts @@ -0,0 +1,54 @@ +import { IdempotencyStore } from '../idempotency'; +import { + getIdempotencyRetentionMetrics, + pruneStaleIdempotencyRecords, + resetIdempotencyRetentionStateForTests, +} from '../idempotencyRetention'; + +jest.mock('../rateLimiter', () => ({ + redisClientManager: { + isReady: () => false, + getClient: () => null, + }, +})); + +describe('idempotencyRetention', () => { + beforeEach(() => { + resetIdempotencyRetentionStateForTests(); + process.env.IDEMPOTENCY_KEY_TTL_MS = '1000'; + process.env.IDEMPOTENCY_RETENTION_ENABLED = 'true'; + }); + + it('reports retention policy and store metrics', () => { + const metrics = getIdempotencyRetentionMetrics(); + expect(metrics.policy.retentionMs).toBe(1000); + expect(metrics.storeMetrics).toBeDefined(); + }); + + it('prunes stale local idempotency keys', async () => { + const store = new IdempotencyStore(1000); + const staleCreatedAt = new Date(Date.now() - 10_000).toISOString(); + + (store as any).localCache.set('stale-key', { + statusCode: 200, + body: { ok: true }, + fingerprint: 'fp', + metadata: { + createdAt: staleCreatedAt, + lastAccessedAt: staleCreatedAt, + replayCount: 0, + status: 'completed', + }, + }); + + const result = await store.pruneStaleKeys(1000, false); + expect(result.localPruned).toBe(1); + expect(result.pruned).toBe(1); + }); + + it('supports dry-run retention sweeps', async () => { + const result = await pruneStaleIdempotencyRecords(true); + expect(result.dryRun).toBe(true); + expect(result.pruned).toBeGreaterThanOrEqual(0); + }); +}); diff --git a/backend/src/__tests__/issues711.test.ts b/backend/src/__tests__/issues711.test.ts index c2fd4969..2700516a 100644 --- a/backend/src/__tests__/issues711.test.ts +++ b/backend/src/__tests__/issues711.test.ts @@ -24,7 +24,7 @@ describe('#711 API contract schema snapshots', () => { }); it('defines snapshots for all critical public endpoints', () => { - expect(CRITICAL_ENDPOINTS.length).toBeGreaterThanOrEqual(2); + expect(CRITICAL_ENDPOINTS.length).toBeGreaterThanOrEqual(4); for (const endpoint of CRITICAL_ENDPOINTS) { const snapshot = loadSnapshot(endpoint); expect(snapshot).not.toBeNull(); @@ -80,6 +80,46 @@ describe('#711 API contract schema snapshots', () => { expect(result.success).toBe(false); }); + it('validates a conforming vault summary payload', () => { + const result = validateResponseAgainstSchema('GET /api/v1/vault/summary', { + totalAssets: 1000, + totalShares: 500, + apy: 8.5, + timestamp: new Date().toISOString(), + }); + + expect(result.success).toBe(true); + }); + + it('validates a conforming transactions list payload', () => { + const result = validateResponseAgainstSchema('GET /api/v1/transactions', { + data: [{ + id: 'tx-1', + type: 'deposit', + status: 'completed', + amount: '100', + asset: 'USDC', + timestamp: new Date().toISOString(), + transactionHash: 'abc123', + walletAddress: 'GABC123', + }], + pagination: { + count: 1, + limit: 20, + total: 1, + nextCursor: null, + prevCursor: null, + currentPage: 1, + totalPages: 1, + hasNextPage: false, + hasPrevPage: false, + }, + timestamp: new Date().toISOString(), + }); + + expect(result.success).toBe(true); + }); + it('writes snapshot files under schema-snapshots/', () => { for (const endpoint of CRITICAL_ENDPOINTS) { const filename = endpoint.replace(/\s+/g, '-').replace(/\//g, '_').toLowerCase() + '.json'; diff --git a/backend/src/apiContractSnapshots.ts b/backend/src/apiContractSnapshots.ts index fcb31eaa..8918cc5e 100644 --- a/backend/src/apiContractSnapshots.ts +++ b/backend/src/apiContractSnapshots.ts @@ -15,6 +15,8 @@ export const SNAPSHOT_DIR = path.join(__dirname, '..', 'schema-snapshots'); export const CRITICAL_ENDPOINTS = [ 'GET /health', 'GET /ready', + 'GET /api/v1/vault/summary', + 'GET /api/v1/transactions', ] as const; export type CriticalEndpoint = (typeof CRITICAL_ENDPOINTS)[number]; @@ -57,9 +59,55 @@ export const ReadyResponseSchema = z }) .strict(); +export const VaultSummaryResponseSchema = z + .object({ + totalAssets: z.number(), + totalShares: z.number(), + apy: z.number(), + timestamp: z.string(), + }) + .strict(); + +export const TransactionItemSchema = z + .object({ + id: z.string(), + type: z.enum(['deposit', 'withdrawal']), + status: z.enum(['pending', 'completed', 'failed']), + amount: z.string(), + asset: z.string(), + timestamp: z.string(), + transactionHash: z.string(), + walletAddress: z.string(), + }) + .strict(); + +export const PaginationMetaSchema = z + .object({ + count: z.number(), + limit: z.number(), + total: z.number().nullable(), + nextCursor: z.string().nullable(), + prevCursor: z.string().nullable(), + currentPage: z.number().nullable(), + totalPages: z.number().nullable(), + hasNextPage: z.boolean(), + hasPrevPage: z.boolean(), + }) + .strict(); + +export const TransactionsListResponseSchema = z + .object({ + data: z.array(TransactionItemSchema), + pagination: PaginationMetaSchema, + timestamp: z.string(), + }) + .strict(); + export const ENDPOINT_SCHEMAS: Record = { 'GET /health': HealthResponseSchema, 'GET /ready': ReadyResponseSchema, + 'GET /api/v1/vault/summary': VaultSummaryResponseSchema, + 'GET /api/v1/transactions': TransactionsListResponseSchema, }; export function endpointToFilename(endpoint: CriticalEndpoint): string { @@ -125,6 +173,10 @@ export function zodToJsonShape(schema: z.ZodTypeAny): JsonSchemaShape { return { type: 'boolean' }; } + if (schema instanceof z.ZodArray) { + return { type: 'array', items: zodToJsonShape(schema.element as z.ZodTypeAny) }; + } + return { type: 'unknown' }; } @@ -188,6 +240,14 @@ export function diffSchemaShapes( } } + if (baseline.type === 'array' && current.type === 'array') { + if (baseline.items && current.items) { + issues.push(...diffSchemaShapes(baseline.items, current.items, `${prefix}[]`)); + } else if (baseline.items && !current.items) { + issues.push({ path: `${prefix}[]`, message: 'array item schema removed' }); + } + } + if (baseline.enum && current.enum) { const removed = baseline.enum.filter((value) => !current.enum?.includes(value)); if (removed.length > 0) { diff --git a/backend/src/governanceSnapshotExport.ts b/backend/src/governanceSnapshotExport.ts new file mode 100644 index 00000000..8b749408 --- /dev/null +++ b/backend/src/governanceSnapshotExport.ts @@ -0,0 +1,203 @@ +/** + * Historical governance snapshot export for reporting (Issue #715). + * + * Aggregates persisted reconciliation snapshots, admin config changes, and + * export manifests into a unified query/export surface for governance reporting. + */ + +import { getPrismaClient } from './prismaClient'; +import { listAdminConfigChanges } from './adminConfigChangeAudit'; +import { createExportManifest } from './exportManifest'; +import { logger } from './middleware/structuredLogging'; + +export type GovernanceSnapshotType = 'reconciliation' | 'config-change' | 'export-manifest'; + +export interface GovernanceSnapshotRecord { + id: string; + type: GovernanceSnapshotType; + generatedAt: string; + status?: string; + summary: Record; +} + +export interface ListGovernanceSnapshotsFilters { + type?: GovernanceSnapshotType; + types?: GovernanceSnapshotType[]; + start?: string; + end?: string; + limit?: number; + offset?: number; +} + +export interface GovernanceSnapshotExportInput { + requester: string; + types?: GovernanceSnapshotType[]; + start?: string; + end?: string; + limit?: number; +} + +function parseDateRange(start?: string, end?: string): { gte?: Date; lte?: Date } { + const range: { gte?: Date; lte?: Date } = {}; + if (start) range.gte = new Date(start); + if (end) range.lte = new Date(end); + return range; +} + +async function listReconciliationSnapshots( + filters: ListGovernanceSnapshotsFilters, +): Promise { + try { + const prisma = getPrismaClient(); + const generatedAt = parseDateRange(filters.start, filters.end); + const where = Object.keys(generatedAt).length > 0 ? { generatedAt } : {}; + + const rows = await prisma.reconciliationSnapshot.findMany({ + where, + orderBy: { generatedAt: 'desc' }, + take: filters.limit ?? 100, + skip: filters.offset ?? 0, + }); + + return rows.map((row) => { + let summary: Record = {}; + try { + summary = JSON.parse(row.summaryJson) as Record; + } catch { + summary = { driftCount: row.driftCount }; + } + + return { + id: row.id, + type: 'reconciliation' as const, + generatedAt: row.generatedAt.toISOString(), + status: row.status, + summary: { + windowFrom: row.windowFrom.toISOString(), + windowTo: row.windowTo.toISOString(), + driftCount: row.driftCount, + traceId: row.traceId ?? undefined, + ...summary, + }, + }; + }); + } catch (error) { + logger.log('warn', 'Failed to list reconciliation snapshots for governance export', { + error: error instanceof Error ? error.message : String(error), + }); + return []; + } +} + +async function listConfigChangeSnapshots( + filters: ListGovernanceSnapshotsFilters, +): Promise { + const records = await listAdminConfigChanges({ + start: filters.start, + end: filters.end, + limit: filters.limit ?? 100, + }); + + return records.map((record) => ({ + id: record.id, + type: 'config-change' as const, + generatedAt: record.createdAt, + status: record.action, + summary: { + configType: record.configType, + actor: record.actor, + preChangeSnapshot: record.preChangeSnapshot, + postChangeSnapshot: record.postChangeSnapshot, + metadata: record.metadata, + }, + })); +} + +async function listExportManifestSnapshots( + filters: ListGovernanceSnapshotsFilters, +): Promise { + try { + const prisma = getPrismaClient(); + const generatedAt = parseDateRange(filters.start, filters.end); + const where = Object.keys(generatedAt).length > 0 ? { generatedAt } : {}; + + const rows = await prisma.exportManifest.findMany({ + where, + orderBy: { generatedAt: 'desc' }, + take: filters.limit ?? 100, + skip: filters.offset ?? 0, + }); + + return rows.map((row) => ({ + id: row.id, + type: 'export-manifest' as const, + generatedAt: row.generatedAt.toISOString(), + status: row.reportType, + summary: { + requester: row.requester, + reportType: row.reportType, + checksum: row.checksum, + rowCount: row.rowCount, + fileName: row.fileName, + }, + })); + } catch (error) { + logger.log('warn', 'Failed to list export manifests for governance export', { + error: error instanceof Error ? error.message : String(error), + }); + return []; + } +} + +export async function listGovernanceSnapshots( + filters: ListGovernanceSnapshotsFilters = {}, +): Promise<{ data: GovernanceSnapshotRecord[]; total: number }> { + const types: GovernanceSnapshotType[] = filters.types + ?? (filters.type ? [filters.type] : ['reconciliation', 'config-change', 'export-manifest']); + + const results: GovernanceSnapshotRecord[] = []; + + if (types.includes('reconciliation')) { + results.push(...await listReconciliationSnapshots(filters)); + } + if (types.includes('config-change')) { + results.push(...await listConfigChangeSnapshots(filters)); + } + if (types.includes('export-manifest')) { + results.push(...await listExportManifestSnapshots(filters)); + } + + results.sort((a, b) => b.generatedAt.localeCompare(a.generatedAt)); + + const offset = filters.offset ?? 0; + const limit = filters.limit ?? 100; + const data = results.slice(offset, offset + limit); + + return { data, total: results.length }; +} + +export async function exportGovernanceSnapshots( + input: GovernanceSnapshotExportInput, +): Promise<{ manifest: Awaited>; rows: GovernanceSnapshotRecord[] }> { + const types = input.types ?? ['reconciliation', 'config-change', 'export-manifest']; + const { data: rows } = await listGovernanceSnapshots({ + types, + start: input.start, + end: input.end, + limit: input.limit ?? 500, + }); + + const manifest = await createExportManifest({ + requester: input.requester, + reportType: 'governance-snapshots', + filters: { + types, + start: input.start, + end: input.end, + limit: input.limit ?? 500, + }, + rows, + }); + + return { manifest, rows }; +} diff --git a/backend/src/idempotency.ts b/backend/src/idempotency.ts index fe9f43e5..5e7a0aa9 100644 --- a/backend/src/idempotency.ts +++ b/backend/src/idempotency.ts @@ -261,6 +261,62 @@ export class IdempotencyStore { pendingKeys: this.pendingResponses.size, }; } + + // ─── Retention cleanup ───────────────────────────────────────────────────── + + async pruneStaleKeys( + retentionMs: number, + dryRun = false, + ): Promise<{ pruned: number; localPruned: number; redisPruned: number }> { + const cutoff = Date.now() - retentionMs; + let localPruned = 0; + let redisPruned = 0; + + for (const key of this.localCache.keys()) { + const entry = this.localCache.get>(key); + if (!entry) continue; + const createdAt = Date.parse(entry.metadata.createdAt); + if (Number.isNaN(createdAt) || createdAt >= cutoff) continue; + if (!dryRun) { + this.localCache.del(key); + this._evictions++; + } + localPruned++; + } + + const r = this.redis; + if (r) { + let cursor = '0'; + do { + const [nextCursor, keys] = await r.scan(cursor, 'MATCH', `${REDIS_PREFIX}*`, 'COUNT', 100); + cursor = nextCursor; + for (const redisKey of keys) { + try { + const raw = await r.get(redisKey); + if (!raw) continue; + const entry = JSON.parse(raw) as StoredResponse; + const createdAt = Date.parse(entry.metadata?.createdAt ?? ''); + const ttl = await r.ttl(redisKey); + const isStale = (!Number.isNaN(createdAt) && createdAt < cutoff) || ttl === 0; + if (!isStale) continue; + if (!dryRun) { + await r.del(redisKey); + this._evictions++; + } + redisPruned++; + } catch { + if (!dryRun) { + await r.del(redisKey); + this._evictions++; + } + redisPruned++; + } + } + } while (cursor !== '0'); + } + + return { pruned: localPruned + redisPruned, localPruned, redisPruned }; + } } // ─── Singleton ──────────────────────────────────────────────────────────────── diff --git a/backend/src/idempotencyRetention.ts b/backend/src/idempotencyRetention.ts new file mode 100644 index 00000000..d89fa9f9 --- /dev/null +++ b/backend/src/idempotencyRetention.ts @@ -0,0 +1,97 @@ +/** + * Policy-driven retention cleanup for stale idempotency records (Issue #716). + * + * Complements Redis TTL expiry with scheduled sweeps of local cache and orphaned + * Redis keys, reporting metrics for governance observability. + */ + +import { idempotencyStore } from './idempotency'; +import { logger } from './middleware/structuredLogging'; + +export interface IdempotencyRetentionPolicy { + retentionMs: number; + sweepIntervalMs: number; + enabled: boolean; +} + +export interface IdempotencyRetentionMetrics { + lastSweepAt: string | null; + lastSweepDurationMs: number | null; + totalPruned: number; + lastPrunedCount: number; + policy: IdempotencyRetentionPolicy; + storeMetrics: ReturnType; +} + +const retentionState = { + lastSweepAt: null as string | null, + lastSweepDurationMs: null as number | null, + totalPruned: 0, + lastPrunedCount: 0, +}; + +export function getIdempotencyRetentionPolicy(): IdempotencyRetentionPolicy { + return { + retentionMs: parseInt(process.env.IDEMPOTENCY_KEY_TTL_MS || '86400000', 10), + sweepIntervalMs: parseInt(process.env.IDEMPOTENCY_RETENTION_SWEEP_MS || '3600000', 10), + enabled: process.env.IDEMPOTENCY_RETENTION_ENABLED !== 'false', + }; +} + +export function getIdempotencyRetentionMetrics(): IdempotencyRetentionMetrics { + return { + ...retentionState, + policy: getIdempotencyRetentionPolicy(), + storeMetrics: idempotencyStore.getMetrics(), + }; +} + +export async function pruneStaleIdempotencyRecords( + dryRun = false, +): Promise<{ pruned: number; dryRun: boolean }> { + const startedAt = Date.now(); + const policy = getIdempotencyRetentionPolicy(); + const result = await idempotencyStore.pruneStaleKeys(policy.retentionMs, dryRun); + + if (!dryRun) { + retentionState.totalPruned += result.pruned; + retentionState.lastPrunedCount = result.pruned; + retentionState.lastSweepAt = new Date().toISOString(); + retentionState.lastSweepDurationMs = Date.now() - startedAt; + + if (result.pruned > 0) { + logger.log('info', 'Idempotency retention sweep completed', { + pruned: result.pruned, + retentionMs: policy.retentionMs, + durationMs: retentionState.lastSweepDurationMs, + }); + } + } + + return { pruned: result.pruned, dryRun }; +} + +export function startIdempotencyRetentionScheduler(): () => void { + const policy = getIdempotencyRetentionPolicy(); + if (!policy.enabled) { + return () => undefined; + } + + const intervalMs = Math.max(60_000, policy.sweepIntervalMs); + const timer = setInterval(() => { + void pruneStaleIdempotencyRecords(false); + }, intervalMs); + + if (typeof timer.unref === 'function') { + timer.unref(); + } + + return () => clearInterval(timer); +} + +export function resetIdempotencyRetentionStateForTests(): void { + retentionState.lastSweepAt = null; + retentionState.lastSweepDurationMs = null; + retentionState.totalPruned = 0; + retentionState.lastPrunedCount = 0; +} diff --git a/backend/src/index.ts b/backend/src/index.ts index 6683bb39..62f73a5e 100644 --- a/backend/src/index.ts +++ b/backend/src/index.ts @@ -170,6 +170,16 @@ import { automatedReconciliationSummaryHandler, } from './reconciliationReport'; import { diagnosticsBundleHandler } from './diagnosticsBundle'; +import { errorBoundaryMiddleware } from './middleware/errorBoundary'; +import { + exportGovernanceSnapshots, + listGovernanceSnapshots, +} from './governanceSnapshotExport'; +import { + getIdempotencyRetentionMetrics, + pruneStaleIdempotencyRecords, + startIdempotencyRetentionScheduler, +} from './idempotencyRetention'; declare global { namespace Express { @@ -858,6 +868,21 @@ app.get('/api/v1/vault/transactions/export', handleTransactionExport); // ─── Versioned vault summary/metrics/apy endpoints ─────────────────────── +/** + * @openapi + * /vault/summary: + * get: + * summary: Vault summary + * description: Returns high-level vault metrics including total assets, shares, and APY. + * tags: [Vault] + * responses: + * 200: + * description: Vault summary metrics + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/VaultSummary' + */ /** * GET /api/v1/vault/summary – read-only summary; relaxed rate limit. */ @@ -3150,6 +3175,61 @@ app.get('/admin/transactions/backfill/:jobId', validateApiKey, async (req: Reque }); }); +/** + * GET /admin/governance/snapshots - list historical governance snapshots + */ +app.get('/admin/governance/snapshots', validateApiKey, async (req: Request, res: Response) => { + const limit = parseLimited(req.query.limit, 50, 1, 500); + const offset = parseLimited(req.query.offset, 0, 0, 10_000); + const type = typeof req.query.type === 'string' ? req.query.type : undefined; + const start = typeof req.query.start === 'string' ? req.query.start : undefined; + const end = typeof req.query.end === 'string' ? req.query.end : undefined; + + const result = await listGovernanceSnapshots({ + type: type as 'reconciliation' | 'config-change' | 'export-manifest' | undefined, + start, + end, + limit, + offset, + }); + + res.status(200).json({ + data: result.data, + total: result.total, + limit, + offset, + timestamp: new Date().toISOString(), + }); +}); + +/** + * POST /admin/governance/snapshots/export - export historical governance snapshots + */ +app.post('/admin/governance/snapshots/export', validateApiKey, async (req: Request, res: Response) => { + const requester = resolveActingAdminAddress(req); + const types = Array.isArray(req.body?.types) + ? (req.body.types as string[]) + : undefined; + const start = typeof req.body?.start === 'string' ? req.body.start : undefined; + const end = typeof req.body?.end === 'string' ? req.body.end : undefined; + const limit = parseLimited(req.body?.limit, 500, 1, 5000); + + const { manifest, rows } = await exportGovernanceSnapshots({ + requester, + types: types as ('reconciliation' | 'config-change' | 'export-manifest')[] | undefined, + start, + end, + limit, + }); + + res.status(201).json({ + message: 'Governance snapshot export generated', + rowCount: rows.length, + manifest, + timestamp: new Date().toISOString(), + }); +}); + /** * POST /admin/reports/exports - generate a report export and immutable manifest record */ @@ -3347,6 +3427,31 @@ app.delete('/admin/idempotency/keys/:key', validateApiKey, (req: Request, res: R }); }); +/** + * GET /admin/idempotency/retention/metrics + * Returns idempotency retention policy and sweep metrics. + */ +app.get('/admin/idempotency/retention/metrics', validateApiKey, (_req: Request, res: Response) => { + res.status(200).json({ + metrics: getIdempotencyRetentionMetrics(), + timestamp: new Date().toISOString(), + }); +}); + +/** + * POST /admin/idempotency/retention/cleanup + * Runs a retention sweep for stale idempotency records. + */ +app.post('/admin/idempotency/retention/cleanup', validateApiKey, async (req: Request, res: Response) => { + const dryRun = isDryRunRequest(req); + const result = await pruneStaleIdempotencyRecords(dryRun); + res.status(200).json({ + ...result, + metrics: getIdempotencyRetentionMetrics(), + timestamp: new Date().toISOString(), + }); +}); + /** * DELETE /admin/idempotency/keys * Flushes the entire idempotency store. @@ -4165,6 +4270,11 @@ if (process.env.NODE_ENV !== 'test') { stopLedgerReconciliationScheduler(); }); + const stopIdempotencyRetentionScheduler = startIdempotencyRetentionScheduler(); + shutdownHandler.onShutdown(async () => { + stopIdempotencyRetentionScheduler(); + }); + // Register event polling service shutdown shutdownHandler.onShutdown(async () => { stopEventPollingService(); diff --git a/backend/src/listEndpoints.ts b/backend/src/listEndpoints.ts index 762e0de8..8e0f3971 100644 --- a/backend/src/listEndpoints.ts +++ b/backend/src/listEndpoints.ts @@ -50,9 +50,11 @@ const CACHE_TTL_MS = parseInt(process.env.CACHE_LIST_ENDPOINTS_TTL_MS || '30000' * schemas: * Transaction: * type: object + * required: [id, type, status, amount, asset, timestamp, transactionHash, walletAddress] * properties: * id: { type: string } * type: { type: string, enum: [deposit, withdrawal] } + * status: { type: string, enum: [pending, completed, failed] } * amount: { type: string } * asset: { type: string } * timestamp: { type: string, format: "date-time" } @@ -731,6 +733,9 @@ export function buildVaultHistoryResponse( * $ref: '#/components/schemas/Transaction' * pagination: * $ref: '#/components/schemas/PaginationMeta' + * timestamp: + * type: string + * format: date-time */ router.get('/transactions', cacheMiddleware({ ttl: CACHE_TTL_MS }), diff --git a/backend/src/swagger.ts b/backend/src/swagger.ts index 338de9c8..a372a06b 100644 --- a/backend/src/swagger.ts +++ b/backend/src/swagger.ts @@ -38,15 +38,25 @@ const options: swaggerJsdoc.Options = { type: 'object', properties: { count: { type: 'integer' }, - total: { type: 'integer' }, - nextCursor: { type: 'string' }, - prevCursor: { type: 'string' }, - currentPage: { type: 'integer' }, - totalPages: { type: 'integer' }, + limit: { type: 'integer' }, + total: { type: 'integer', nullable: true }, + nextCursor: { type: 'string', nullable: true }, + prevCursor: { type: 'string', nullable: true }, + currentPage: { type: 'integer', nullable: true }, + totalPages: { type: 'integer', nullable: true }, hasNextPage: { type: 'boolean' }, hasPrevPage: { type: 'boolean' }, }, }, + VaultSummary: { + type: 'object', + properties: { + totalAssets: { type: 'number' }, + totalShares: { type: 'number' }, + apy: { type: 'number' }, + timestamp: { type: 'string', format: 'date-time' }, + }, + }, Error: { type: 'object', properties: {