From bb4c939822cb2489e963c7501c7d267f9552c0f9 Mon Sep 17 00:00:00 2001 From: Momo Kornher Date: Tue, 6 Oct 2026 14:15:36 +0100 Subject: [PATCH] feat(npmjs): keep known versions in a DynamoDB table The follower kept the package versions it has processed in a single JSON file in S3. Every run loaded all of it and wrote all of it back, so only one worker could ever use it. Known versions now live in a DynamoDB table, keyed by package name and version. Writes are conditional, so the stored modified timestamp only moves forward. The table billing mode is configurable through the new knownVersionsTableBilling prop and defaults to on-demand. Existing deployments migrate automatically: when the migration marker object is missing, the first run copies the known versions file (or the legacy marker file) into the table and then writes the marker. The old file is left in place and no longer updated, so a rollback only re-stages versions modified since the migration. --- .gitattributes | 1 + .gitignore | 1 + .prettierignore | 1 + .projen/deps.json | 4 + .projen/files.json | 1 + .projen/tasks.json | 29 +- .projenrc.ts | 1 + API.md | 33 + docs/application-overview.md | 10 +- package.json | 3 + .../__snapshots__/construct-hub.test.ts.snap | 2331 +++++++++++++++-- .../__snapshots__/snapshot.test.ts.snap | 465 +++- src/__tests__/package-sources/npmjs.test.ts | 126 + .../known-versions-migration.lambda.test.ts | 126 + .../known-versions.lambda-shared.test.ts | 175 ++ .../npmjs/npm-js-follower.lambda.test.ts | 113 +- src/construct-hub.ts | 25 +- src/deep-link.ts | 5 + src/package-sources/npmjs.ts | 85 +- .../npmjs/constants.lambda-shared.ts | 19 +- .../npmjs/known-versions-migration.lambda.ts | 140 + .../npmjs/known-versions-migration.ts | 20 + .../npmjs/known-versions-table-migration.ts | 84 + .../npmjs/known-versions.lambda-shared.ts | 201 ++ .../npmjs/npm-js-follower.lambda.ts | 408 +-- .../npmjs/staging-bucket.lambda-shared.ts | 98 + yarn.lock | 70 + 27 files changed, 3984 insertions(+), 591 deletions(-) create mode 100644 src/__tests__/package-sources/npmjs.test.ts create mode 100644 src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts create mode 100644 src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts create mode 100644 src/package-sources/npmjs/known-versions-migration.lambda.ts create mode 100644 src/package-sources/npmjs/known-versions-migration.ts create mode 100644 src/package-sources/npmjs/known-versions-table-migration.ts create mode 100644 src/package-sources/npmjs/known-versions.lambda-shared.ts create mode 100644 src/package-sources/npmjs/staging-bucket.lambda-shared.ts diff --git a/.gitattributes b/.gitattributes index 562f4ad06..8def7a98f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -66,6 +66,7 @@ /src/overview-dashboard/sqs-dlq-stats-widget-function.ts linguist-generated /src/package-sources/codeartifact/code-artifact-forwarder.ts linguist-generated /src/package-sources/npmjs/canary/npmjs-package-canary.ts linguist-generated +/src/package-sources/npmjs/known-versions-migration.ts linguist-generated /src/package-sources/npmjs/npm-js-follower.ts linguist-generated /src/package-sources/npmjs/re-stage-package-version.ts linguist-generated /src/package-sources/npmjs/stage-and-notify.ts linguist-generated diff --git a/.gitignore b/.gitignore index 330f0b481..ec6ca9dda 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ lib/__tests__/devapp/cdk.out !/src/package-sources/npmjs/stage-and-notify.ts !/src/package-sources/npmjs/re-stage-package-version.ts !/src/package-sources/npmjs/npm-js-follower.ts +!/src/package-sources/npmjs/known-versions-migration.ts !/src/package-sources/npmjs/canary/npmjs-package-canary.ts !/src/package-sources/codeartifact/code-artifact-forwarder.ts !/src/overview-dashboard/sqs-dlq-stats-widget-function.ts diff --git a/.prettierignore b/.prettierignore index a050ba12f..c9902bb09 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,6 +2,7 @@ src/package-sources/npmjs/stage-and-notify.ts src/package-sources/npmjs/re-stage-package-version.ts src/package-sources/npmjs/npm-js-follower.ts +src/package-sources/npmjs/known-versions-migration.ts src/package-sources/npmjs/canary/npmjs-package-canary.ts src/package-sources/codeartifact/code-artifact-forwarder.ts src/overview-dashboard/sqs-dlq-stats-widget-function.ts diff --git a/.projen/deps.json b/.projen/deps.json index 61587305a..cfe664c0d 100644 --- a/.projen/deps.json +++ b/.projen/deps.json @@ -23,6 +23,10 @@ "name": "@aws-sdk/client-codeartifact", "type": "build" }, + { + "name": "@aws-sdk/client-dynamodb", + "type": "build" + }, { "name": "@aws-sdk/client-ecs", "type": "build" diff --git a/.projen/files.json b/.projen/files.json index 188e1349b..a8dae9bd6 100644 --- a/.projen/files.json +++ b/.projen/files.json @@ -55,6 +55,7 @@ "src/overview-dashboard/sqs-dlq-stats-widget-function.ts", "src/package-sources/codeartifact/code-artifact-forwarder.ts", "src/package-sources/npmjs/canary/npmjs-package-canary.ts", + "src/package-sources/npmjs/known-versions-migration.ts", "src/package-sources/npmjs/npm-js-follower.ts", "src/package-sources/npmjs/re-stage-package-version.ts", "src/package-sources/npmjs/stage-and-notify.ts", diff --git a/.projen/tasks.json b/.projen/tasks.json index fcb1eaec9..b5296b1f6 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -82,6 +82,9 @@ { "spawn": "bundle:npm-js-follower" }, + { + "spawn": "bundle:known-versions-migration" + }, { "spawn": "bundle:npmjs-package-canary" }, @@ -363,12 +366,30 @@ } ] }, + "bundle:known-versions-migration": { + "name": "bundle:known-versions-migration", + "description": "Create an AWS Lambda bundle from src/package-sources/npmjs/known-versions-migration.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/known-versions-migration.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/known-versions-migration.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json" + } + ] + }, + "bundle:known-versions-migration:watch": { + "name": "bundle:known-versions-migration:watch", + "description": "Continuously update an AWS Lambda bundle from src/package-sources/npmjs/known-versions-migration.lambda.ts", + "steps": [ + { + "exec": "esbuild --bundle src/package-sources/npmjs/known-versions-migration.lambda.ts --target=\"node22\" --platform=\"node\" --outfile=\"lib/package-sources/npmjs/known-versions-migration.lambda.bundle/index.js\" --external:aws-sdk --sourcemap --tsconfig=tsconfig.json --watch" + } + ] + }, "bundle:lambda:watch": { "name": "bundle:lambda:watch", "description": "Continuously bundle all AWS Lambda functions", "steps": [ { - "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" + "exec": "esbuild --bundle src/package-sources/npmjs/stage-and-notify.lambda.ts src/package-sources/npmjs/re-stage-package-version.lambda.ts src/package-sources/npmjs/npm-js-follower.lambda.ts src/package-sources/npmjs/known-versions-migration.lambda.ts src/package-sources/npmjs/canary/npmjs-package-canary.lambda.ts src/package-sources/codeartifact/code-artifact-forwarder.lambda.ts src/overview-dashboard/sqs-dlq-stats-widget-function.lambda.ts src/monitoring/http-get-function.lambda.ts src/monitored-certificate/certificate-monitor.lambda.ts src/backend/version-tracker/version-tracker.lambda.ts src/backend/release-notes/release-notes-trigger.lambda.ts src/backend/release-notes/get-messages-from-worker-queue.lambda.ts src/backend/release-notes/generate-release-notes.lambda.ts src/backend/package-stats/package-stats-processor.lambda.ts src/backend/package-stats/package-stats-chunker.lambda.ts src/backend/package-stats/package-stats-aggregator.lambda.ts src/backend/orchestration/redrive-state-machine.lambda.ts src/backend/orchestration/read-uninstallable-report.lambda.ts src/backend/orchestration/needs-catalog-update.lambda.ts src/backend/inventory/package-versions-table-widget-function.lambda.ts src/backend/inventory/canary.lambda.ts src/backend/ingestion/re-ingest.lambda.ts src/backend/ingestion/ingestion.lambda.ts src/backend/feed-builder/update-feed.lambda.ts src/backend/ecs-task-monitor/monitor.lambda.ts src/backend/deny-list/prune-queue-handler.lambda.ts src/backend/deny-list/prune-handler.lambda.ts src/backend/catalog-builder/catalog-builder.lambda.ts src/__tests__/backend/deny-list/mocks/prune-test.lambda.ts src/__tests__/backend/deny-list/mocks/client-test.lambda.ts src/__tests__/backend/deny-list/mocks/catalog-builder-mock.lambda.ts --target=\"node22 --platform=\"node\" --outbase=\"src\" --outdir=\"lib\" --entry-names=\"[dir]/[name].bundle/index\" --external:aws-sdk --sourcemap --watch --tsconfig=tsconfig.json" } ] }, @@ -843,6 +864,9 @@ { "spawn": "bundle:npm-js-follower" }, + { + "spawn": "bundle:known-versions-migration" + }, { "spawn": "bundle:npmjs-package-canary" }, @@ -1321,7 +1345,7 @@ "--peer", "--no-deprecated", "--dep=dev", - "--filter=@aws-sdk/client-cloudwatch,@aws-sdk/client-codeartifact,@aws-sdk/client-ecs,@aws-sdk/client-lambda,@aws-sdk/client-s3,@aws-sdk/client-sfn,@aws-sdk/client-sqs,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-retry,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,jsii-rosetta,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" + "--filter=@aws-sdk/client-cloudwatch,@aws-sdk/client-codeartifact,@aws-sdk/client-dynamodb,@aws-sdk/client-ecs,@aws-sdk/client-lambda,@aws-sdk/client-s3,@aws-sdk/client-sfn,@aws-sdk/client-sqs,@jsii/spec,@octokit/rest,@smithy/types,@smithy/util-retry,@smithy/util-stream,@types/aws-lambda,@types/fs-extra,@types/jest,@types/markdown-it,@types/markdown-it-emoji,@types/semver,@types/tough-cookie,@types/uuid,async-sema,aws-embedded-metrics,aws-sdk-client-mock,aws-sdk-client-mock-jest,aws-xray-sdk-core,case,cdk-dia,cdk-watchful,chalk,construct-hub-webapp,dotenv,esbuild,eslint-config-prettier,eslint-import-resolver-typescript,eslint-plugin-import,eslint-plugin-prettier,feed,fs-extra,glob,got,jest,jsii-diff,jsii-pacmak,jsii-rosetta,markdown-it,markdown-it-emoji,nock,normalize-registry-metadata,prettier,semver,spdx-license-list,streamcount,streamx,tar-stream,ts-node,uuid,yaml" ] }, { @@ -1337,6 +1361,7 @@ "@aws-cdk/integ-tests-alpha", "@aws-sdk/client-cloudwatch", "@aws-sdk/client-codeartifact", + "@aws-sdk/client-dynamodb", "@aws-sdk/client-ecs", "@aws-sdk/client-lambda", "@aws-sdk/client-s3", diff --git a/.projenrc.ts b/.projenrc.ts index c38f8b4c6..b0b3c4776 100644 --- a/.projenrc.ts +++ b/.projenrc.ts @@ -56,6 +56,7 @@ const project = new CdklabsConstructLibrary({ cdkCli, '@aws-sdk/client-cloudwatch', '@aws-sdk/client-codeartifact', + '@aws-sdk/client-dynamodb', '@aws-sdk/client-lambda', '@aws-sdk/client-s3', '@aws-sdk/client-sfn', diff --git a/API.md b/API.md index 628b9e841..25f5a18fd 100644 --- a/API.md +++ b/API.md @@ -1724,6 +1724,8 @@ const npmJsProps: sources.NpmJsProps = { ... } | canaryPackage | string | The package that is monitored by the package canary, if enabled by `enableCanary`. | | canarySla | aws-cdk-lib.Duration | The maximum amount of time it is supposed to take for packages to become visible in this ConstructHub instance. | | enableCanary | boolean | Registers a package canary, which will track availability of a canary package in ConstructHub, and emit dedicated metrics. | +| enableKnownVersionsMigration | boolean | Copies the known versions of a deployment created before the known versions table existed from the staging bucket to the table, when the table is created. | +| knownVersionsTableBilling | aws-cdk-lib.aws_dynamodb.Billing | The billing mode of the table in which the follower records the package versions it has processed. | | stagingBucket | aws-cdk-lib.aws_s3.IBucket | The bucket to use for staging npm packages. | --- @@ -1787,6 +1789,37 @@ Registers a package canary, which will track availability of a canary package in --- +##### `enableKnownVersionsMigration`Optional + +```typescript +public readonly enableKnownVersionsMigration: boolean; +``` + +- *Type:* boolean +- *Default:* true + +Copies the known versions of a deployment created before the known versions table existed from the staging bucket to the table, when the table is created. + +Without the copy, an existing deployment starts with an empty table and +processes every version of every construct library again, as their +packages change. Turn this off for new deployments, or if the table was +filled some other way. + +--- + +##### `knownVersionsTableBilling`Optional + +```typescript +public readonly knownVersionsTableBilling: Billing; +``` + +- *Type:* aws-cdk-lib.aws_dynamodb.Billing +- *Default:* Billing.onDemand() + +The billing mode of the table in which the follower records the package versions it has processed. + +--- + ##### `stagingBucket`Optional ```typescript diff --git a/docs/application-overview.md b/docs/application-overview.md index 12521402d..319a94322 100644 --- a/docs/application-overview.md +++ b/docs/application-overview.md @@ -44,7 +44,14 @@ ConstructHub provides two package source implementations: `NpmJs` and scheduled to run every `5 minutes`, and stores its state (receipts for the change entries received, time/sequence checkpoints, and laggy packument expectations) in a specific object in the S3 bucket used for staging package - tarballs. See [npm-sync-protocol.md](./npm-sync-protocol.md) for the + tarballs. + The package versions it has processed are recorded in a DynamoDB table, so + each version is only processed once. + Existing deployments kept this list in the S3 bucket; it is copied to the + table by a function that runs when the table is first deployed. + The follower is only updated once the copy completed, and the copy runs once + more afterwards, to pick up what the old follower recorded in the meantime. + See [npm-sync-protocol.md](./npm-sync-protocol.md) for the guarantees the feed does and does not give, and how the follower compensates. > Back-filling is automatic for the `NpmJs` source. Upon initial deployment, @@ -60,6 +67,7 @@ ConstructHub provides two package source implementations: `NpmJs` and > ``` > > The next scan then covers everything from that position to the feed head. + > Versions recorded in the known versions table are not staged again. > The state object is linked from the backend dashboard. - A **high-severity** alarm triggers if the NpmJs Follower is not running at diff --git a/package.json b/package.json index 8f395fa73..0e391f502 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,8 @@ "bundle:http-get-function:watch": "projen bundle:http-get-function:watch", "bundle:ingestion": "projen bundle:ingestion", "bundle:ingestion:watch": "projen bundle:ingestion:watch", + "bundle:known-versions-migration": "projen bundle:known-versions-migration", + "bundle:known-versions-migration:watch": "projen bundle:known-versions-migration:watch", "bundle:lambda:watch": "projen bundle:lambda:watch", "bundle:monitor": "projen bundle:monitor", "bundle:monitor:watch": "projen bundle:monitor:watch", @@ -120,6 +122,7 @@ "@aws-cdk/integ-tests-alpha": "2.194.0-alpha.0", "@aws-sdk/client-cloudwatch": "^3.1141.0", "@aws-sdk/client-codeartifact": "^3.1141.0", + "@aws-sdk/client-dynamodb": "^3.1141.0", "@aws-sdk/client-ecs": "^3.1141.0", "@aws-sdk/client-lambda": "^3.1141.0", "@aws-sdk/client-s3": "^3.1141.0", diff --git a/src/__tests__/__snapshots__/construct-hub.test.ts.snap b/src/__tests__/__snapshots__/construct-hub.test.ts.snap index 99561bf20..40e4220e0 100644 --- a/src/__tests__/__snapshots__/construct-hub.test.ts.snap +++ b/src/__tests__/__snapshots__/construct-hub.test.ts.snap @@ -296,6 +296,114 @@ exports[`minimal usage 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -640,11 +748,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -9959,10 +10067,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -9974,7 +10124,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -9990,6 +10140,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -10542,6 +10695,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -10642,13 +11038,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -10705,7 +11094,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -10713,40 +11102,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -10859,7 +11234,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -14375,6 +14750,114 @@ exports[`piggy-backing on an existing CodeArtifact domain 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -14719,11 +15202,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -24412,10 +24895,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -24427,7 +24952,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -24443,6 +24968,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -24995,6 +25523,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -25095,13 +25866,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -25158,7 +25922,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -25166,40 +25930,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -25312,7 +26062,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -28766,6 +29516,114 @@ exports[`with all internet access 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -29110,11 +29968,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -38429,10 +39287,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -38444,7 +39344,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -38460,6 +39360,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -39012,51 +39915,111 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, - "ConstructHubSourcesNpmJsSchedule34031870": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", "DependsOn": [ - "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", ], "Properties": { - "Description": "Test/ConstructHub/Sources/NpmJs/Schedule", - "ScheduleExpression": "rate(5 minutes)", - "State": "ENABLED", - "Targets": [ - { - "Arn": { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJs15A77D2D", - "Arn", - ], - }, - "Id": "Target0", - }, - ], + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", }, - "Type": "AWS::Events::Rule", + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", }, - "ConstructHubSourcesNpmJsScheduleAllowEventRuleTestConstructHubSourcesNpmJs5171C2A673CCB598": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", "DependsOn": [ - "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", ], "Properties": { - "Action": "lambda:InvokeFunction", - "FunctionName": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { "Fn::GetAtt": [ - "ConstructHubSourcesNpmJs15A77D2D", + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", "Arn", ], }, - "Principal": "events.amazonaws.com", - "SourceArn": { + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsSchedule34031870", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", "Arn", ], }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, }, - "Type": "AWS::Lambda::Permission", + "Type": "AWS::Lambda::Function", }, - "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -39091,7 +40054,7 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::IAM::Role", }, - "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22": { + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", @@ -39112,13 +40075,36 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", ], "Effect": "Allow", "Resource": [ @@ -39144,18 +40130,134 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, + "ConstructHubSourcesNpmJsSchedule34031870": { + "DependsOn": [ + "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + ], + "Properties": { + "Description": "Test/ConstructHub/Sources/NpmJs/Schedule", + "ScheduleExpression": "rate(5 minutes)", + "State": "ENABLED", + "Targets": [ + { + "Arn": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJs15A77D2D", + "Arn", + ], + }, + "Id": "Target0", + }, + ], + }, + "Type": "AWS::Events::Rule", + }, + "ConstructHubSourcesNpmJsScheduleAllowEventRuleTestConstructHubSourcesNpmJs5171C2A673CCB598": { + "DependsOn": [ + "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", + ], + "Properties": { + "Action": "lambda:InvokeFunction", + "FunctionName": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJs15A77D2D", + "Arn", + ], + }, + "Principal": "events.amazonaws.com", + "SourceArn": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsSchedule34031870", + "Arn", + ], + }, + }, + "Type": "AWS::Lambda::Permission", + }, + "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, { "Action": [ "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -39175,7 +40277,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-last-transaction-id.2", ], ], }, @@ -39218,6 +40320,29 @@ Direct link to Lambda function: /lambda/home#/functions/", }, ], }, + { + "Action": [ + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, { "Action": [ "s3:GetObject*", @@ -39329,7 +40454,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -42791,6 +43916,114 @@ exports[`with domain 1`] = ` }, }, "Resources": { + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -43275,11 +44508,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -52603,10 +53836,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -52618,7 +53893,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -52634,6 +53909,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -53173,6 +54451,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -53273,13 +54794,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -53336,7 +54850,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -53344,40 +54858,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -53490,7 +54990,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { @@ -57281,6 +58781,114 @@ exports[`with limited internet access 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -57625,11 +59233,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -66760,10 +68368,52 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -66775,7 +68425,7 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[Test/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -66791,6 +68441,9 @@ RunBook: https://github.com/cdklabs/construct-hub/blob/main/docs/operator-runboo "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -67343,6 +68996,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A70125de5fd221ef5bc3c75774031d1fa6da": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[Test/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -67443,13 +69339,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -67506,7 +69395,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -67514,40 +69403,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -67660,7 +69535,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap index 770f4ace2..59db2a477 100644 --- a/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap +++ b/src/__tests__/devapp/__snapshots__/snapshot.test.ts.snap @@ -358,6 +358,114 @@ exports[`golden snapshot 1`] = ` }, "Type": "AWS::IAM::Role", }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91": { + "DependsOn": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + ], + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "b585179dc46d9100f31367c1eb03e063c4e1a396f99a6a188ac4a05a16ae4fa5.zip", + }, + "Handler": "__entrypoint__.handler", + "MemorySize": 128, + "Role": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A", + "Arn", + ], + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region", + }, + "value", + ], + }, + "Timeout": 900, + }, + "Type": "AWS::Lambda::Function", + }, + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderRoleE18FAF0A": { + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:\${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + }, + ], + "Policies": [ + { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + { + "Action": [ + "lambda:InvokeFunction", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + "Arn", + ], + }, + ":*", + ], + ], + }, + ], + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "Inline", + }, + ], + }, + "Type": "AWS::IAM::Role", + }, "AppRegistryAssociation": { "Properties": { "Application": { @@ -702,11 +810,11 @@ def sort_filter_rules(json_obj): { "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", }, - "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/s3/object/", + "?prefix=couchdb-last-transaction-id.2)\\n[button:Known Versions](/dynamodbv2/home#table?name=", { - "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Ref": "ConstructHubSourcesKnownVersions1480B28B", }, - "?prefix=couchdb-known-versions.2)\\n[button:Stager](/lambda/home#/functions/", + ")\\n[button:Stager](/lambda/home#/functions/", { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, @@ -11234,10 +11342,52 @@ Direct link to the function: /lambda/home#/functions/", }, "Type": "AWS::S3::BucketPolicy", }, + "ConstructHubSourcesKnownVersions1480B28B": { + "DeletionPolicy": "Retain", + "Properties": { + "AttributeDefinitions": [ + { + "AttributeName": "name", + "AttributeType": "S", + }, + { + "AttributeName": "version", + "AttributeType": "S", + }, + ], + "BillingMode": "PAY_PER_REQUEST", + "KeySchema": [ + { + "AttributeName": "name", + "KeyType": "HASH", + }, + { + "AttributeName": "version", + "KeyType": "RANGE", + }, + ], + "Replicas": [ + { + "PointInTimeRecoverySpecification": { + "PointInTimeRecoveryEnabled": true, + }, + "Region": { + "Ref": "AWS::Region", + }, + }, + ], + "SSESpecification": { + "SSEEnabled": false, + }, + }, + "Type": "AWS::DynamoDB::GlobalTable", + "UpdateReplacePolicy": "Retain", + }, "ConstructHubSourcesNpmJs15A77D2D": { "DependsOn": [ "ConstructHubLicenseListAwsCliLayer59592811", "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619", "ConstructHubSourcesNpmJsServiceRoleDefaultPolicy65FBFA22", "ConstructHubSourcesNpmJsServiceRoleAC3F7AA6", ], @@ -11249,7 +11399,7 @@ Direct link to the function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "ffa98ec8d7f020c136187b40964432444ae3eb94878c781f5e02ad6297b3c7d8.zip", + "S3Key": "9df4ea6f33aadd84d0a7f36cbfad610289ea74b3849c6ff98f8d8731c9f9e93e.zip", }, "Description": "[dev/ConstructHub/Sources/NpmJs] Periodically query npmjs.com index for new packages", "Environment": { @@ -11265,6 +11415,9 @@ Direct link to the function: /lambda/home#/functions/", "FUNCTION_NAME": { "Ref": "ConstructHubSourcesNpmJsStageAndNotify591C0CFA", }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, "LICENSE_LIST_BUCKET_NAME": { "Ref": "ConstructHubLicenseListBucket9334047F", }, @@ -11811,6 +11964,249 @@ Direct link to Lambda function: /lambda/home#/functions/", }, "Type": "AWS::CloudWatch::Alarm", }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationAfterFollowerUpdateDB1AA774": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJs15A77D2D", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf", + }, + "InvocationType": "Event", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "120000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationBeforeFollowerUpdateF06D8619": { + "DeletionPolicy": "Delete", + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesKnownVersions1480B28B", + ], + "Properties": { + "ExecuteOnHandlerChange": false, + "HandlerArn": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf", + }, + "InvocationType": "RequestResponse", + "ServiceToken": { + "Fn::GetAtt": [ + "AWSCDKTriggerCustomResourceProviderCustomResourceProviderHandler97BECD91", + "Arn", + ], + }, + "Timeout": "720000", + }, + "Type": "Custom::Trigger", + "UpdateReplacePolicy": "Delete", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionCurrentVersionA8B4A7011fc689a372f2abb95be7d24b67a416cf": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "FunctionName": { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF", + }, + }, + "Type": "AWS::Lambda::Version", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionEB7112AF": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + ], + "Properties": { + "Architectures": [ + "arm64", + ], + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", + }, + "S3Key": "8f27fe5ee9f5c0c99dfa0d66e98295ec31b3d124970fc7162d4b121f37b053a3.zip", + }, + "Description": "[dev/ConstructHub/Sources/NpmJs] Copies the known versions from the staging bucket to the known versions table", + "Environment": { + "Variables": { + "BUCKET_NAME": { + "Ref": "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + }, + "KNOWN_VERSIONS_TABLE_NAME": { + "Ref": "ConstructHubSourcesKnownVersions1480B28B", + }, + }, + }, + "Handler": "index.handler", + "MemorySize": 2048, + "Role": { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + "Arn", + ], + }, + "Runtime": "nodejs22.x", + "Timeout": 600, + "TracingConfig": { + "Mode": "Active", + }, + }, + "Type": "AWS::Lambda::Function", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com", + }, + }, + ], + "Version": "2012-10-17", + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition", + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole", + ], + ], + }, + ], + }, + "Type": "AWS::IAM::Role", + }, + "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218": { + "DependsOn": [ + "ConstructHubLicenseListAwsCliLayer59592811", + "ConstructHubLicenseListCustomResource323F0FD4", + ], + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "xray:PutTraceSegments", + "xray:PutTelemetryRecords", + ], + "Effect": "Allow", + "Resource": "*", + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-known-versions.2", + ], + ], + }, + ], + }, + { + "Action": [ + "s3:GetObject*", + "s3:GetBucket*", + "s3:List*", + ], + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "ConstructHubSourcesNpmJsStagingBucketB286F0E6", + "Arn", + ], + }, + "/couchdb-last-transaction-id.2", + ], + ], + }, + ], + }, + { + "Action": [ + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, + }, + ], + "Version": "2012-10-17", + }, + "PolicyName": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleDefaultPolicy80D50218", + "Roles": [ + { + "Ref": "ConstructHubSourcesNpmJsKnownVersionsMigrationFunctionServiceRoleAA17733A", + }, + ], + }, + "Type": "AWS::IAM::Policy", + }, "ConstructHubSourcesNpmJsSchedule34031870": { "DependsOn": [ "ConstructHubSourcesNpmJsFollowerFailures86BCBA0D", @@ -11911,13 +12307,6 @@ Direct link to Lambda function: /lambda/home#/functions/", "s3:GetObject*", "s3:GetBucket*", "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", ], "Effect": "Allow", "Resource": [ @@ -11974,7 +12363,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "Arn", ], }, - "/couchdb-known-versions.2", + "/couchdb-follower-state", ], ], }, @@ -11982,40 +12371,26 @@ Direct link to Lambda function: /lambda/home#/functions/", }, { "Action": [ - "s3:GetObject*", - "s3:GetBucket*", - "s3:List*", - "s3:DeleteObject*", - "s3:PutObject", - "s3:PutObjectLegalHold", - "s3:PutObjectRetention", - "s3:PutObjectTagging", - "s3:PutObjectVersionTagging", - "s3:Abort*", + "dynamodb:BatchGetItem", + "dynamodb:GetRecords", + "dynamodb:GetShardIterator", + "dynamodb:Query", + "dynamodb:GetItem", + "dynamodb:Scan", + "dynamodb:ConditionCheckItem", + "dynamodb:BatchWriteItem", + "dynamodb:PutItem", + "dynamodb:UpdateItem", + "dynamodb:DeleteItem", + "dynamodb:DescribeTable", ], "Effect": "Allow", - "Resource": [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - { - "Fn::Join": [ - "", - [ - { - "Fn::GetAtt": [ - "ConstructHubSourcesNpmJsStagingBucketB286F0E6", - "Arn", - ], - }, - "/couchdb-follower-state", - ], - ], - }, - ], + "Resource": { + "Fn::GetAtt": [ + "ConstructHubSourcesKnownVersions1480B28B", + "Arn", + ], + }, }, { "Action": [ @@ -12128,7 +12503,7 @@ Direct link to Lambda function: /lambda/home#/functions/", "S3Bucket": { "Fn::Sub": "cdk-hnb659fds-assets-\${AWS::AccountId}-\${AWS::Region}", }, - "S3Key": "e2b7f901593926244f91584643ade337341313ba64f6d0492691d5b148564261.zip", + "S3Key": "8625ecb0c4c91c2ca0404a682cb734f265ca9331fd380d51dd6759eb64acffdc.zip", }, "DeadLetterConfig": { "TargetArn": { diff --git a/src/__tests__/package-sources/npmjs.test.ts b/src/__tests__/package-sources/npmjs.test.ts new file mode 100644 index 000000000..a5b6b5edd --- /dev/null +++ b/src/__tests__/package-sources/npmjs.test.ts @@ -0,0 +1,126 @@ +import { App, Stack } from 'aws-cdk-lib'; +import { Match, Template } from 'aws-cdk-lib/assertions'; +import { Billing, Capacity } from 'aws-cdk-lib/aws-dynamodb'; +import { ConstructHub } from '../../construct-hub'; +import { NpmJs } from '../../package-sources'; + +function synth(npmJs: NpmJs): Template { + const app = new App(); + const stack = new Stack(app, 'Test'); + new ConstructHub(stack, 'ConstructHub', { packageSources: [npmJs] }); + return Template.fromStack(stack); +} + +test('creates a retained, backed-up, on-demand known versions table', () => { + const template = synth(new NpmJs()); + + template.hasResource('AWS::DynamoDB::GlobalTable', { + DeletionPolicy: 'Retain', + UpdateReplacePolicy: 'Retain', + Properties: Match.objectLike({ + BillingMode: 'PAY_PER_REQUEST', + KeySchema: [ + { AttributeName: 'name', KeyType: 'HASH' }, + { AttributeName: 'version', KeyType: 'RANGE' }, + ], + Replicas: [ + Match.objectLike({ + PointInTimeRecoverySpecification: { + PointInTimeRecoveryEnabled: true, + }, + }), + ], + }), + }); +}); + +test('passes the known versions table to the follower', () => { + const template = synth(new NpmJs()); + + template.hasResourceProperties('AWS::Lambda::Function', { + Environment: { + Variables: Match.objectLike({ + KNOWN_VERSIONS_TABLE_NAME: Match.anyValue(), + }), + }, + }); +}); + +test('migrates existing known versions before the follower is updated, and catches up after', () => { + const template = synth(new NpmJs()); + + const migrations = template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Copies the known versions'), + }, + }); + expect(Object.keys(migrations)).toHaveLength(1); + const [migrationId] = Object.keys(migrations); + + const versions = template.findResources('AWS::Lambda::Version', { + Properties: { FunctionName: { Ref: migrationId } }, + }); + expect(Object.keys(versions)).toHaveLength(1); + + const triggers = template.findResources('Custom::Trigger', { + Properties: { + HandlerArn: { Ref: Object.keys(versions)[0] }, + InvocationType: 'RequestResponse', + ExecuteOnHandlerChange: false, + }, + }); + expect(Object.keys(triggers)).toHaveLength(1); + const [triggerId] = Object.keys(triggers); + + // The follower function is only deployed after the migration ran. + const followers = template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Periodically query npmjs.com'), + }, + }); + expect(Object.keys(followers)).toHaveLength(1); + expect(Object.values(followers)[0].DependsOn).toContain(triggerId); + const [followerId] = Object.keys(followers); + + // A second pass after the follower update picks up what the old follower + // code recorded in the meantime. + const catchUps = template.findResources('Custom::Trigger', { + Properties: { + HandlerArn: { Ref: Object.keys(versions)[0] }, + InvocationType: 'Event', + ExecuteOnHandlerChange: false, + }, + }); + expect(Object.keys(catchUps)).toHaveLength(1); + expect(Object.values(catchUps)[0].DependsOn).toContain(followerId); +}); + +test('the known versions migration can be turned off', () => { + const template = synth(new NpmJs({ enableKnownVersionsMigration: false })); + + template.resourceCountIs('Custom::Trigger', 0); + expect( + Object.keys( + template.findResources('AWS::Lambda::Function', { + Properties: { + Description: Match.stringLikeRegexp('Copies the known versions'), + }, + }) + ) + ).toHaveLength(0); +}); + +test('the billing mode of the known versions table is configurable', () => { + const template = synth( + new NpmJs({ + knownVersionsTableBilling: Billing.provisioned({ + readCapacity: Capacity.fixed(10), + writeCapacity: Capacity.autoscaled({ maxCapacity: 20 }), + }), + }) + ); + + template.hasResourceProperties('AWS::DynamoDB::GlobalTable', { + BillingMode: 'PROVISIONED', + }); +}); diff --git a/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts b/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts new file mode 100644 index 000000000..41190ba0b --- /dev/null +++ b/src/__tests__/package-sources/npmjs/known-versions-migration.lambda.test.ts @@ -0,0 +1,126 @@ +import { + BatchWriteItemCommand, + DynamoDBClient, +} from '@aws-sdk/client-dynamodb'; +import { GetObjectCommand, S3Client } from '@aws-sdk/client-s3'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from '../../../package-sources/npmjs/constants.lambda-shared'; +import { handler } from '../../../package-sources/npmjs/known-versions-migration.lambda'; +import { stringToStream } from '../../streams'; + +const MOCK_BUCKET = 'mock-staging-bucket'; +const MOCK_TABLE = 'mock-known-versions-table'; + +const mockS3 = mockClient(S3Client); +const mockDynamoDB = mockClient(DynamoDBClient); + +beforeEach(() => { + mockS3.reset(); + mockDynamoDB.reset(); + process.env.BUCKET_NAME = MOCK_BUCKET; + process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_TABLE; + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + givenNoObject(KNOWN_VERSIONS_FILE_NAME); + givenNoObject(MARKER_FILE_NAME); +}); + +afterEach(() => { + delete process.env.BUCKET_NAME; + delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME]; +}); + +function givenNoObject(key: string) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: key }) + .rejects(Object.assign(new Error('no such key'), { name: 'NoSuchKey' })); +} + +function givenObject(key: string, content: unknown) { + mockS3 + .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: key }) + .resolves({ Body: stringToStream(JSON.stringify(content)) }); +} + +/** The `name@version` keys written to the table, sorted. */ +function writtenKeys(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap((call) => call.args[0].input.RequestItems![MOCK_TABLE]) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +test('copies the known versions file to the table', async () => { + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { + 'some-package@1.0.0': '2023-06-01T00:00:00.000Z', + '@scope/other-package@2.0.0-pre.1': '2023-06-02T00:00:00.000Z', + }, + }); + + const result = await handler(); + + expect(result).toEqual({ written: 2, skipped: [] }); + expect(writtenKeys()).toEqual([ + '@scope/other-package@2.0.0-pre.1', + 'some-package@1.0.0', + ]); +}); + +test('copies the known versions embedded in the legacy marker file', async () => { + givenObject(MARKER_FILE_NAME, { + marker: 1_000, + knownVersions: { 'legacy-package@1.0.0': 1685577600000 }, + }); + + const result = await handler(); + + expect(result).toEqual({ written: 1, skipped: [] }); + expect(writtenKeys()).toEqual(['legacy-package@1.0.0']); +}); + +test('does nothing on a new deployment', async () => { + const result = await handler(); + + expect(result).toEqual({ written: 0, skipped: [] }); + expect(mockDynamoDB).not.toHaveReceivedCommand(BatchWriteItemCommand); +}); + +test('skips records the table cannot store', async () => { + const tooLong = `long-package@${'1'.repeat(1_025)}`; + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { + 'good-package@1.0.0': '2023-06-01T00:00:00.000Z', + '@scope/no-version': '2023-06-01T00:00:00.000Z', + 'no-version@': '2023-06-01T00:00:00.000Z', + [tooLong]: '2023-06-01T00:00:00.000Z', + }, + }); + + const result = await handler(); + + expect(result).toEqual({ + written: 1, + skipped: ['@scope/no-version', 'no-version@', tooLong], + }); + expect(writtenKeys()).toEqual(['good-package@1.0.0']); +}); + +test('fails on other errors, so the invocation is retried', async () => { + givenObject(KNOWN_VERSIONS_FILE_NAME, { + knownVersions: { 'good-package@1.0.0': '2023-06-01T00:00:00.000Z' }, + }); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('network')); + + await expect(handler()).rejects.toThrow(/network/); +}); diff --git a/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts b/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts new file mode 100644 index 000000000..d12c8185a --- /dev/null +++ b/src/__tests__/package-sources/npmjs/known-versions.lambda-shared.test.ts @@ -0,0 +1,175 @@ +import { + BatchWriteItemCommand, + DynamoDB, + DynamoDBClient, + QueryCommand, +} from '@aws-sdk/client-dynamodb'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest'; +import { KnownVersions } from '../../../package-sources/npmjs/known-versions.lambda-shared'; + +const TABLE = 'known-versions-table'; +const mockDynamoDB = mockClient(DynamoDBClient); + +function newKnownVersions() { + return new KnownVersions(TABLE, new DynamoDB({})); +} + +/** The `name@version` keys written by `BatchWriteItem`, sorted. */ +function writtenKeys(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap((call) => call.args[0].input.RequestItems![TABLE]) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +beforeEach(() => { + mockDynamoDB.reset(); + mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); +}); + +describe('has', () => { + test('returns whether a version is recorded for the package', async () => { + mockDynamoDB.on(QueryCommand).resolves({ + Items: [{ version: { S: '1.0.0' } }, { version: { S: '1.1.0' } }], + }); + const known = newKnownVersions(); + + expect(await known.has('some-package', '1.0.0')).toBe(true); + expect(await known.has('some-package', '1.1.0')).toBe(true); + expect(await known.has('some-package', '2.0.0')).toBe(false); + expect(mockDynamoDB).toHaveReceivedCommandWith(QueryCommand, { + TableName: TABLE, + ConsistentRead: true, + ExpressionAttributeValues: { ':name': { S: 'some-package' } }, + }); + }); + + test('follows query pagination', async () => { + mockDynamoDB + .on(QueryCommand) + .resolvesOnce({ + Items: [{ version: { S: '1.0.0' } }], + LastEvaluatedKey: { + name: { S: 'big-package' }, + version: { S: '1.0.0' }, + }, + }) + .resolvesOnce({ Items: [{ version: { S: '2.0.0' } }] }); + const known = newKnownVersions(); + + expect(await known.has('big-package', '1.0.0')).toBe(true); + expect(await known.has('big-package', '2.0.0')).toBe(true); + expect(mockDynamoDB).toHaveReceivedCommandTimes(QueryCommand, 2); + }); + + test('queries each package only once, also for concurrent callers', async () => { + const known = newKnownVersions(); + + await Promise.all([ + known.has('some-package', '1.0.0'), + known.has('some-package', '2.0.0'), + ]); + await known.has('some-package', '3.0.0'); + + expect(mockDynamoDB).toHaveReceivedCommandTimes(QueryCommand, 1); + }); + + test('queries again after a failed query', async () => { + mockDynamoDB + .on(QueryCommand) + .rejectsOnce(new Error('network')) + .resolves({ Items: [{ version: { S: '1.0.0' } }] }); + const known = newKnownVersions(); + + await expect(known.has('some-package', '1.0.0')).rejects.toThrow(/network/); + expect(await known.has('some-package', '1.0.0')).toBe(true); + }); + + test('sees versions added locally right away, without a query', async () => { + const known = newKnownVersions(); + + known.add('@scope/package', '1.0.0'); + + expect(await known.has('@scope/package', '1.0.0')).toBe(true); + expect(mockDynamoDB).not.toHaveReceivedCommand(QueryCommand); + }); +}); + +describe('add and flush', () => { + test('flush persists the added versions', async () => { + const known = newKnownVersions(); + + known.add('@scope/package', '1.0.0'); + known.add('other-package', '2.0.0'); + known.add('other-package', '2.0.0'); + expect(mockDynamoDB).not.toHaveReceivedCommand(BatchWriteItemCommand); + expect(known.pendingCount).toBe(2); + + await known.flush(); + + expect(known.pendingCount).toBe(0); + expect(writtenKeys()).toEqual([ + '@scope/package@1.0.0', + 'other-package@2.0.0', + ]); + }); + + test('keeps the records of a failed flush for the next attempt', async () => { + mockDynamoDB + .on(BatchWriteItemCommand) + .rejectsOnce(new Error('network')) + .resolves({ UnprocessedItems: {} }); + const known = newKnownVersions(); + known.add('some-package', '1.0.0'); + + await expect(known.flush()).rejects.toThrow(/network/); + expect(known.pendingCount).toBe(1); + + await known.flush(); + expect(known.pendingCount).toBe(0); + }); +}); + +describe('flush batching', () => { + test('writes the records in batches of 25', async () => { + const known = newKnownVersions(); + for (let i = 0; i < 60; i++) { + known.add(`package-${i}`, `1.0.${i}`); + } + known.add('@scope/package', '2.0.0'); + + await known.flush(); + + expect(mockDynamoDB).toHaveReceivedCommandTimes(BatchWriteItemCommand, 3); + expect(writtenKeys()).toHaveLength(61); + expect(writtenKeys()).toContain('@scope/package@2.0.0'); + }); + + test('retries unprocessed items', async () => { + const unprocessed = { + PutRequest: { + Item: { name: { S: 'some-package' }, version: { S: '1.0.0' } }, + }, + }; + mockDynamoDB + .on(BatchWriteItemCommand) + .resolvesOnce({ UnprocessedItems: { [TABLE]: [unprocessed] } }) + .resolves({ UnprocessedItems: {} }); + const known = newKnownVersions(); + known.add('some-package', '1.0.0'); + + await known.flush(); + + const calls = mockDynamoDB.commandCalls(BatchWriteItemCommand); + expect(calls).toHaveLength(2); + expect(calls[1].args[0].input.RequestItems![TABLE]).toEqual([unprocessed]); + }); +}); diff --git a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts index bb92a1c16..e93482311 100644 --- a/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts +++ b/src/__tests__/package-sources/npmjs/npm-js-follower.lambda.test.ts @@ -1,4 +1,9 @@ import { gunzipSync, gzipSync } from 'zlib'; +import { + BatchWriteItemCommand, + DynamoDBClient, + QueryCommand, +} from '@aws-sdk/client-dynamodb'; import { InvokeCommand, LambdaClient } from '@aws-sdk/client-lambda'; import { GetObjectCommand, @@ -17,8 +22,8 @@ import { } from '../../../backend/deny-list/constants'; import { EnvironmentVariables as LicenseListEnv } from '../../../backend/license-list/constants'; import { + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, - KNOWN_VERSIONS_FILE_NAME, MARKER_FILE_NAME, MetricName, } from '../../../package-sources/npmjs/constants.lambda-shared'; @@ -53,6 +58,7 @@ const REGISTRY = 'https://registry.npmjs.org'; const MOCK_BUCKET = 'mock-staging-bucket'; const MOCK_STAGING_FUNCTION = 'mock-staging-function'; +const MOCK_KNOWN_VERSIONS_TABLE = 'mock-known-versions-table'; const MOCK_DENY_LIST_BUCKET = 'deny-list-bucket-name'; const MOCK_DENY_LIST_OBJECT = 'my-deny-list.json'; const MOCK_LICENSE_LIST_BUCKET = 'license-list-bucket-name'; @@ -62,6 +68,7 @@ const HOUR = 60 * 60 * 1_000; const mockS3 = mockClient(S3Client); const mockLambda = mockClient(LambdaClient); +const mockDynamoDB = mockClient(DynamoDBClient); const context: Context = { awsRequestId: 'mock-request-id', @@ -75,10 +82,12 @@ const event: ScheduledEvent = {} as any; beforeEach(() => { mockS3.reset(); mockLambda.reset(); + mockDynamoDB.reset(); mockPutMetric.mockClear(); process.env.BUCKET_NAME = MOCK_BUCKET; process.env.FUNCTION_NAME = MOCK_STAGING_FUNCTION; + process.env[ENV_KNOWN_VERSIONS_TABLE_NAME] = MOCK_KNOWN_VERSIONS_TABLE; process.env[ENV_DENY_LIST_BUCKET_NAME] = MOCK_DENY_LIST_BUCKET; process.env[ENV_DENY_LIST_OBJECT_KEY] = MOCK_DENY_LIST_OBJECT; process.env[LicenseListEnv.BUCKET_NAME] = MOCK_LICENSE_LIST_BUCKET; @@ -98,18 +107,15 @@ beforeEach(() => { Key: MOCK_LICENSE_LIST_OBJECT, }) .resolves({ Body: stringToStream(JSON.stringify(['Apache-2.0'])) }); - // Empty known versions - mockS3 - .on(GetObjectCommand, { - Bucket: MOCK_BUCKET, - Key: KNOWN_VERSIONS_FILE_NAME, - }) - .resolves({ Body: stringToStream(JSON.stringify({ knownVersions: {} })) }); // No legacy marker by default mockS3 .on(GetObjectCommand, { Bucket: MOCK_BUCKET, Key: MARKER_FILE_NAME }) .rejects(noSuchKey()); + // Empty known versions table + mockDynamoDB.on(QueryCommand).resolves({ Items: [] }); + mockDynamoDB.on(BatchWriteItemCommand).resolves({ UnprocessedItems: {} }); + mockS3.on(PutObjectCommand).resolves({}); mockLambda.on(InvokeCommand).resolves({ StatusCode: 202 }); }); @@ -117,6 +123,7 @@ beforeEach(() => { afterEach(() => { delete process.env.BUCKET_NAME; delete process.env.FUNCTION_NAME; + delete process.env[ENV_KNOWN_VERSIONS_TABLE_NAME]; delete process.env[ENV_DENY_LIST_BUCKET_NAME]; delete process.env[ENV_DENY_LIST_OBJECT_KEY]; delete process.env[LicenseListEnv.BUCKET_NAME]; @@ -757,3 +764,93 @@ test('resets its position when the feed head regresses below the newest checkpoi expect(saved.newestCheckpointSeq()).toBe(3_000); expect(saved.has(4_500)).toBe(false); }); + +//#region Known versions table +function givenKnownVersionsInTable(name: string, versions: string[]) { + mockDynamoDB + .on(QueryCommand, { + ExpressionAttributeValues: { ':name': { S: name } }, + }) + .resolves({ + Items: versions.map((version) => ({ version: { S: version } })), + }); +} + +/** The `name@version` keys recorded in the table, sorted. */ +function recordedVersions(): string[] { + return mockDynamoDB + .commandCalls(BatchWriteItemCommand) + .flatMap( + (call) => call.args[0].input.RequestItems![MOCK_KNOWN_VERSIONS_TABLE] + ) + .map( + (request) => + `${request.PutRequest!.Item!.name.S}@${ + request.PutRequest!.Item!.version.S + }` + ) + .sort(); +} + +function givenOneChange(id: string, doc: object) { + nock(REPLICA) + .get('/registry/_changes') + .query({ limit: '10000', since: '1000' }) + .reply(200, { + results: [ + { seq: 1_005, id, changes: [{ rev: '1-aaaaaa' }], deleted: false }, + ], + last_seq: 1_005, + }); + nock(REGISTRY).get(`/${id}`).reply(200, doc); +} + +test('stages only versions not recorded in the table, and records them', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange( + 'known-package', + packageDoc('known-package', '1-aaaaaa', '1.0.0', '1.1.0') + ); + givenKnownVersionsInTable('known-package', ['1.0.0']); + + await handler(event, context); + + expect(stagedPackages()).toEqual([ + { name: 'known-package', version: '1.1.0' }, + ]); + expect(recordedVersions()).toEqual(['known-package@1.1.0']); +}); + +test('records versions before the follower state, so a failed save never skips a version', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange( + 'fresh-package', + packageDoc('fresh-package', '1-aaaaaa', '1.0.0') + ); + mockDynamoDB.on(BatchWriteItemCommand).rejects(new Error('network')); + + await expect(handler(event, context)).rejects.toThrow(/network/); + + // The chunk's receipts were not saved: the next run processes it again. + const statePuts = mockS3 + .commandCalls(PutObjectCommand) + .filter((call) => call.args[0].input.Key === FOLLOWER_STATE_FILE_NAME); + expect(statePuts).toHaveLength(0); +}); + +test('does not query the table for packages without construct library versions', async () => { + givenHead(1_010); + givenNoStateFile(); + givenLegacyMarker(1_000); + givenOneChange('boring-package', nonConstructDoc('boring-package')); + + await handler(event, context); + + expect(mockDynamoDB).not.toHaveReceivedCommand(QueryCommand); + expect(mockLambda).not.toHaveReceivedCommand(InvokeCommand); +}); +//#endregion diff --git a/src/construct-hub.ts b/src/construct-hub.ts index fa78a37fa..faeb9e28e 100644 --- a/src/construct-hub.ts +++ b/src/construct-hub.ts @@ -501,19 +501,18 @@ export class ConstructHub extends Construct implements iam.IGrantable { feedBuilder.setConstructHubUrl(webApp.baseUrl); const sources = new Construct(this, 'Sources'); - const packageSources = ( - props.packageSources ?? [new NpmJs()] - ).map((source) => - source.bind(sources, { - baseUrl: webApp.baseUrl, - denyList, - ingestion: this.ingestion, - licenseList, - monitoring: this.monitoring, - queue: this.ingestion.queue, - repository: codeArtifact, - overviewDashboard: overviewDashboard, - }) + const packageSources = (props.packageSources ?? [new NpmJs()]).map( + (source) => + source.bind(sources, { + baseUrl: webApp.baseUrl, + denyList, + ingestion: this.ingestion, + licenseList, + monitoring: this.monitoring, + queue: this.ingestion.queue, + repository: codeArtifact, + overviewDashboard: overviewDashboard, + }) ); const inventory = new Inventory(this, 'InventoryCanary', { diff --git a/src/deep-link.ts b/src/deep-link.ts index f67a62562..bbaa8f126 100644 --- a/src/deep-link.ts +++ b/src/deep-link.ts @@ -1,5 +1,6 @@ import { Fn, Stack } from 'aws-cdk-lib'; import { CfnRepository } from 'aws-cdk-lib/aws-codeartifact'; +import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb'; import { ICluster } from 'aws-cdk-lib/aws-ecs'; import { IFunction } from 'aws-cdk-lib/aws-lambda'; import { ILogGroup } from 'aws-cdk-lib/aws-logs'; @@ -11,6 +12,10 @@ export function codeArtifactRepositoryUrl(repository: CfnRepository) { return `/codesuite/codeartifact/d/${repository.attrDomainOwner}/${repository.attrDomainName}/r/${repository.attrName}`; } +export function dynamoDbTableUrl(table: ITableV2): string { + return `/dynamodbv2/home#table?name=${table.tableName}`; +} + export function ecsClusterUrl(cluster: ICluster): string { return `/ecs/home#/clusters/${cluster.clusterName}/tasks`; } diff --git a/src/package-sources/npmjs.ts b/src/package-sources/npmjs.ts index c11b3aa71..5ae6a3b33 100644 --- a/src/package-sources/npmjs.ts +++ b/src/package-sources/npmjs.ts @@ -1,4 +1,4 @@ -import { Duration } from 'aws-cdk-lib'; +import { Duration, RemovalPolicy } from 'aws-cdk-lib'; import { AlarmRule, ComparisonOperator, @@ -13,6 +13,12 @@ import { Stats, TreatMissingData, } from 'aws-cdk-lib/aws-cloudwatch'; +import { + AttributeType, + Billing, + TableEncryptionV2, + TableV2, +} from 'aws-cdk-lib/aws-dynamodb'; import { Rule, Schedule } from 'aws-cdk-lib/aws-events'; import { LambdaFunction } from 'aws-cdk-lib/aws-events-targets'; import { Tracing } from 'aws-cdk-lib/aws-lambda'; @@ -22,6 +28,7 @@ import { Queue, QueueEncryption } from 'aws-cdk-lib/aws-sqs'; import { Construct } from 'constructs'; import { AlarmSeverity } from '../api'; import { + dynamoDbTableUrl, lambdaFunctionUrl, logAnalyticsUrl, s3ObjectUrl, @@ -29,25 +36,28 @@ import { } from '../deep-link'; import { fillMetric } from '../metric-utils'; import { addAlarm } from '../monitoring'; +import { IMonitoring } from '../monitoring/api'; +import type { + IPackageSource, + PackageSourceBindOptions, + PackageSourceBindResult, +} from '../package-source'; +import { RUNBOOK_URL } from '../runbook-url'; import { NpmJsPackageCanary } from './npmjs/canary'; import { + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, KNOWN_VERSIONS_FILE_NAME, + KnownVersionsAttribute, MARKER_FILE_NAME, METRICS_NAMESPACE, MetricName, S3KeyPrefix, SCAN_WINDOW_MS, } from './npmjs/constants.lambda-shared'; +import { KnownVersionsTableMigration } from './npmjs/known-versions-table-migration'; import { NpmJsFollower } from './npmjs/npm-js-follower'; import { StageAndNotify } from './npmjs/stage-and-notify'; -import { IMonitoring } from '../monitoring/api'; -import type { - IPackageSource, - PackageSourceBindOptions, - PackageSourceBindResult, -} from '../package-source'; -import { RUNBOOK_URL } from '../runbook-url'; import { S3StorageFactory } from '../s3/storage'; import { ReStagePackageVersion } from './npmjs/re-stage-package-version'; @@ -114,6 +124,28 @@ export interface NpmJsProps { * @default Duration.days(1) */ readonly canaryMaxStale?: Duration; + + /** + * The billing mode of the table in which the follower records the package + * versions it has processed. + * + * @default Billing.onDemand() + */ + readonly knownVersionsTableBilling?: Billing; + + /** + * Copies the known versions of a deployment created before the known + * versions table existed from the staging bucket to the table, when the + * table is created. + * + * Without the copy, an existing deployment starts with an empty table and + * processes every version of every construct library again, as their + * packages change. Turn this off for new deployments, or if the table was + * filled some other way. + * + * @default true + */ + readonly enableKnownVersionsMigration?: boolean; } /** @@ -202,22 +234,51 @@ export class NpmJs implements IPackageSource { }) ); + // The package versions the follower has processed. Losing this table + // means re-processing every construct library version ever published, so + // it is retained and backed up. + const knownVersions = new TableV2(scope, 'KnownVersions', { + partitionKey: { + name: KnownVersionsAttribute.NAME, + type: AttributeType.STRING, + }, + sortKey: { + name: KnownVersionsAttribute.VERSION, + type: AttributeType.STRING, + }, + billing: this.props.knownVersionsTableBilling ?? Billing.onDemand(), + encryption: TableEncryptionV2.dynamoOwnedKey(), + pointInTimeRecoverySpecification: { pointInTimeRecoveryEnabled: true }, + removalPolicy: RemovalPolicy.RETAIN, + }); + const follower = new NpmJsFollower(scope, 'NpmJs', { description: `[${scope.node.path}/NpmJs] Periodically query npmjs.com index for new packages`, environment: { AWS_EMF_ENVIRONMENT: 'Local', BUCKET_NAME: bucket.bucketName, FUNCTION_NAME: stager.functionName, + [ENV_KNOWN_VERSIONS_TABLE_NAME]: knownVersions.tableName, }, memorySize: 10_024, // 10 GiB - reservedConcurrentExecutions: 1, // Only one execution at a time, to avoid race conditions on the S3 marker object + reservedConcurrentExecutions: 1, // Only one execution at a time, to avoid race conditions on the S3 follower state object timeout: FOLLOWER_RUN_RATE, tracing: Tracing.ACTIVE, }); - bucket.grantReadWrite(follower, MARKER_FILE_NAME); - bucket.grantReadWrite(follower, KNOWN_VERSIONS_FILE_NAME); + // The legacy marker is only read, to seed the follower state of existing + // deployments. + bucket.grantRead(follower, MARKER_FILE_NAME); bucket.grantReadWrite(follower, FOLLOWER_STATE_FILE_NAME); + knownVersions.grantReadWriteData(follower); + + if (this.props.enableKnownVersionsMigration ?? true) { + new KnownVersionsTableMigration(follower, 'KnownVersionsMigration', { + bucket, + table: knownVersions, + follower, + }); + } denyList?.grantRead(follower); licenseList.grantRead(follower); stager.grantInvoke(follower); @@ -278,7 +339,7 @@ export class NpmJs implements IPackageSource { }, { name: 'Known Versions', - url: s3ObjectUrl(bucket, KNOWN_VERSIONS_FILE_NAME), + url: dynamoDbTableUrl(knownVersions), }, { name: 'Stager', url: lambdaFunctionUrl(stager) }, { name: 'Stager DLQ', url: sqsQueueUrl(stager.deadLetterQueue!) }, diff --git a/src/package-sources/npmjs/constants.lambda-shared.ts b/src/package-sources/npmjs/constants.lambda-shared.ts index ea5f09ef6..c32054769 100644 --- a/src/package-sources/npmjs/constants.lambda-shared.ts +++ b/src/package-sources/npmjs/constants.lambda-shared.ts @@ -81,10 +81,27 @@ export const enum S3KeyPrefix { export const MARKER_FILE_NAME = 'couchdb-last-transaction-id.2'; /** - * The name of the object that contains the list of known versions. + * The name of the object that contains the list of known versions. Known + * versions are now kept in a DynamoDB table; this file is only read once, by + * the migration that seeds the table of an existing deployment. */ export const KNOWN_VERSIONS_FILE_NAME = 'couchdb-known-versions.2'; +/** + * The environment variable holding the name of the known versions table. + */ +export const ENV_KNOWN_VERSIONS_TABLE_NAME = 'KNOWN_VERSIONS_TABLE_NAME'; + +/** + * The attribute names of the known versions table. The table is keyed by + * package name (partition key) and version (sort key); an item exists for + * every package version the follower has processed. + */ +export const enum KnownVersionsAttribute { + NAME = 'name', + VERSION = 'version', +} + /** * The name of the object that contains the follower state: receipts for the * change entries received from the `_changes` feed, rolling time/sequence diff --git a/src/package-sources/npmjs/known-versions-migration.lambda.ts b/src/package-sources/npmjs/known-versions-migration.lambda.ts new file mode 100644 index 000000000..647174106 --- /dev/null +++ b/src/package-sources/npmjs/known-versions-migration.lambda.ts @@ -0,0 +1,140 @@ +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from './constants.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; +import { + loadContentFromS3, + MarkerFileSchema, +} from './staging-bucket.lambda-shared'; +import { requireEnv } from '../../backend/shared/env.lambda-shared'; + +/** + * Copies the known versions of an existing deployment from the known versions + * file in S3 to the known versions table. Invoked twice during the deployment + * that creates the table: before the follower function is updated (so the new + * follower code only runs once the table is filled; if this fails, the + * deployment fails), and again after it (to pick up the versions the old + * follower code recorded in the file in the meantime). + * + * Writes are idempotent, so running it again only adds what is missing. Records + * that can never be written are skipped and logged. A new deployment has no + * file, and this does nothing: the table fills up during the backfill. + * + * The file is left in place, and is no longer updated: if a deployment is + * rolled back to a version that still reads it, only the versions processed + * since the migration are processed again. + */ +export async function handler(): Promise<{ + readonly written: number; + readonly skipped: readonly string[]; +}> { + const stagingBucket = requireEnv('BUCKET_NAME'); + const knownVersions = new KnownVersions( + requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) + ); + + const keys = await loadKnownVersionsFile(stagingBucket); + console.log( + `Migrating ${keys.length} known version(s) to the known versions table` + ); + + // Records that can never be written are skipped, so they cannot block the + // others. + const skipped = new Array(); + for (const key of keys) { + const parsed = parseKey(key); + if (parsed == null) { + console.warn(`Skipping invalid known version record: ${key}`); + skipped.push(key); + continue; + } + knownVersions.add(...parsed); + } + const written = knownVersions.pendingCount; + await knownVersions.flush(); + + console.log( + `Known versions migration complete: ${written} migrated, ${skipped.length} skipped` + ); + return { written, skipped }; +} + +/** + * The maximum sizes of the table's key attributes, in bytes. + * @see https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ServiceQuotas.html#limits-partition-sort-keys + */ +const MAX_PARTITION_KEY_BYTES = 2_048; +const MAX_SORT_KEY_BYTES = 1_024; + +/** + * Splits a `name@version` key. Scoped package names start with `@`, so the + * separator is the last `@`. + * + * @returns `undefined` for a key the table cannot store: no name, no version, + * or a part longer than DynamoDB allows for a key attribute. + */ +function parseKey(key: string): [name: string, version: string] | undefined { + const at = key.lastIndexOf('@'); + if (at <= 0 || at === key.length - 1) { + return undefined; + } + const name = key.slice(0, at); + const version = key.slice(at + 1); + if ( + Buffer.byteLength(name) > MAX_PARTITION_KEY_BYTES || + Buffer.byteLength(version) > MAX_SORT_KEY_BYTES + ) { + return undefined; + } + return [name, version]; +} + +/** + * Loads the known versions from the known versions file in S3. For legacy + * reasons, they may also be embedded in the transaction marker file; when the + * dedicated file does not exist, that location is used. + * + * @returns the `name@version` keys; empty if neither file holds known + * versions (a new deployment). + */ +async function loadKnownVersionsFile(stagingBucket: string): Promise { + const content = await loadContentFromS3( + stagingBucket, + KNOWN_VERSIONS_FILE_NAME, + `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist` + ); + if (content != null) { + const contentsObj: KnownVersionsFileSchema = JSON.parse(content); + console.log('Loaded known versions file'); + return Object.keys(contentsObj.knownVersions); + } + + // Legacy location: embedded in the transaction marker file. + const markerContent = await loadContentFromS3( + stagingBucket, + MARKER_FILE_NAME, + `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` + ); + if (markerContent != null) { + try { + const parsed: MarkerFileSchema = JSON.parse(markerContent); + if (typeof parsed !== 'number' && parsed.knownVersions) { + console.log('Loaded known versions from legacy marker file'); + return Object.keys(parsed.knownVersions); + } + } catch (error) { + console.warn(`Could not parse legacy marker: ${error}`); + } + } + return []; +} + +/** + * The known versions file format: `name@version` keys mapped to the + * `modified` timestamps of the versions (no longer used). + */ +interface KnownVersionsFileSchema { + knownVersions: Record; +} diff --git a/src/package-sources/npmjs/known-versions-migration.ts b/src/package-sources/npmjs/known-versions-migration.ts new file mode 100644 index 000000000..208b36d8f --- /dev/null +++ b/src/package-sources/npmjs/known-versions-migration.ts @@ -0,0 +1,20 @@ +// ~~ Generated by projen. To modify, edit .projenrc.ts and run "yarn projen". +import * as path from 'path'; +import * as lambda from 'aws-cdk-lib/aws-lambda'; +import { Construct } from 'constructs'; + +export interface KnownVersionsMigrationProps extends lambda.FunctionOptions { +} + +export class KnownVersionsMigration extends lambda.Function { + constructor(scope: Construct, id: string, props?: KnownVersionsMigrationProps) { + super(scope, id, { + description: 'package-sources/npmjs/known-versions-migration.lambda.ts', + ...props, + architecture: lambda.Architecture.ARM_64, + runtime: lambda.Runtime.NODEJS_22_X, + handler: 'index.handler', + code: lambda.Code.fromAsset(path.join(__dirname, '/known-versions-migration.lambda.bundle')), + }); + } +} \ No newline at end of file diff --git a/src/package-sources/npmjs/known-versions-table-migration.ts b/src/package-sources/npmjs/known-versions-table-migration.ts new file mode 100644 index 000000000..9f144fca0 --- /dev/null +++ b/src/package-sources/npmjs/known-versions-table-migration.ts @@ -0,0 +1,84 @@ +import { Duration } from 'aws-cdk-lib'; +import { ITableV2 } from 'aws-cdk-lib/aws-dynamodb'; +import { CfnFunction, Function, Tracing } from 'aws-cdk-lib/aws-lambda'; +import { IBucket } from 'aws-cdk-lib/aws-s3'; +import { InvocationType, Trigger } from 'aws-cdk-lib/triggers'; +import { Construct } from 'constructs'; +import { + ENV_KNOWN_VERSIONS_TABLE_NAME, + KNOWN_VERSIONS_FILE_NAME, + MARKER_FILE_NAME, +} from './constants.lambda-shared'; +import { KnownVersionsMigration } from './known-versions-migration'; + +export interface KnownVersionsTableMigrationProps { + /** The bucket holding the known versions file. */ + readonly bucket: IBucket; + + /** The table to copy the known versions to. */ + readonly table: ITableV2; + + /** The follower function, which is only updated once the copy completed. */ + readonly follower: Function; +} + +/** + * Copies the known versions of an existing deployment from the staging bucket + * to the known versions table, when the table is created. The copy runs in two + * passes: + * + * - Before the follower function is updated, so the new follower code never + * runs against an empty table. If this copy fails, the deployment fails and + * rolls back. + * - After the follower function is updated, to pick up the versions the old + * follower code recorded in the file while the first copy ran. + * + * Copying is idempotent, so the second pass only adds what is missing. + */ +export class KnownVersionsTableMigration extends Construct { + public constructor( + scope: Construct, + id: string, + props: KnownVersionsTableMigrationProps + ) { + super(scope, id); + + const handler = new KnownVersionsMigration(this, 'Function', { + description: `[${props.follower.node.path}] Copies the known versions from the staging bucket to the known versions table`, + environment: { + BUCKET_NAME: props.bucket.bucketName, + [ENV_KNOWN_VERSIONS_TABLE_NAME]: props.table.tableName, + }, + memorySize: 2_048, + timeout: Duration.minutes(10), + tracing: Tracing.ACTIVE, + }); + props.bucket.grantRead(handler, KNOWN_VERSIONS_FILE_NAME); + props.bucket.grantRead(handler, MARKER_FILE_NAME); + props.table.grantWriteData(handler); + + // The function resource only, not the follower's whole scope (which may + // contain this construct). + const followerFunction = props.follower.node.defaultChild as CfnFunction; + + new Trigger(this, 'BeforeFollowerUpdate', { + handler, + invocationType: InvocationType.REQUEST_RESPONSE, + // Longer than the function's own timeout (the trigger provider allows + // up to 15 minutes). + timeout: Duration.minutes(12), + // Only when the trigger is created, not on later updates. + executeOnHandlerChange: false, + executeAfter: [props.table], + executeBefore: [followerFunction], + }); + + new Trigger(this, 'AfterFollowerUpdate', { + handler, + // Nothing waits for this one; a failure is retried by Lambda. + invocationType: InvocationType.EVENT, + executeOnHandlerChange: false, + executeAfter: [followerFunction], + }); + } +} diff --git a/src/package-sources/npmjs/known-versions.lambda-shared.ts b/src/package-sources/npmjs/known-versions.lambda-shared.ts new file mode 100644 index 000000000..07282eaa6 --- /dev/null +++ b/src/package-sources/npmjs/known-versions.lambda-shared.ts @@ -0,0 +1,201 @@ +import { DynamoDB, WriteRequest } from '@aws-sdk/client-dynamodb'; +import { Sema } from 'async-sema'; +import * as AWSXRay from 'aws-xray-sdk-core'; +import { KnownVersionsAttribute } from './constants.lambda-shared'; + +/** The maximum number of concurrent requests to the table. */ +const MAX_CONCURRENT_REQUESTS = 32; + +/** The maximum number of items in a single `BatchWriteItem` request. */ +const BATCH_WRITE_SIZE = 25; + +/** How often unprocessed items of a batch write are retried. */ +const MAX_BATCH_WRITE_ATTEMPTS = 10; + +/** + * The DynamoDB client. Throttling errors are retried by the SDK; the default + * of 3 attempts is raised, since a burst of writes (for example during a + * backfill) can be throttled for longer than that. + */ +const DYNAMODB = AWSXRay.captureAWSv3Client(new DynamoDB({ maxAttempts: 10 })); + +/** + * The package versions the follower has processed, backed by a DynamoDB + * table keyed by package name and version. + * + * `has` loads all recorded versions of a package on first use and caches them. + * `add` records a version locally right away; `flush` persists the records, + * in batches. + */ +export class KnownVersions { + private readonly loads = new Map>>(); + private readonly added = new Set(); + private readonly pending = new Set(); + private readonly semaphore = new Sema(MAX_CONCURRENT_REQUESTS); + + public constructor( + private readonly tableName: string, + private readonly client: DynamoDB = DYNAMODB + ) {} + + /** + * Whether a package version has been processed. + */ + public async has(name: string, version: string): Promise { + if (this.added.has(keyOf(name, version))) { + return true; + } + return (await this.versionsOf(name)).has(version); + } + + /** + * Records a processed package version. The record is persisted by the next + * `flush`. + */ + public add(name: string, version: string): void { + const key = keyOf(name, version); + this.added.add(key); + this.pending.add(key); + } + + /** The number of records waiting to be persisted. */ + public get pendingCount(): number { + return this.pending.size; + } + + /** + * Persists all records made by `add` since the last flush. + */ + public async flush(): Promise { + const keys = Array.from(this.pending); + this.pending.clear(); + try { + const requests = keys.map((key) => { + const [name, version] = splitKey(key); + const request: WriteRequest = { + PutRequest: { Item: toItem(name, version) }, + }; + return request; + }); + const batches = new Array(); + for (let i = 0; i < requests.length; i += BATCH_WRITE_SIZE) { + batches.push(requests.slice(i, i + BATCH_WRITE_SIZE)); + } + await Promise.all( + batches.map((batch) => this.limited(() => this.batchWrite(batch))) + ); + } catch (error) { + // Keep the records for the next attempt (writes are idempotent). + for (const key of keys) { + this.pending.add(key); + } + throw error; + } + } + + /** + * The recorded versions of a package. Each package is queried at most once; + * concurrent callers share the same query. + */ + private versionsOf(name: string): Promise> { + let load = this.loads.get(name); + if (load == null) { + load = this.query(name); + this.loads.set(name, load); + // A failed query must not be cached. + load.catch(() => this.loads.delete(name)); + } + return load; + } + + private async query(name: string): Promise> { + const versions = new Set(); + let exclusiveStartKey: Parameters< + DynamoDB['query'] + >[0]['ExclusiveStartKey']; + do { + const response = await this.limited(() => + this.client.query({ + TableName: this.tableName, + ConsistentRead: true, + KeyConditionExpression: '#name = :name', + ProjectionExpression: '#version', + ExpressionAttributeNames: { + '#name': KnownVersionsAttribute.NAME, + '#version': KnownVersionsAttribute.VERSION, + }, + ExpressionAttributeValues: { ':name': { S: name } }, + ExclusiveStartKey: exclusiveStartKey, + }) + ); + for (const item of response.Items ?? []) { + const version = item[KnownVersionsAttribute.VERSION]?.S; + if (version != null) { + versions.add(version); + } + } + exclusiveStartKey = response.LastEvaluatedKey; + } while (exclusiveStartKey != null); + return versions; + } + + /** + * Writes a batch, retrying unprocessed items. The SDK retries throttling + * errors, but not unprocessed items (which DynamoDB returns when part of a + * batch was throttled). + */ + private async batchWrite(batch: WriteRequest[]): Promise { + let unprocessed: WriteRequest[] = batch; + for ( + let attempt = 1; + unprocessed.length > 0 && attempt <= MAX_BATCH_WRITE_ATTEMPTS; + attempt++ + ) { + if (attempt > 1) { + await sleep(Math.min(2 ** attempt * 50, 5_000)); + } + const response = await this.client.batchWriteItem({ + RequestItems: { [this.tableName]: unprocessed }, + }); + unprocessed = response.UnprocessedItems?.[this.tableName] ?? []; + } + if (unprocessed.length > 0) { + throw new Error( + `Could not write ${unprocessed.length} known version(s) to ${this.tableName} after ${MAX_BATCH_WRITE_ATTEMPTS} attempts` + ); + } + } + + private async limited(fn: () => Promise): Promise { + await this.semaphore.acquire(); + try { + return await fn(); + } finally { + this.semaphore.release(); + } + } +} + +function toItem(name: string, version: string) { + return { + [KnownVersionsAttribute.NAME]: { S: name }, + [KnownVersionsAttribute.VERSION]: { S: version }, + }; +} + +function keyOf(name: string, version: string): string { + return `${name}@${version}`; +} + +/** + * Splits a `name@version` key made by `keyOf`. Scoped package names start + * with `@`, so the separator is the last `@`. + */ +function splitKey(key: string): [name: string, version: string] { + const at = key.lastIndexOf('@'); + return [key.slice(0, at), key.slice(at + 1)]; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/package-sources/npmjs/npm-js-follower.lambda.ts b/src/package-sources/npmjs/npm-js-follower.lambda.ts index a7aa872b2..7fc296ab1 100644 --- a/src/package-sources/npmjs/npm-js-follower.lambda.ts +++ b/src/package-sources/npmjs/npm-js-follower.lambda.ts @@ -1,15 +1,5 @@ import * as console from 'console'; import { InvokeCommand } from '@aws-sdk/client-lambda'; -import { - GetObjectCommand, - NoSuchKey, - PutObjectCommand, - PutObjectCommandInput, -} from '@aws-sdk/client-s3'; -import type { - NodeJsRuntimeStreamingBlobPayloadOutputTypes, - StreamingBlobPayloadInputTypes, -} from '@smithy/types'; import { metricScope, Configuration, @@ -23,7 +13,7 @@ import { MetricName, MARKER_FILE_NAME, METRICS_NAMESPACE, - KNOWN_VERSIONS_FILE_NAME, + ENV_KNOWN_VERSIONS_TABLE_NAME, FOLLOWER_STATE_FILE_NAME, SCAN_WINDOW_MS, STATE_RETENTION_MS, @@ -36,17 +26,17 @@ import { parseSequentialRevision, } from './couch-changes.lambda-shared'; import { FollowerState } from './follower-state.lambda-shared'; +import { KnownVersions } from './known-versions.lambda-shared'; import { PackageVersion } from './stage-and-notify.lambda'; +import { + loadContentFromS3, + MarkerFileSchema, + putObject, +} from './staging-bucket.lambda-shared'; import { DenyListClient } from '../../backend/deny-list/client.lambda-shared'; import { LicenseListClient } from '../../backend/license-list/client.lambda-shared'; -import { - LAMBDA_CLIENT, - S3_CLIENT, -} from '../../backend/shared/aws.lambda-shared'; -import { - compressContent, - decompressContent, -} from '../../backend/shared/compress-content.lambda-shared'; +import { LAMBDA_CLIENT } from '../../backend/shared/aws.lambda-shared'; +import { compressContent } from '../../backend/shared/compress-content.lambda-shared'; import { requireEnv } from '../../backend/shared/env.lambda-shared'; // eslint-disable-next-line @typescript-eslint/no-require-imports const normalizeNPMMetadata = require('normalize-registry-metadata'); @@ -135,6 +125,9 @@ export async function handler(event: ScheduledEvent, context: Context) { const stagingBucket = requireEnv('BUCKET_NAME'); const stagingFunction = requireEnv('FUNCTION_NAME'); + const knownVersions = new KnownVersions( + requireEnv(ENV_KNOWN_VERSIONS_TABLE_NAME) + ); const denyList = await DenyListClient.newClient(); const licenseList = await LicenseListClient.newClient(); @@ -143,11 +136,6 @@ export async function handler(event: ScheduledEvent, context: Context) { const head = Number((await npm.info()).update_seq); - const { knownVersions, didLoadVersionsFromLegacy } = await loadKnownVersions( - stagingBucket - ); - let writeKnownVersionsFile = didLoadVersionsFromLegacy; - const state = await loadFollowerState(stagingBucket, head); // The feed's sequence space can change (entries can disappear from the end, @@ -164,7 +152,7 @@ export async function handler(event: ScheduledEvent, context: Context) { try { // Re-check laggy packuments (packages whose registry packument was behind // the revision announced by the feed when we first saw them). - const laggyStaged = await retryLaggyPackuments( + await retryLaggyPackuments( context, npm, state, @@ -173,19 +161,13 @@ export async function handler(event: ScheduledEvent, context: Context) { licenseList, knownVersions ); - writeKnownVersionsFile ||= laggyStaged > 0; // Scan the trailing window of the feed. Progress is persisted after // every processed chunk (see PersistState), so even a run that is hard // killed by the Lambda timeout keeps its completed work. - const persist: PersistState = async (saveKnownVersions) => - void (await Promise.all([ - saveFollowerState(context, stagingBucket, state), - ...(saveKnownVersions - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ])); - const scanStaged = await runScan( + const persist: PersistState = () => + persistProgress(context, stagingBucket, state, knownVersions); + await runScan( context, npm, state, @@ -196,7 +178,6 @@ export async function handler(event: ScheduledEvent, context: Context) { knownVersions, persist ); - writeKnownVersionsFile ||= scanStaged > 0; // The laggy packument gauge is emitted at the end of the run, so it // reflects the expectations recorded by this run's scan (emitting it during @@ -214,12 +195,7 @@ export async function handler(event: ScheduledEvent, context: Context) { // Persist the state even when a scan failed part-way: receipts, interim // checkpoints, and staged known versions represent completed work that // the next run should not redo. - await Promise.all([ - saveFollowerState(context, stagingBucket, state), - ...(writeKnownVersionsFile - ? [saveLastKnownVersions(context, stagingBucket, knownVersions)] - : []), - ]); + await persistProgress(context, stagingBucket, state, knownVersions); } console.log('All done here, we have success!'); @@ -236,7 +212,27 @@ export async function handler(event: ScheduledEvent, context: Context) { * backlog too large for a single execution would never complete (every run * would restart it from scratch). */ -type PersistState = (saveKnownVersions: boolean) => Promise; +type PersistState = () => Promise; + +/** + * Persists the known versions recorded since the last call, then the follower + * state. Known versions go first: if saving the state fails, the next run + * redoes the work, and the known versions prevent it from staging the same + * versions again. The reverse order could record receipts for entries whose + * staged versions were never recorded as known. + */ +async function persistProgress( + context: Context, + stagingBucket: string, + state: FollowerState, + knownVersions: KnownVersions +): Promise { + if (knownVersions.pendingCount > 0) { + console.log(`Recording ${knownVersions.pendingCount} known version(s)`); + await knownVersions.flush(); + } + await saveFollowerState(context, stagingBucket, state); +} /** * Re-reads the trailing window of the feed and processes every change entry @@ -254,7 +250,7 @@ async function runScan( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map, + knownVersions: KnownVersions, persist: PersistState ): Promise { const now = Date.now(); @@ -328,7 +324,7 @@ async function runScan( // importantly: a backfill) resume from here instead of the floor. if (next > (state.newestCheckpointSeq() ?? 0)) { state.addCheckpoint(next); - await persist(false); + await persist(); } } })(); @@ -382,7 +378,7 @@ async function processBatch( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map, + knownVersions: KnownVersions, persist: PersistState ): Promise<{ staged: number; fullyProcessed: boolean }> { const startTime = Date.now(); @@ -475,7 +471,7 @@ async function processBatch( ); } - const versionInfos = getRelevantVersionInfos( + const versionInfos = await getRelevantVersionInfos( attached.map((a) => a.change) as unknown as readonly Change[], metrics, denyList, @@ -505,7 +501,7 @@ async function processBatch( .map((change) => Number(change.seq)) .filter((seq) => !isNaN(seq) && !failed.has(seq)) ); - await persist(chunkStaged > 0); + await persist(); } } finally { metrics.putMetric(MetricName.LAST_SEQ, head, Unit.None); @@ -542,7 +538,7 @@ async function retryLaggyPackuments( stagingFunction: string, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map + knownVersions: KnownVersions ): Promise { let staged = 0; await metricScope((metrics) => async () => { @@ -573,7 +569,7 @@ async function retryLaggyPackuments( seq: laggy.seq, doc, }; - const versionInfos = getRelevantVersionInfos( + const versionInfos = await getRelevantVersionInfos( [change] as unknown as readonly Change[], metrics, denyList, @@ -621,10 +617,10 @@ async function retryLaggyPackuments( async function stageVersions( versionInfos: readonly UpdatedVersion[], stagingFunction: string, - knownVersions: Map + knownVersions: KnownVersions ): Promise { await Promise.all( - versionInfos.map(async ({ infos, modified, seq }) => { + versionInfos.map(async ({ packageName, infos, modified, seq }) => { const invokeArgs: PackageVersion = { integrity: infos.dist.shasum, modified: modified.toISOString(), @@ -643,49 +639,12 @@ async function stageVersions( }) ); // Record that this is now a "known" version (no need to re-discover) - knownVersions.set(`${infos.name}@${infos.version}`, modified); + knownVersions.add(packageName, infos.version); }) ); } -//#region State and known versions -/** - * Common function to load data from an S3 file with error handling - * - * @param stagingBucket The S3 bucket name - * @param key The file key in the bucket - * @param warningMessage Message to log when file doesn't exist - * @returns The decompressed file content as string, or null if file doesn't exist - */ -async function loadContentFromS3( - stagingBucket: string, - key: string, - warningMessage: string -): Promise { - try { - const response = await S3_CLIENT.send( - new GetObjectCommand({ - Bucket: stagingBucket, - Key: key, - }) - ); - if (!response.Body) { - throw new Error(`Response Body for ${key} is empty`); - } - return await decompressContent( - response.Body as NodeJsRuntimeStreamingBlobPayloadOutputTypes, - response.ContentEncoding - ); - } catch (error: any) { - if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { - console.warn(warningMessage); - return null; - } - // re-throw unexpected errors - throw error; - } -} - +//#region State /** * Loads the follower state from S3. When the state file does not exist (or * cannot be parsed), a fresh state is seeded: from the legacy transaction @@ -778,113 +737,6 @@ async function saveFollowerState( console.log('Successfully updated follower state'); } -/** - * Loads the known versions from S3. For legacy reasons, the map may also be - * embedded in the transaction marker file; when the dedicated file does not - * exist, that location is used and the map is migrated on the next save. - */ -async function loadKnownVersions(stagingBucket: string): Promise<{ - knownVersions: Map; - didLoadVersionsFromLegacy: boolean; -}> { - const warningMessage = `Known versions object (s3://${stagingBucket}/${KNOWN_VERSIONS_FILE_NAME}) does not exist, starting from scratch`; - const content = await loadContentFromS3( - stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - warningMessage - ); - if (content != null) { - const contentsObj: KnownVersionsFileSchema = JSON.parse(content); - console.log('Loaded last known versions data'); - return { - knownVersions: dateMapFromObj(contentsObj.knownVersions), - didLoadVersionsFromLegacy: false, - }; - } - - // Legacy location: embedded in the transaction marker file. - const markerContent = await loadContentFromS3( - stagingBucket, - MARKER_FILE_NAME, - `No legacy marker object (s3://${stagingBucket}/${MARKER_FILE_NAME})` - ); - if (markerContent != null) { - try { - const parsed: MarkerFileSchema = JSON.parse(markerContent); - if (typeof parsed !== 'number' && parsed.knownVersions) { - console.log('Loaded known versions from legacy marker file'); - return { - knownVersions: dateMapFromObj(parsed.knownVersions), - didLoadVersionsFromLegacy: true, - }; - } - } catch (error) { - console.warn(`Could not parse legacy marker: ${error}`); - } - } - return { knownVersions: new Map(), didLoadVersionsFromLegacy: true }; -} - -/** - * Updates the last known versions in S3. - * - * @param knownVersions the map of package name + version to last modified timestamp of packages that have been processed. - */ -async function saveLastKnownVersions( - context: Context, - stagingBucket: string, - knownVersions: Map -) { - const contentsObj: KnownVersionsFileSchema = { - knownVersions: objFromDateMap(knownVersions), - }; - - console.log(`Known versions changed, updating...`); - await putObject( - context, - stagingBucket, - KNOWN_VERSIONS_FILE_NAME, - JSON.stringify(contentsObj, undefined, 2), - { - ContentType: 'application/json', - } - ); - console.log('Successfully updated known versions'); -} -//#endregion - -//#region Asynchronous Primitives -/** - * Puts an object in the staging bucket, with standardized object metadata. - * - * @param key the key for the object to be put. - * @param body the body of the object to be put. - * @param opts any other options to use when sending the S3 request. - * - * @returns the result of the S3 request. - */ -function putObject( - context: Context, - bucket: string, - key: string, - body: StreamingBlobPayloadInputTypes, - opts: Omit = {} -) { - return S3_CLIENT.send( - new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { - 'Lambda-Log-Group': context.logGroupName, - 'Lambda-Log-Stream': context.logStreamName, - 'Lambda-Run-Id': context.awsRequestId, - ...opts.Metadata, - }, - ...opts, - }) - ); -} //#endregion /** @@ -892,22 +744,32 @@ function putObject( * provided `Change` objects, ensures they are relevant (construct libraries), * and returns those only. * + * Known versions are only looked up for packages that have at least one + * construct library version: only those versions are ever recorded, so every + * other package has no known versions by definition. Changes are processed + * concurrently; the result keeps their order. + * * @param changes the changes to be processed. * @param metrics the metrics logger to use. * @param denyList deny list client * * @returns a list of `VersionInfo` objects */ -function getRelevantVersionInfos( +async function getRelevantVersionInfos( changes: readonly Change[], metrics: MetricsLogger, denyList: DenyListClient, licenseList: LicenseListClient, - knownVersions: Map -): readonly UpdatedVersion[] { - const result = new Array(); + knownVersions: KnownVersions +): Promise { + const perChange = await Promise.all( + changes.map((change) => relevantVersionInfosOf(change)) + ); + return perChange.flat(); - for (const change of changes) { + async function relevantVersionInfosOf( + change: Change + ): Promise { // Filter out all elements that don't have a "name" in the document, as // these are schemas, which are not relevant to our business here. if (change.doc.name === undefined) { @@ -915,7 +777,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'name': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // The normalize function change the object in place, if the doc object is invalid it will return undefined @@ -924,7 +786,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document invalid, npm normalize returned undefined: ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // Sometimes, there are no versions in the document. We skip those. @@ -933,7 +795,7 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'versions': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } // Sometimes, there is no 'time' entry in the document. We skip those. @@ -942,9 +804,14 @@ function getRelevantVersionInfos( `[${change.seq}] Changed document contains no 'time': ${change.id}` ); metrics.putMetric(MetricName.UNPROCESSABLE_ENTITY, 1, Unit.Count); - continue; + return []; } + const name = change.doc.name; + const isRelevant = Object.values(change.doc.versions).some( + (infos) => infos != null && isConstructLibrary(infos) + ); + // Get the last modification date from the change const packageVersionUpdates = Object.entries(change.doc.time) // Ignore the "created" and "modified" keys here @@ -957,54 +824,59 @@ function getRelevantVersionInfos( Unit.Count ); + const result = new Array(); const unpublishedVersions: string[] = []; for (const [version, modified] of packageVersionUpdates) { - const knownKey = `${change.doc.name}@${version}`; - const known = knownVersions.get(knownKey); - if (known == null || known < modified) { - const infos = change.doc.versions[version]; - if (infos == null) { - // Could be the version in question was un-published. - unpublishedVersions.push(knownKey); - } else if (isConstructLibrary(infos)) { - // skip if this package is denied - const denied = denyList.lookup(infos.name, infos.version); - if (denied) { - console.log( - `[${change.seq}] Package denied: ${JSON.stringify(denied)}` - ); - knownVersions.set(knownKey, modified); - metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); - continue; - } - - metrics.putMetric( - MetricName.PACKAGE_VERSION_AGE, - Date.now() - modified.getTime(), - Unit.Milliseconds + if (isRelevant && (await knownVersions.has(name, version))) { + continue; + } + const infos = change.doc.versions[version]; + if (infos == null) { + // Could be the version in question was un-published. + unpublishedVersions.push(`${name}@${version}`); + } else if (isConstructLibrary(infos)) { + // skip if this package is denied + const denied = denyList.lookup(infos.name, infos.version); + if (denied) { + console.log( + `[${change.seq}] Package denied: ${JSON.stringify(denied)}` ); - const isEligible = - licenseList.lookup(infos.license ?? 'UNLICENSED') != null; - metrics.putMetric( - MetricName.INELIGIBLE_LICENSE, - isEligible ? 0 : 1, - Unit.Count + knownVersions.add(name, version); + metrics.putMetric(MetricName.DENY_LISTED_COUNT, 1, Unit.Count); + continue; + } + + metrics.putMetric( + MetricName.PACKAGE_VERSION_AGE, + Date.now() - modified.getTime(), + Unit.Milliseconds + ); + const isEligible = + licenseList.lookup(infos.license ?? 'UNLICENSED') != null; + metrics.putMetric( + MetricName.INELIGIBLE_LICENSE, + isEligible ? 0 : 1, + Unit.Count + ); + if (isEligible) { + result.push({ + packageName: name, + infos, + modified, + seq: change.seq, + }); + } else { + console.log( + `[${ + change.seq + }] Package "${name}@${version}" does not use allow-listed license: ${ + infos.license ?? 'UNLICENSED' + }` ); - if (isEligible) { - result.push({ infos, modified, seq: change.seq }); - } else { - console.log( - `[${change.seq}] Package "${ - change.doc.name - }@${version}" does not use allow-listed license: ${ - infos.license ?? 'UNLICENSED' - }` - ); - knownVersions.set(knownKey, modified); - } + knownVersions.add(name, version); } - // Else this is not a construct library, so we'll just ignore it... } + // Else this is not a construct library, so we'll just ignore it... } if (unpublishedVersions.length > 0) { @@ -1016,8 +888,8 @@ function getRelevantVersionInfos( )}` ); } + return result; } - return result; /** * This determines whether a package is "interesting" to ConstructHub or not. This is related but @@ -1076,18 +948,6 @@ function getRelevantVersionInfos( } } -function objFromDateMap(xs: Map): Record { - return Object.fromEntries( - Array.from(xs).map(([k, v]) => [k, v.toISOString()]) - ); -} - -function dateMapFromObj( - xs: Record -): Map { - return new Map(Object.entries(xs).map(([k, v]) => [k, new Date(v)])); -} - /** * The scheme of a package version in the update. Includes the package.json keys, as well as some additional npm metadata * @see https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md#version @@ -1109,6 +969,12 @@ export interface VersionInfo { } interface UpdatedVersion { + /** + * The name of the package document the version was found in (the key under + * which known versions are recorded). + */ + readonly packageName: string; + /** * The `VersionInfo` for the modified package version. */ @@ -1125,26 +991,6 @@ interface UpdatedVersion { readonly seq?: string | number; } -/** - * The legacy transaction marker file format, kept around to seed the follower - * state (and known versions) of existing deployments. - * - * The file can be a just a number, or a combination of a sequence number (which - * is potentially encoded as a string) and a set of known versions and the dates - * we first saw them. The date can be encoded as an ISO string, or a timestamp - * number. - */ -type MarkerFileSchema = - | number - | { - marker: number | string; - knownVersions?: Record; - }; - -interface KnownVersionsFileSchema { - knownVersions: Record; -} - interface Document { /** * a List of all Version objects for the package diff --git a/src/package-sources/npmjs/staging-bucket.lambda-shared.ts b/src/package-sources/npmjs/staging-bucket.lambda-shared.ts new file mode 100644 index 000000000..c25103f80 --- /dev/null +++ b/src/package-sources/npmjs/staging-bucket.lambda-shared.ts @@ -0,0 +1,98 @@ +import { + GetObjectCommand, + NoSuchKey, + PutObjectCommand, + PutObjectCommandInput, +} from '@aws-sdk/client-s3'; +import type { + NodeJsRuntimeStreamingBlobPayloadOutputTypes, + StreamingBlobPayloadInputTypes, +} from '@smithy/types'; +import type { Context } from 'aws-lambda'; +import { S3_CLIENT } from '../../backend/shared/aws.lambda-shared'; +import { decompressContent } from '../../backend/shared/compress-content.lambda-shared'; + +/** + * The legacy transaction marker file format, kept around to seed the follower + * state (and known versions) of existing deployments. + * + * The file can be a just a number, or a combination of a sequence number (which + * is potentially encoded as a string) and a set of known versions and the dates + * we first saw them. The date can be encoded as an ISO string, or a timestamp + * number. + */ +export type MarkerFileSchema = + | number + | { + marker: number | string; + knownVersions?: Record; + }; + +/** + * Common function to load data from an S3 file with error handling + * + * @param stagingBucket The S3 bucket name + * @param key The file key in the bucket + * @param warningMessage Message to log when file doesn't exist + * @returns The decompressed file content as string, or null if file doesn't exist + */ +export async function loadContentFromS3( + stagingBucket: string, + key: string, + warningMessage: string +): Promise { + try { + const response = await S3_CLIENT.send( + new GetObjectCommand({ + Bucket: stagingBucket, + Key: key, + }) + ); + if (!response.Body) { + throw new Error(`Response Body for ${key} is empty`); + } + return await decompressContent( + response.Body as NodeJsRuntimeStreamingBlobPayloadOutputTypes, + response.ContentEncoding + ); + } catch (error: any) { + if (error instanceof NoSuchKey || error.name === 'NoSuchKey') { + console.warn(warningMessage); + return null; + } + // re-throw unexpected errors + throw error; + } +} + +/** + * Puts an object in the staging bucket, with standardized object metadata. + * + * @param key the key for the object to be put. + * @param body the body of the object to be put. + * @param opts any other options to use when sending the S3 request. + * + * @returns the result of the S3 request. + */ +export function putObject( + context: Context, + bucket: string, + key: string, + body: StreamingBlobPayloadInputTypes, + opts: Omit = {} +) { + return S3_CLIENT.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + 'Lambda-Log-Group': context.logGroupName, + 'Lambda-Log-Stream': context.logStreamName, + 'Lambda-Run-Id': context.awsRequestId, + ...opts.Metadata, + }, + ...opts, + }) + ); +} diff --git a/yarn.lock b/yarn.lock index d1d452e99..7ad137d2e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -155,6 +155,24 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/client-dynamodb@npm:^3.1141.0": + version: 3.1146.0 + resolution: "@aws-sdk/client-dynamodb@npm:3.1146.0" + dependencies: + "@aws-sdk/core": "npm:^3.978.1" + "@aws-sdk/credential-provider-node": "npm:^3.972.84" + "@aws-sdk/dynamodb-codec": "npm:^3.973.46" + "@aws-sdk/middleware-endpoint-discovery": "npm:^3.972.31" + "@aws-sdk/types": "npm:^3.974.6" + "@smithy/core": "npm:^3.35.0" + "@smithy/fetch-http-handler": "npm:^5.8.0" + "@smithy/node-http-handler": "npm:^4.12.1" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/764f9df99ffbbed62f241cab101bc41ac1874744b27f4825f571206f59ba5ea21293717eeac3f89c56c29c35e30c35f77fb9c4b3a9b4cc53f96e3dd57845f940 + languageName: node + linkType: hard + "@aws-sdk/client-ecs@npm:^3.1141.0": version: 3.1141.0 resolution: "@aws-sdk/client-ecs@npm:3.1141.0" @@ -379,6 +397,41 @@ __metadata: languageName: node linkType: hard +"@aws-sdk/dynamodb-codec@npm:^3.973.46": + version: 3.973.46 + resolution: "@aws-sdk/dynamodb-codec@npm:3.973.46" + dependencies: + "@aws-sdk/core": "npm:^3.978.1" + "@smithy/core": "npm:^3.35.0" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/34343d4a082807748e6bb0b5bbf411ce6616d07359317a281419f2ad63362ce4fe25972a2e8c93dc0ff5fb6ea66e65f5abb9e9d12607458266cd95e43c1d352c + languageName: node + linkType: hard + +"@aws-sdk/endpoint-cache@npm:^3.972.11": + version: 3.972.11 + resolution: "@aws-sdk/endpoint-cache@npm:3.972.11" + dependencies: + mnemonist: "npm:0.38.3" + tslib: "npm:^2.6.2" + checksum: 10c0/8f3c039fb2dd8e434cd2992bce2a840e6f6470be5ae6d71c762f6cb842149fb60d5566ca470319d5abcf90b115d2c1a0114ed3328afbfc0abed4cbcd52ad03a1 + languageName: node + linkType: hard + +"@aws-sdk/middleware-endpoint-discovery@npm:^3.972.31": + version: 3.972.31 + resolution: "@aws-sdk/middleware-endpoint-discovery@npm:3.972.31" + dependencies: + "@aws-sdk/endpoint-cache": "npm:^3.972.11" + "@aws-sdk/types": "npm:^3.974.6" + "@smithy/core": "npm:^3.35.0" + "@smithy/types": "npm:^4.19.0" + tslib: "npm:^2.6.2" + checksum: 10c0/ede83171b14fa392b6b499d6fd8fe5554cd0b4a0468df99362cf82eeb2d907eab9f76973a00e8a321fc55ff167c4774697029121745748b0ca2334dc9c246f36 + languageName: node + linkType: hard + "@aws-sdk/middleware-sdk-s3@npm:^3.972.77": version: 3.972.77 resolution: "@aws-sdk/middleware-sdk-s3@npm:3.972.77" @@ -5246,6 +5299,7 @@ __metadata: "@aws-cdk/integ-tests-alpha": "npm:2.194.0-alpha.0" "@aws-sdk/client-cloudwatch": "npm:^3.1141.0" "@aws-sdk/client-codeartifact": "npm:^3.1141.0" + "@aws-sdk/client-dynamodb": "npm:^3.1141.0" "@aws-sdk/client-ecs": "npm:^3.1141.0" "@aws-sdk/client-lambda": "npm:^3.1141.0" "@aws-sdk/client-s3": "npm:^3.1141.0" @@ -9619,6 +9673,15 @@ __metadata: languageName: node linkType: hard +"mnemonist@npm:0.38.3": + version: 0.38.3 + resolution: "mnemonist@npm:0.38.3" + dependencies: + obliterator: "npm:^1.6.1" + checksum: 10c0/064aa1ee1a89fce2754423b3617c598fd65bc34311eb3c01dc063976f6b819b073bd23532415cf8c92240157b4c8fbb7ec5d79d717f2bd4fcd95d8131cb23acb + languageName: node + linkType: hard + "modify-values@npm:^1.0.1": version: 1.0.1 resolution: "modify-values@npm:1.0.1" @@ -9906,6 +9969,13 @@ __metadata: languageName: node linkType: hard +"obliterator@npm:^1.6.1": + version: 1.6.1 + resolution: "obliterator@npm:1.6.1" + checksum: 10c0/5fad57319aae0ef6e34efa640541d41c2dd9790a7ab808f17dcb66c83a81333963fc2dfcfa6e1b62158e5cef6291cdcf15c503ad6c3de54b2227dd4c3d7e1b55 + languageName: node + linkType: hard + "oblivious-set@npm:1.0.0": version: 1.0.0 resolution: "oblivious-set@npm:1.0.0"